Diff of the two buildlogs: -- --- b1/build.log 2021-10-31 23:21:36.699774575 +0000 +++ b2/build.log 2021-10-31 23:29:37.695302932 +0000 @@ -1,6 +1,6 @@ I: pbuilder: network access will be disabled during build -I: Current time: Sun Oct 31 11:02:21 -12 2021 -I: pbuilder-time-stamp: 1635721341 +I: Current time: Sun Dec 4 19:44:40 +14 2022 +I: pbuilder-time-stamp: 1670132680 I: Building the build Environment I: extracting base tarball [/var/cache/pbuilder/bullseye-reproducible-base.tgz] I: copying local configuration @@ -16,8 +16,8 @@ I: copying [./consul_1.8.7+dfsg1-2.debian.tar.xz] I: Extracting source gpgv: unknown type of key resource 'trustedkeys.kbx' -gpgv: keyblock resource '/tmp/dpkg-verify-sig.pR6o03sR/trustedkeys.kbx': General error -gpgv: Signature made Fri Apr 23 23:25:37 2021 -12 +gpgv: keyblock resource '/tmp/dpkg-verify-sig.uUOuzHEo/trustedkeys.kbx': General error +gpgv: Signature made Sun Apr 25 01:25:37 2021 +14 gpgv: using RSA key C5A5B9DDC33D93FBB63D67C83287D89A97CDA87B gpgv: issuer "vvidic@debian.org" gpgv: Can't check signature: No public key @@ -31,135 +31,169 @@ dpkg-source: info: applying CVE-2020-25864.patch I: using fakeroot in build. I: Installing the build-deps -I: user script /srv/workspace/pbuilder/2649205/tmp/hooks/D02_print_environment starting +I: user script /srv/workspace/pbuilder/1426341/tmp/hooks/D01_modify_environment starting +debug: Running on ionos5-amd64. +I: Changing host+domainname to test build reproducibility +I: Adding a custom variable just for the fun of it... +I: Changing /bin/sh to bash +Removing 'diversion of /bin/sh to /bin/sh.distrib by dash' +Adding 'diversion of /bin/sh to /bin/sh.distrib by bash' +Removing 'diversion of /usr/share/man/man1/sh.1.gz to /usr/share/man/man1/sh.distrib.1.gz by dash' +Adding 'diversion of /usr/share/man/man1/sh.1.gz to /usr/share/man/man1/sh.distrib.1.gz by bash' +I: Setting pbuilder2's login shell to /bin/bash +I: Setting pbuilder2's GECOS to second user,second room,second work-phone,second home-phone,second other +I: user script /srv/workspace/pbuilder/1426341/tmp/hooks/D01_modify_environment finished +I: user script /srv/workspace/pbuilder/1426341/tmp/hooks/D02_print_environment starting I: set - BUILDDIR='/build' - BUILDUSERGECOS='first user,first room,first work-phone,first home-phone,first other' - BUILDUSERNAME='pbuilder1' - BUILD_ARCH='amd64' - DEBIAN_FRONTEND='noninteractive' - DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all,-fixfilepath parallel=15' - DISTRIBUTION='' - HOME='/root' - HOST_ARCH='amd64' + BASH=/bin/sh + BASHOPTS=checkwinsize:cmdhist:complete_fullquote:extquote:force_fignore:globasciiranges:hostcomplete:interactive_comments:progcomp:promptvars:sourcepath + BASH_ALIASES=() + BASH_ARGC=() + BASH_ARGV=() + BASH_CMDS=() + BASH_LINENO=([0]="12" [1]="0") + BASH_SOURCE=([0]="/tmp/hooks/D02_print_environment" [1]="/tmp/hooks/D02_print_environment") + BASH_VERSINFO=([0]="5" [1]="1" [2]="4" [3]="1" [4]="release" [5]="x86_64-pc-linux-gnu") + BASH_VERSION='5.1.4(1)-release' + BUILDDIR=/build + BUILDUSERGECOS='second user,second room,second work-phone,second home-phone,second other' + BUILDUSERNAME=pbuilder2 + BUILD_ARCH=amd64 + DEBIAN_FRONTEND=noninteractive + DEB_BUILD_OPTIONS='buildinfo=+all reproducible=+all,-fixfilepath parallel=16' + DIRSTACK=() + DISTRIBUTION= + EUID=0 + FUNCNAME=([0]="Echo" [1]="main") + GROUPS=() + HOME=/root + HOSTNAME=i-capture-the-hostname + HOSTTYPE=x86_64 + HOST_ARCH=amd64 IFS=' ' - INVOCATION_ID='1826b52c58f34820953525ef7eef82ed' - LANG='C' - LANGUAGE='en_US:en' - LC_ALL='C' - MAIL='/var/mail/root' - OPTIND='1' - PATH='/usr/sbin:/usr/bin:/sbin:/bin:/usr/games' - PBCURRENTCOMMANDLINEOPERATION='build' - PBUILDER_OPERATION='build' - PBUILDER_PKGDATADIR='/usr/share/pbuilder' - PBUILDER_PKGLIBDIR='/usr/lib/pbuilder' - PBUILDER_SYSCONFDIR='/etc' - PPID='2649205' - PS1='# ' - PS2='> ' + INVOCATION_ID=b6125863832147e680ec1b346314e138 + LANG=C + LANGUAGE=et_EE:et + LC_ALL=C + MACHTYPE=x86_64-pc-linux-gnu + MAIL=/var/mail/root + OPTERR=1 + OPTIND=1 + OSTYPE=linux-gnu + PATH=/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path + PBCURRENTCOMMANDLINEOPERATION=build + PBUILDER_OPERATION=build + PBUILDER_PKGDATADIR=/usr/share/pbuilder + PBUILDER_PKGLIBDIR=/usr/lib/pbuilder + PBUILDER_SYSCONFDIR=/etc + PIPESTATUS=([0]="0") + POSIXLY_CORRECT=y + PPID=1426341 PS4='+ ' - PWD='/' - SHELL='/bin/bash' - SHLVL='2' - SUDO_COMMAND='/usr/bin/timeout -k 18.1h 18h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/tmp.qdGrYdlvhD/pbuilderrc_sowt --hookdir /etc/pbuilder/first-build-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/bullseye-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/tmp.qdGrYdlvhD/b1 --logfile b1/build.log consul_1.8.7+dfsg1-2.dsc' - SUDO_GID='111' - SUDO_UID='106' - SUDO_USER='jenkins' - TERM='unknown' - TZ='/usr/share/zoneinfo/Etc/GMT+12' - USER='root' - _='/usr/bin/systemd-run' - http_proxy='http://78.137.99.97:3128' + PWD=/ + SHELL=/bin/bash + SHELLOPTS=braceexpand:errexit:hashall:interactive-comments:posix + SHLVL=3 + SUDO_COMMAND='/usr/bin/timeout -k 24.1h 24h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder --build --configfile /srv/reproducible-results/rbuild-debian/tmp.qdGrYdlvhD/pbuilderrc_Zaq7 --hookdir /etc/pbuilder/rebuild-hooks --debbuildopts -b --basetgz /var/cache/pbuilder/bullseye-reproducible-base.tgz --buildresult /srv/reproducible-results/rbuild-debian/tmp.qdGrYdlvhD/b2 --logfile b2/build.log consul_1.8.7+dfsg1-2.dsc' + SUDO_GID=110 + SUDO_UID=105 + SUDO_USER=jenkins + TERM=unknown + TZ=/usr/share/zoneinfo/Etc/GMT-14 + UID=0 + USER=root + _='I: set' + http_proxy=http://85.184.249.68:3128 I: uname -a - Linux ionos11-amd64 5.10.0-9-amd64 #1 SMP Debian 5.10.70-1 (2021-09-30) x86_64 GNU/Linux + Linux i-capture-the-hostname 5.14.0-0.bpo.2-amd64 #1 SMP Debian 5.14.9-2~bpo11+1 (2021-10-10) x86_64 GNU/Linux I: ls -l /bin total 5476 - -rwxr-xr-x 1 root root 1234376 Aug 4 08:25 bash - -rwxr-xr-x 3 root root 38984 Jul 20 2020 bunzip2 - -rwxr-xr-x 3 root root 38984 Jul 20 2020 bzcat - lrwxrwxrwx 1 root root 6 Jul 20 2020 bzcmp -> bzdiff - -rwxr-xr-x 1 root root 2225 Jul 20 2020 bzdiff - lrwxrwxrwx 1 root root 6 Jul 20 2020 bzegrep -> bzgrep - -rwxr-xr-x 1 root root 4877 Sep 4 2019 bzexe - lrwxrwxrwx 1 root root 6 Jul 20 2020 bzfgrep -> bzgrep - -rwxr-xr-x 1 root root 3775 Jul 20 2020 bzgrep - -rwxr-xr-x 3 root root 38984 Jul 20 2020 bzip2 - -rwxr-xr-x 1 root root 18424 Jul 20 2020 bzip2recover - lrwxrwxrwx 1 root root 6 Jul 20 2020 bzless -> bzmore - -rwxr-xr-x 1 root root 1297 Jul 20 2020 bzmore - -rwxr-xr-x 1 root root 43936 Sep 23 2020 cat - -rwxr-xr-x 1 root root 72672 Sep 23 2020 chgrp - -rwxr-xr-x 1 root root 64448 Sep 23 2020 chmod - -rwxr-xr-x 1 root root 72672 Sep 23 2020 chown - -rwxr-xr-x 1 root root 151168 Sep 23 2020 cp - -rwxr-xr-x 1 root root 125560 Dec 10 2020 dash - -rwxr-xr-x 1 root root 113664 Sep 23 2020 date - -rwxr-xr-x 1 root root 80968 Sep 23 2020 dd - -rwxr-xr-x 1 root root 93936 Sep 23 2020 df - -rwxr-xr-x 1 root root 147176 Sep 23 2020 dir - -rwxr-xr-x 1 root root 84440 Jul 28 07:09 dmesg - lrwxrwxrwx 1 root root 8 Nov 6 2019 dnsdomainname -> hostname - lrwxrwxrwx 1 root root 8 Nov 6 2019 domainname -> hostname - -rwxr-xr-x 1 root root 39712 Sep 23 2020 echo - -rwxr-xr-x 1 root root 28 Nov 9 2020 egrep - -rwxr-xr-x 1 root root 39680 Sep 23 2020 false - -rwxr-xr-x 1 root root 28 Nov 9 2020 fgrep - -rwxr-xr-x 1 root root 69032 Jul 28 07:09 findmnt - -rwsr-xr-x 1 root root 34896 Feb 26 2021 fusermount - -rwxr-xr-x 1 root root 203072 Nov 9 2020 grep - -rwxr-xr-x 2 root root 2346 Mar 2 2021 gunzip - -rwxr-xr-x 1 root root 6376 Mar 2 2021 gzexe - -rwxr-xr-x 1 root root 98048 Mar 2 2021 gzip - -rwxr-xr-x 1 root root 22600 Nov 6 2019 hostname - -rwxr-xr-x 1 root root 72840 Sep 23 2020 ln - -rwxr-xr-x 1 root root 56952 Feb 7 2020 login - -rwxr-xr-x 1 root root 147176 Sep 23 2020 ls - -rwxr-xr-x 1 root root 149736 Jul 28 07:09 lsblk - -rwxr-xr-x 1 root root 85184 Sep 23 2020 mkdir - -rwxr-xr-x 1 root root 76896 Sep 23 2020 mknod - -rwxr-xr-x 1 root root 48064 Sep 23 2020 mktemp - -rwxr-xr-x 1 root root 59632 Jul 28 07:09 more - -rwsr-xr-x 1 root root 55528 Jul 28 07:09 mount - -rwxr-xr-x 1 root root 18664 Jul 28 07:09 mountpoint - -rwxr-xr-x 1 root root 147080 Sep 23 2020 mv - lrwxrwxrwx 1 root root 8 Nov 6 2019 nisdomainname -> hostname - lrwxrwxrwx 1 root root 14 Apr 18 2021 pidof -> /sbin/killall5 - -rwxr-xr-x 1 root root 43872 Sep 23 2020 pwd - lrwxrwxrwx 1 root root 4 Aug 4 08:25 rbash -> bash - -rwxr-xr-x 1 root root 52032 Sep 23 2020 readlink - -rwxr-xr-x 1 root root 72704 Sep 23 2020 rm - -rwxr-xr-x 1 root root 52032 Sep 23 2020 rmdir - -rwxr-xr-x 1 root root 27472 Sep 27 2020 run-parts - -rwxr-xr-x 1 root root 122224 Dec 22 2018 sed - lrwxrwxrwx 1 root root 4 Oct 20 23:41 sh -> dash - -rwxr-xr-x 1 root root 43808 Sep 23 2020 sleep - -rwxr-xr-x 1 root root 84928 Sep 23 2020 stty - -rwsr-xr-x 1 root root 71912 Jul 28 07:09 su - -rwxr-xr-x 1 root root 39744 Sep 23 2020 sync - -rwxr-xr-x 1 root root 531928 Feb 16 2021 tar - -rwxr-xr-x 1 root root 14456 Sep 27 2020 tempfile - -rwxr-xr-x 1 root root 101408 Sep 23 2020 touch - -rwxr-xr-x 1 root root 39680 Sep 23 2020 true - -rwxr-xr-x 1 root root 14328 Feb 26 2021 ulockmgr_server - -rwsr-xr-x 1 root root 35040 Jul 28 07:09 umount - -rwxr-xr-x 1 root root 39744 Sep 23 2020 uname - -rwxr-xr-x 2 root root 2346 Mar 2 2021 uncompress - -rwxr-xr-x 1 root root 147176 Sep 23 2020 vdir - -rwxr-xr-x 1 root root 63744 Jul 28 07:09 wdctl - lrwxrwxrwx 1 root root 8 Nov 6 2019 ypdomainname -> hostname - -rwxr-xr-x 1 root root 1984 Mar 2 2021 zcat - -rwxr-xr-x 1 root root 1678 Mar 2 2021 zcmp - -rwxr-xr-x 1 root root 5880 Mar 2 2021 zdiff - -rwxr-xr-x 1 root root 29 Mar 2 2021 zegrep - -rwxr-xr-x 1 root root 29 Mar 2 2021 zfgrep - -rwxr-xr-x 1 root root 2081 Mar 2 2021 zforce - -rwxr-xr-x 1 root root 7585 Mar 2 2021 zgrep - -rwxr-xr-x 1 root root 2206 Mar 2 2021 zless - -rwxr-xr-x 1 root root 1842 Mar 2 2021 zmore - -rwxr-xr-x 1 root root 4553 Mar 2 2021 znew -I: user script /srv/workspace/pbuilder/2649205/tmp/hooks/D02_print_environment finished + -rwxr-xr-x 1 root root 1234376 Aug 5 2021 bash + -rwxr-xr-x 3 root root 38984 Jul 21 2020 bunzip2 + -rwxr-xr-x 3 root root 38984 Jul 21 2020 bzcat + lrwxrwxrwx 1 root root 6 Jul 21 2020 bzcmp -> bzdiff + -rwxr-xr-x 1 root root 2225 Jul 21 2020 bzdiff + lrwxrwxrwx 1 root root 6 Jul 21 2020 bzegrep -> bzgrep + -rwxr-xr-x 1 root root 4877 Sep 5 2019 bzexe + lrwxrwxrwx 1 root root 6 Jul 21 2020 bzfgrep -> bzgrep + -rwxr-xr-x 1 root root 3775 Jul 21 2020 bzgrep + -rwxr-xr-x 3 root root 38984 Jul 21 2020 bzip2 + -rwxr-xr-x 1 root root 18424 Jul 21 2020 bzip2recover + lrwxrwxrwx 1 root root 6 Jul 21 2020 bzless -> bzmore + -rwxr-xr-x 1 root root 1297 Jul 21 2020 bzmore + -rwxr-xr-x 1 root root 43936 Sep 24 2020 cat + -rwxr-xr-x 1 root root 72672 Sep 24 2020 chgrp + -rwxr-xr-x 1 root root 64448 Sep 24 2020 chmod + -rwxr-xr-x 1 root root 72672 Sep 24 2020 chown + -rwxr-xr-x 1 root root 151168 Sep 24 2020 cp + -rwxr-xr-x 1 root root 125560 Dec 11 2020 dash + -rwxr-xr-x 1 root root 113664 Sep 24 2020 date + -rwxr-xr-x 1 root root 80968 Sep 24 2020 dd + -rwxr-xr-x 1 root root 93936 Sep 24 2020 df + -rwxr-xr-x 1 root root 147176 Sep 24 2020 dir + -rwxr-xr-x 1 root root 84440 Jul 29 2021 dmesg + lrwxrwxrwx 1 root root 8 Nov 8 2019 dnsdomainname -> hostname + lrwxrwxrwx 1 root root 8 Nov 8 2019 domainname -> hostname + -rwxr-xr-x 1 root root 39712 Sep 24 2020 echo + -rwxr-xr-x 1 root root 28 Nov 10 2020 egrep + -rwxr-xr-x 1 root root 39680 Sep 24 2020 false + -rwxr-xr-x 1 root root 28 Nov 10 2020 fgrep + -rwxr-xr-x 1 root root 69032 Jul 29 2021 findmnt + -rwsr-xr-x 1 root root 34896 Feb 27 2021 fusermount + -rwxr-xr-x 1 root root 203072 Nov 10 2020 grep + -rwxr-xr-x 2 root root 2346 Mar 3 2021 gunzip + -rwxr-xr-x 1 root root 6376 Mar 3 2021 gzexe + -rwxr-xr-x 1 root root 98048 Mar 3 2021 gzip + -rwxr-xr-x 1 root root 22600 Nov 8 2019 hostname + -rwxr-xr-x 1 root root 72840 Sep 24 2020 ln + -rwxr-xr-x 1 root root 56952 Feb 8 2020 login + -rwxr-xr-x 1 root root 147176 Sep 24 2020 ls + -rwxr-xr-x 1 root root 149736 Jul 29 2021 lsblk + -rwxr-xr-x 1 root root 85184 Sep 24 2020 mkdir + -rwxr-xr-x 1 root root 76896 Sep 24 2020 mknod + -rwxr-xr-x 1 root root 48064 Sep 24 2020 mktemp + -rwxr-xr-x 1 root root 59632 Jul 29 2021 more + -rwsr-xr-x 1 root root 55528 Jul 29 2021 mount + -rwxr-xr-x 1 root root 18664 Jul 29 2021 mountpoint + -rwxr-xr-x 1 root root 147080 Sep 24 2020 mv + lrwxrwxrwx 1 root root 8 Nov 8 2019 nisdomainname -> hostname + lrwxrwxrwx 1 root root 14 Apr 19 2021 pidof -> /sbin/killall5 + -rwxr-xr-x 1 root root 43872 Sep 24 2020 pwd + lrwxrwxrwx 1 root root 4 Aug 5 2021 rbash -> bash + -rwxr-xr-x 1 root root 52032 Sep 24 2020 readlink + -rwxr-xr-x 1 root root 72704 Sep 24 2020 rm + -rwxr-xr-x 1 root root 52032 Sep 24 2020 rmdir + -rwxr-xr-x 1 root root 27472 Sep 28 2020 run-parts + -rwxr-xr-x 1 root root 122224 Dec 23 2018 sed + lrwxrwxrwx 1 root root 4 Dec 4 19:44 sh -> bash + lrwxrwxrwx 1 root root 4 Nov 24 08:04 sh.distrib -> dash + -rwxr-xr-x 1 root root 43808 Sep 24 2020 sleep + -rwxr-xr-x 1 root root 84928 Sep 24 2020 stty + -rwsr-xr-x 1 root root 71912 Jul 29 2021 su + -rwxr-xr-x 1 root root 39744 Sep 24 2020 sync + -rwxr-xr-x 1 root root 531928 Feb 17 2021 tar + -rwxr-xr-x 1 root root 14456 Sep 28 2020 tempfile + -rwxr-xr-x 1 root root 101408 Sep 24 2020 touch + -rwxr-xr-x 1 root root 39680 Sep 24 2020 true + -rwxr-xr-x 1 root root 14328 Feb 27 2021 ulockmgr_server + -rwsr-xr-x 1 root root 35040 Jul 29 2021 umount + -rwxr-xr-x 1 root root 39744 Sep 24 2020 uname + -rwxr-xr-x 2 root root 2346 Mar 3 2021 uncompress + -rwxr-xr-x 1 root root 147176 Sep 24 2020 vdir + -rwxr-xr-x 1 root root 63744 Jul 29 2021 wdctl + lrwxrwxrwx 1 root root 8 Nov 8 2019 ypdomainname -> hostname + -rwxr-xr-x 1 root root 1984 Mar 3 2021 zcat + -rwxr-xr-x 1 root root 1678 Mar 3 2021 zcmp + -rwxr-xr-x 1 root root 5880 Mar 3 2021 zdiff + -rwxr-xr-x 1 root root 29 Mar 3 2021 zegrep + -rwxr-xr-x 1 root root 29 Mar 3 2021 zfgrep + -rwxr-xr-x 1 root root 2081 Mar 3 2021 zforce + -rwxr-xr-x 1 root root 7585 Mar 3 2021 zgrep + -rwxr-xr-x 1 root root 2206 Mar 3 2021 zless + -rwxr-xr-x 1 root root 1842 Mar 3 2021 zmore + -rwxr-xr-x 1 root root 4553 Mar 3 2021 znew +I: user script /srv/workspace/pbuilder/1426341/tmp/hooks/D02_print_environment finished -> Attempting to satisfy build-dependencies -> Creating pbuilder-satisfydepends-dummy package Package: pbuilder-satisfydepends-dummy @@ -559,7 +593,7 @@ Get: 210 http://deb.debian.org/debian bullseye/main amd64 golang-k8s-sigs-yaml-dev all 1.2.0-3 [15.0 kB] Get: 211 http://deb.debian.org/debian bullseye/main amd64 mockery amd64 0.0~git20181123.e78b021-3+b6 [1587 kB] Get: 212 http://deb.debian.org/debian bullseye/main amd64 golang-github-hashicorp-go-rootcerts-dev all 1.0.2-2 [8356 B] -Fetched 126 MB in 7s (17.3 MB/s) +Fetched 126 MB in 2s (68.3 MB/s) debconf: delaying package configuration, since apt-utils is not installed Selecting previously unselected package bsdextrautils. (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 19655 files and directories currently installed.) @@ -1436,7 +1470,8 @@ fakeroot is already the newest version (1.25.3-1.1). 0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded. I: Building the package -I: Running cd /build/consul-1.8.7+dfsg1/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" HOME="/nonexistent/first-build" dpkg-genchanges -S > ../consul_1.8.7+dfsg1-2_source.changes +hostname: Name or service not known +I: Running cd /build/consul-1.8.7+dfsg1/ && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-buildpackage -us -uc -b && env PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" HOME="/nonexistent/second-build" dpkg-genchanges -S > ../consul_1.8.7+dfsg1-2_source.changes dpkg-buildpackage: info: source package consul dpkg-buildpackage: info: source version 1.8.7+dfsg1-2 dpkg-buildpackage: info: source distribution unstable @@ -1567,7 +1602,7 @@ CGO_FFLAGS="-g -O2 -fdebug-prefix-map=/build/consul-1.8.7+dfsg1=. -fstack-protector-strong" CGO_LDFLAGS="-Wl,-z,relro" PKG_CONFIG="pkg-config" -GOGCCFLAGS="-fPIC -m64 -pthread -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build209960683=/tmp/go-build -gno-record-gcc-switches" +GOGCCFLAGS="-fPIC -m64 -pthread -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build480917737=/tmp/go-build -gno-record-gcc-switches" dh_auto_build: warning: github.com/hashicorp/consul/agent/routine-leak-checker contains no non-test Go files, removing it from build cd _build && go generate -v github.com/hashicorp/consul github.com/hashicorp/consul/acl github.com/hashicorp/consul/agent github.com/hashicorp/consul/agent/ae github.com/hashicorp/consul/agent/auto-config github.com/hashicorp/consul/agent/cache github.com/hashicorp/consul/agent/cache-types github.com/hashicorp/consul/agent/checks github.com/hashicorp/consul/agent/config github.com/hashicorp/consul/agent/connect github.com/hashicorp/consul/agent/connect/ca github.com/hashicorp/consul/agent/consul github.com/hashicorp/consul/agent/consul/authmethod github.com/hashicorp/consul/agent/consul/authmethod/kubeauth github.com/hashicorp/consul/agent/consul/authmethod/ssoauth github.com/hashicorp/consul/agent/consul/authmethod/testauth github.com/hashicorp/consul/agent/consul/autopilot github.com/hashicorp/consul/agent/consul/discoverychain github.com/hashicorp/consul/agent/consul/fsm github.com/hashicorp/consul/agent/consul/prepared_query github.com/hashicorp/consul/agent/consul/state github.com/hashicorp/consul/agent/consul/wanfed github.com/hashicorp/consul/agent/debug github.com/hashicorp/consul/agent/dns github.com/hashicorp/consul/agent/exec github.com/hashicorp/consul/agent/local github.com/hashicorp/consul/agent/metadata github.com/hashicorp/consul/agent/mock github.com/hashicorp/consul/agent/pool github.com/hashicorp/consul/agent/proxycfg github.com/hashicorp/consul/agent/router github.com/hashicorp/consul/agent/structs github.com/hashicorp/consul/agent/systemd github.com/hashicorp/consul/agent/token github.com/hashicorp/consul/agent/xds github.com/hashicorp/consul/agent/xds/proxysupport github.com/hashicorp/consul/api github.com/hashicorp/consul/api/watch github.com/hashicorp/consul/command github.com/hashicorp/consul/command/acl github.com/hashicorp/consul/command/acl/agenttokens github.com/hashicorp/consul/command/acl/authmethod github.com/hashicorp/consul/command/acl/authmethod/create github.com/hashicorp/consul/command/acl/authmethod/delete github.com/hashicorp/consul/command/acl/authmethod/list github.com/hashicorp/consul/command/acl/authmethod/read github.com/hashicorp/consul/command/acl/authmethod/update github.com/hashicorp/consul/command/acl/bindingrule github.com/hashicorp/consul/command/acl/bindingrule/create github.com/hashicorp/consul/command/acl/bindingrule/delete github.com/hashicorp/consul/command/acl/bindingrule/list github.com/hashicorp/consul/command/acl/bindingrule/read github.com/hashicorp/consul/command/acl/bindingrule/update github.com/hashicorp/consul/command/acl/bootstrap github.com/hashicorp/consul/command/acl/policy github.com/hashicorp/consul/command/acl/policy/create github.com/hashicorp/consul/command/acl/policy/delete github.com/hashicorp/consul/command/acl/policy/list github.com/hashicorp/consul/command/acl/policy/read github.com/hashicorp/consul/command/acl/policy/update github.com/hashicorp/consul/command/acl/role github.com/hashicorp/consul/command/acl/role/create github.com/hashicorp/consul/command/acl/role/delete github.com/hashicorp/consul/command/acl/role/list github.com/hashicorp/consul/command/acl/role/read github.com/hashicorp/consul/command/acl/role/update github.com/hashicorp/consul/command/acl/rules github.com/hashicorp/consul/command/acl/token github.com/hashicorp/consul/command/acl/token/clone github.com/hashicorp/consul/command/acl/token/create github.com/hashicorp/consul/command/acl/token/delete github.com/hashicorp/consul/command/acl/token/list github.com/hashicorp/consul/command/acl/token/read github.com/hashicorp/consul/command/acl/token/update github.com/hashicorp/consul/command/agent github.com/hashicorp/consul/command/catalog github.com/hashicorp/consul/command/catalog/list/dc github.com/hashicorp/consul/command/catalog/list/nodes github.com/hashicorp/consul/command/catalog/list/services github.com/hashicorp/consul/command/config github.com/hashicorp/consul/command/config/delete github.com/hashicorp/consul/command/config/list github.com/hashicorp/consul/command/config/read github.com/hashicorp/consul/command/config/write github.com/hashicorp/consul/command/connect github.com/hashicorp/consul/command/connect/ca github.com/hashicorp/consul/command/connect/ca/get github.com/hashicorp/consul/command/connect/ca/set github.com/hashicorp/consul/command/connect/envoy github.com/hashicorp/consul/command/connect/envoy/pipe-bootstrap github.com/hashicorp/consul/command/connect/expose github.com/hashicorp/consul/command/connect/proxy github.com/hashicorp/consul/command/debug github.com/hashicorp/consul/command/event github.com/hashicorp/consul/command/exec github.com/hashicorp/consul/command/flags github.com/hashicorp/consul/command/forceleave github.com/hashicorp/consul/command/helpers github.com/hashicorp/consul/command/info github.com/hashicorp/consul/command/intention github.com/hashicorp/consul/command/intention/check github.com/hashicorp/consul/command/intention/create github.com/hashicorp/consul/command/intention/delete github.com/hashicorp/consul/command/intention/finder github.com/hashicorp/consul/command/intention/get github.com/hashicorp/consul/command/intention/match github.com/hashicorp/consul/command/join github.com/hashicorp/consul/command/keygen github.com/hashicorp/consul/command/keyring github.com/hashicorp/consul/command/kv github.com/hashicorp/consul/command/kv/del github.com/hashicorp/consul/command/kv/exp github.com/hashicorp/consul/command/kv/get github.com/hashicorp/consul/command/kv/imp github.com/hashicorp/consul/command/kv/impexp github.com/hashicorp/consul/command/kv/put github.com/hashicorp/consul/command/leave github.com/hashicorp/consul/command/lock github.com/hashicorp/consul/command/login github.com/hashicorp/consul/command/logout github.com/hashicorp/consul/command/maint github.com/hashicorp/consul/command/members github.com/hashicorp/consul/command/monitor github.com/hashicorp/consul/command/operator github.com/hashicorp/consul/command/operator/autopilot github.com/hashicorp/consul/command/operator/autopilot/get github.com/hashicorp/consul/command/operator/autopilot/set github.com/hashicorp/consul/command/operator/raft github.com/hashicorp/consul/command/operator/raft/listpeers github.com/hashicorp/consul/command/operator/raft/removepeer github.com/hashicorp/consul/command/reload github.com/hashicorp/consul/command/rtt github.com/hashicorp/consul/command/services github.com/hashicorp/consul/command/services/deregister github.com/hashicorp/consul/command/services/register github.com/hashicorp/consul/command/snapshot github.com/hashicorp/consul/command/snapshot/inspect github.com/hashicorp/consul/command/snapshot/restore github.com/hashicorp/consul/command/snapshot/save github.com/hashicorp/consul/command/tls github.com/hashicorp/consul/command/tls/ca github.com/hashicorp/consul/command/tls/ca/create github.com/hashicorp/consul/command/tls/cert github.com/hashicorp/consul/command/tls/cert/create github.com/hashicorp/consul/command/validate github.com/hashicorp/consul/command/version github.com/hashicorp/consul/command/watch github.com/hashicorp/consul/connect github.com/hashicorp/consul/connect/certgen github.com/hashicorp/consul/connect/proxy github.com/hashicorp/consul/internal/go-sso/oidcauth github.com/hashicorp/consul/internal/go-sso/oidcauth/internal/strutil github.com/hashicorp/consul/internal/go-sso/oidcauth/oidcauthtest github.com/hashicorp/consul/ipaddr github.com/hashicorp/consul/lib github.com/hashicorp/consul/lib/decode github.com/hashicorp/consul/lib/file github.com/hashicorp/consul/lib/semaphore github.com/hashicorp/consul/lib/stringslice github.com/hashicorp/consul/lib/template github.com/hashicorp/consul/logging github.com/hashicorp/consul/logging/monitor github.com/hashicorp/consul/proto github.com/hashicorp/consul/proto/pbacl github.com/hashicorp/consul/proto/pbautoconf github.com/hashicorp/consul/proto/pbcommon github.com/hashicorp/consul/proto/pbconfig github.com/hashicorp/consul/proto/pbconnect github.com/hashicorp/consul/sdk/freeport github.com/hashicorp/consul/sdk/testutil github.com/hashicorp/consul/sdk/testutil/retry github.com/hashicorp/consul/sentinel github.com/hashicorp/consul/service_os github.com/hashicorp/consul/snapshot github.com/hashicorp/consul/testrpc github.com/hashicorp/consul/tlsutil github.com/hashicorp/consul/types github.com/hashicorp/consul/version src/github.com/hashicorp/consul/main.go @@ -2568,87 +2603,88 @@ src/github.com/hashicorp/consul/types/checks.go src/github.com/hashicorp/consul/types/node_id.go src/github.com/hashicorp/consul/version/version.go - cd _build && go install -trimpath -v -p 15 github.com/hashicorp/consul github.com/hashicorp/consul/acl github.com/hashicorp/consul/agent github.com/hashicorp/consul/agent/ae github.com/hashicorp/consul/agent/auto-config github.com/hashicorp/consul/agent/cache github.com/hashicorp/consul/agent/cache-types github.com/hashicorp/consul/agent/checks github.com/hashicorp/consul/agent/config github.com/hashicorp/consul/agent/connect github.com/hashicorp/consul/agent/connect/ca github.com/hashicorp/consul/agent/consul github.com/hashicorp/consul/agent/consul/authmethod github.com/hashicorp/consul/agent/consul/authmethod/kubeauth github.com/hashicorp/consul/agent/consul/authmethod/ssoauth github.com/hashicorp/consul/agent/consul/authmethod/testauth github.com/hashicorp/consul/agent/consul/autopilot github.com/hashicorp/consul/agent/consul/discoverychain github.com/hashicorp/consul/agent/consul/fsm github.com/hashicorp/consul/agent/consul/prepared_query github.com/hashicorp/consul/agent/consul/state github.com/hashicorp/consul/agent/consul/wanfed github.com/hashicorp/consul/agent/debug github.com/hashicorp/consul/agent/dns github.com/hashicorp/consul/agent/exec github.com/hashicorp/consul/agent/local github.com/hashicorp/consul/agent/metadata github.com/hashicorp/consul/agent/mock github.com/hashicorp/consul/agent/pool github.com/hashicorp/consul/agent/proxycfg github.com/hashicorp/consul/agent/router github.com/hashicorp/consul/agent/structs github.com/hashicorp/consul/agent/systemd github.com/hashicorp/consul/agent/token github.com/hashicorp/consul/agent/xds github.com/hashicorp/consul/agent/xds/proxysupport github.com/hashicorp/consul/api github.com/hashicorp/consul/api/watch github.com/hashicorp/consul/command github.com/hashicorp/consul/command/acl github.com/hashicorp/consul/command/acl/agenttokens github.com/hashicorp/consul/command/acl/authmethod github.com/hashicorp/consul/command/acl/authmethod/create github.com/hashicorp/consul/command/acl/authmethod/delete github.com/hashicorp/consul/command/acl/authmethod/list github.com/hashicorp/consul/command/acl/authmethod/read github.com/hashicorp/consul/command/acl/authmethod/update github.com/hashicorp/consul/command/acl/bindingrule github.com/hashicorp/consul/command/acl/bindingrule/create github.com/hashicorp/consul/command/acl/bindingrule/delete github.com/hashicorp/consul/command/acl/bindingrule/list github.com/hashicorp/consul/command/acl/bindingrule/read github.com/hashicorp/consul/command/acl/bindingrule/update github.com/hashicorp/consul/command/acl/bootstrap github.com/hashicorp/consul/command/acl/policy github.com/hashicorp/consul/command/acl/policy/create github.com/hashicorp/consul/command/acl/policy/delete github.com/hashicorp/consul/command/acl/policy/list github.com/hashicorp/consul/command/acl/policy/read github.com/hashicorp/consul/command/acl/policy/update github.com/hashicorp/consul/command/acl/role github.com/hashicorp/consul/command/acl/role/create github.com/hashicorp/consul/command/acl/role/delete github.com/hashicorp/consul/command/acl/role/list github.com/hashicorp/consul/command/acl/role/read github.com/hashicorp/consul/command/acl/role/update github.com/hashicorp/consul/command/acl/rules github.com/hashicorp/consul/command/acl/token github.com/hashicorp/consul/command/acl/token/clone github.com/hashicorp/consul/command/acl/token/create github.com/hashicorp/consul/command/acl/token/delete github.com/hashicorp/consul/command/acl/token/list github.com/hashicorp/consul/command/acl/token/read github.com/hashicorp/consul/command/acl/token/update github.com/hashicorp/consul/command/agent github.com/hashicorp/consul/command/catalog github.com/hashicorp/consul/command/catalog/list/dc github.com/hashicorp/consul/command/catalog/list/nodes github.com/hashicorp/consul/command/catalog/list/services github.com/hashicorp/consul/command/config github.com/hashicorp/consul/command/config/delete github.com/hashicorp/consul/command/config/list github.com/hashicorp/consul/command/config/read github.com/hashicorp/consul/command/config/write github.com/hashicorp/consul/command/connect github.com/hashicorp/consul/command/connect/ca github.com/hashicorp/consul/command/connect/ca/get github.com/hashicorp/consul/command/connect/ca/set github.com/hashicorp/consul/command/connect/envoy github.com/hashicorp/consul/command/connect/envoy/pipe-bootstrap github.com/hashicorp/consul/command/connect/expose github.com/hashicorp/consul/command/connect/proxy github.com/hashicorp/consul/command/debug github.com/hashicorp/consul/command/event github.com/hashicorp/consul/command/exec github.com/hashicorp/consul/command/flags github.com/hashicorp/consul/command/forceleave github.com/hashicorp/consul/command/helpers github.com/hashicorp/consul/command/info github.com/hashicorp/consul/command/intention github.com/hashicorp/consul/command/intention/check github.com/hashicorp/consul/command/intention/create github.com/hashicorp/consul/command/intention/delete github.com/hashicorp/consul/command/intention/finder github.com/hashicorp/consul/command/intention/get github.com/hashicorp/consul/command/intention/match github.com/hashicorp/consul/command/join github.com/hashicorp/consul/command/keygen github.com/hashicorp/consul/command/keyring github.com/hashicorp/consul/command/kv github.com/hashicorp/consul/command/kv/del github.com/hashicorp/consul/command/kv/exp github.com/hashicorp/consul/command/kv/get github.com/hashicorp/consul/command/kv/imp github.com/hashicorp/consul/command/kv/impexp github.com/hashicorp/consul/command/kv/put github.com/hashicorp/consul/command/leave github.com/hashicorp/consul/command/lock github.com/hashicorp/consul/command/login github.com/hashicorp/consul/command/logout github.com/hashicorp/consul/command/maint github.com/hashicorp/consul/command/members github.com/hashicorp/consul/command/monitor github.com/hashicorp/consul/command/operator github.com/hashicorp/consul/command/operator/autopilot github.com/hashicorp/consul/command/operator/autopilot/get github.com/hashicorp/consul/command/operator/autopilot/set github.com/hashicorp/consul/command/operator/raft github.com/hashicorp/consul/command/operator/raft/listpeers github.com/hashicorp/consul/command/operator/raft/removepeer github.com/hashicorp/consul/command/reload github.com/hashicorp/consul/command/rtt github.com/hashicorp/consul/command/services github.com/hashicorp/consul/command/services/deregister github.com/hashicorp/consul/command/services/register github.com/hashicorp/consul/command/snapshot github.com/hashicorp/consul/command/snapshot/inspect github.com/hashicorp/consul/command/snapshot/restore github.com/hashicorp/consul/command/snapshot/save github.com/hashicorp/consul/command/tls github.com/hashicorp/consul/command/tls/ca github.com/hashicorp/consul/command/tls/ca/create github.com/hashicorp/consul/command/tls/cert github.com/hashicorp/consul/command/tls/cert/create github.com/hashicorp/consul/command/validate github.com/hashicorp/consul/command/version github.com/hashicorp/consul/command/watch github.com/hashicorp/consul/connect github.com/hashicorp/consul/connect/certgen github.com/hashicorp/consul/connect/proxy github.com/hashicorp/consul/internal/go-sso/oidcauth github.com/hashicorp/consul/internal/go-sso/oidcauth/internal/strutil github.com/hashicorp/consul/internal/go-sso/oidcauth/oidcauthtest github.com/hashicorp/consul/ipaddr github.com/hashicorp/consul/lib github.com/hashicorp/consul/lib/decode github.com/hashicorp/consul/lib/file github.com/hashicorp/consul/lib/semaphore github.com/hashicorp/consul/lib/stringslice github.com/hashicorp/consul/lib/template github.com/hashicorp/consul/logging github.com/hashicorp/consul/logging/monitor github.com/hashicorp/consul/proto github.com/hashicorp/consul/proto/pbacl github.com/hashicorp/consul/proto/pbautoconf github.com/hashicorp/consul/proto/pbcommon github.com/hashicorp/consul/proto/pbconfig github.com/hashicorp/consul/proto/pbconnect github.com/hashicorp/consul/sdk/freeport github.com/hashicorp/consul/sdk/testutil github.com/hashicorp/consul/sdk/testutil/retry github.com/hashicorp/consul/sentinel github.com/hashicorp/consul/service_os github.com/hashicorp/consul/snapshot github.com/hashicorp/consul/testrpc github.com/hashicorp/consul/tlsutil github.com/hashicorp/consul/types github.com/hashicorp/consul/version + cd _build && go install -trimpath -v -p 16 github.com/hashicorp/consul github.com/hashicorp/consul/acl github.com/hashicorp/consul/agent github.com/hashicorp/consul/agent/ae github.com/hashicorp/consul/agent/auto-config github.com/hashicorp/consul/agent/cache github.com/hashicorp/consul/agent/cache-types github.com/hashicorp/consul/agent/checks github.com/hashicorp/consul/agent/config github.com/hashicorp/consul/agent/connect github.com/hashicorp/consul/agent/connect/ca github.com/hashicorp/consul/agent/consul github.com/hashicorp/consul/agent/consul/authmethod github.com/hashicorp/consul/agent/consul/authmethod/kubeauth github.com/hashicorp/consul/agent/consul/authmethod/ssoauth github.com/hashicorp/consul/agent/consul/authmethod/testauth github.com/hashicorp/consul/agent/consul/autopilot github.com/hashicorp/consul/agent/consul/discoverychain github.com/hashicorp/consul/agent/consul/fsm github.com/hashicorp/consul/agent/consul/prepared_query github.com/hashicorp/consul/agent/consul/state github.com/hashicorp/consul/agent/consul/wanfed github.com/hashicorp/consul/agent/debug github.com/hashicorp/consul/agent/dns github.com/hashicorp/consul/agent/exec github.com/hashicorp/consul/agent/local github.com/hashicorp/consul/agent/metadata github.com/hashicorp/consul/agent/mock github.com/hashicorp/consul/agent/pool github.com/hashicorp/consul/agent/proxycfg github.com/hashicorp/consul/agent/router github.com/hashicorp/consul/agent/structs github.com/hashicorp/consul/agent/systemd github.com/hashicorp/consul/agent/token github.com/hashicorp/consul/agent/xds github.com/hashicorp/consul/agent/xds/proxysupport github.com/hashicorp/consul/api github.com/hashicorp/consul/api/watch github.com/hashicorp/consul/command github.com/hashicorp/consul/command/acl github.com/hashicorp/consul/command/acl/agenttokens github.com/hashicorp/consul/command/acl/authmethod github.com/hashicorp/consul/command/acl/authmethod/create github.com/hashicorp/consul/command/acl/authmethod/delete github.com/hashicorp/consul/command/acl/authmethod/list github.com/hashicorp/consul/command/acl/authmethod/read github.com/hashicorp/consul/command/acl/authmethod/update github.com/hashicorp/consul/command/acl/bindingrule github.com/hashicorp/consul/command/acl/bindingrule/create github.com/hashicorp/consul/command/acl/bindingrule/delete github.com/hashicorp/consul/command/acl/bindingrule/list github.com/hashicorp/consul/command/acl/bindingrule/read github.com/hashicorp/consul/command/acl/bindingrule/update github.com/hashicorp/consul/command/acl/bootstrap github.com/hashicorp/consul/command/acl/policy github.com/hashicorp/consul/command/acl/policy/create github.com/hashicorp/consul/command/acl/policy/delete github.com/hashicorp/consul/command/acl/policy/list github.com/hashicorp/consul/command/acl/policy/read github.com/hashicorp/consul/command/acl/policy/update github.com/hashicorp/consul/command/acl/role github.com/hashicorp/consul/command/acl/role/create github.com/hashicorp/consul/command/acl/role/delete github.com/hashicorp/consul/command/acl/role/list github.com/hashicorp/consul/command/acl/role/read github.com/hashicorp/consul/command/acl/role/update github.com/hashicorp/consul/command/acl/rules github.com/hashicorp/consul/command/acl/token github.com/hashicorp/consul/command/acl/token/clone github.com/hashicorp/consul/command/acl/token/create github.com/hashicorp/consul/command/acl/token/delete github.com/hashicorp/consul/command/acl/token/list github.com/hashicorp/consul/command/acl/token/read github.com/hashicorp/consul/command/acl/token/update github.com/hashicorp/consul/command/agent github.com/hashicorp/consul/command/catalog github.com/hashicorp/consul/command/catalog/list/dc github.com/hashicorp/consul/command/catalog/list/nodes github.com/hashicorp/consul/command/catalog/list/services github.com/hashicorp/consul/command/config github.com/hashicorp/consul/command/config/delete github.com/hashicorp/consul/command/config/list github.com/hashicorp/consul/command/config/read github.com/hashicorp/consul/command/config/write github.com/hashicorp/consul/command/connect github.com/hashicorp/consul/command/connect/ca github.com/hashicorp/consul/command/connect/ca/get github.com/hashicorp/consul/command/connect/ca/set github.com/hashicorp/consul/command/connect/envoy github.com/hashicorp/consul/command/connect/envoy/pipe-bootstrap github.com/hashicorp/consul/command/connect/expose github.com/hashicorp/consul/command/connect/proxy github.com/hashicorp/consul/command/debug github.com/hashicorp/consul/command/event github.com/hashicorp/consul/command/exec github.com/hashicorp/consul/command/flags github.com/hashicorp/consul/command/forceleave github.com/hashicorp/consul/command/helpers github.com/hashicorp/consul/command/info github.com/hashicorp/consul/command/intention github.com/hashicorp/consul/command/intention/check github.com/hashicorp/consul/command/intention/create github.com/hashicorp/consul/command/intention/delete github.com/hashicorp/consul/command/intention/finder github.com/hashicorp/consul/command/intention/get github.com/hashicorp/consul/command/intention/match github.com/hashicorp/consul/command/join github.com/hashicorp/consul/command/keygen github.com/hashicorp/consul/command/keyring github.com/hashicorp/consul/command/kv github.com/hashicorp/consul/command/kv/del github.com/hashicorp/consul/command/kv/exp github.com/hashicorp/consul/command/kv/get github.com/hashicorp/consul/command/kv/imp github.com/hashicorp/consul/command/kv/impexp github.com/hashicorp/consul/command/kv/put github.com/hashicorp/consul/command/leave github.com/hashicorp/consul/command/lock github.com/hashicorp/consul/command/login github.com/hashicorp/consul/command/logout github.com/hashicorp/consul/command/maint github.com/hashicorp/consul/command/members github.com/hashicorp/consul/command/monitor github.com/hashicorp/consul/command/operator github.com/hashicorp/consul/command/operator/autopilot github.com/hashicorp/consul/command/operator/autopilot/get github.com/hashicorp/consul/command/operator/autopilot/set github.com/hashicorp/consul/command/operator/raft github.com/hashicorp/consul/command/operator/raft/listpeers github.com/hashicorp/consul/command/operator/raft/removepeer github.com/hashicorp/consul/command/reload github.com/hashicorp/consul/command/rtt github.com/hashicorp/consul/command/services github.com/hashicorp/consul/command/services/deregister github.com/hashicorp/consul/command/services/register github.com/hashicorp/consul/command/snapshot github.com/hashicorp/consul/command/snapshot/inspect github.com/hashicorp/consul/command/snapshot/restore github.com/hashicorp/consul/command/snapshot/save github.com/hashicorp/consul/command/tls github.com/hashicorp/consul/command/tls/ca github.com/hashicorp/consul/command/tls/ca/create github.com/hashicorp/consul/command/tls/cert github.com/hashicorp/consul/command/tls/cert/create github.com/hashicorp/consul/command/validate github.com/hashicorp/consul/command/version github.com/hashicorp/consul/command/watch github.com/hashicorp/consul/connect github.com/hashicorp/consul/connect/certgen github.com/hashicorp/consul/connect/proxy github.com/hashicorp/consul/internal/go-sso/oidcauth github.com/hashicorp/consul/internal/go-sso/oidcauth/internal/strutil github.com/hashicorp/consul/internal/go-sso/oidcauth/oidcauthtest github.com/hashicorp/consul/ipaddr github.com/hashicorp/consul/lib github.com/hashicorp/consul/lib/decode github.com/hashicorp/consul/lib/file github.com/hashicorp/consul/lib/semaphore github.com/hashicorp/consul/lib/stringslice github.com/hashicorp/consul/lib/template github.com/hashicorp/consul/logging github.com/hashicorp/consul/logging/monitor github.com/hashicorp/consul/proto github.com/hashicorp/consul/proto/pbacl github.com/hashicorp/consul/proto/pbautoconf github.com/hashicorp/consul/proto/pbcommon github.com/hashicorp/consul/proto/pbconfig github.com/hashicorp/consul/proto/pbconnect github.com/hashicorp/consul/sdk/freeport github.com/hashicorp/consul/sdk/testutil github.com/hashicorp/consul/sdk/testutil/retry github.com/hashicorp/consul/sentinel github.com/hashicorp/consul/service_os github.com/hashicorp/consul/snapshot github.com/hashicorp/consul/testrpc github.com/hashicorp/consul/tlsutil github.com/hashicorp/consul/types github.com/hashicorp/consul/version internal/unsafeheader -unicode/utf8 -runtime/internal/sys internal/race -unicode +unicode/utf8 encoding -unicode/utf16 -container/list -crypto/subtle -internal/nettrace math/bits -crypto/internal/subtle -internal/cpu -vendor/golang.org/x/crypto/cryptobyte/asn1 +runtime/internal/sys +unicode runtime/internal/atomic +internal/cpu sync/atomic +vendor/golang.org/x/crypto/cryptobyte/asn1 +runtime/internal/math vendor/golang.org/x/crypto/internal/subtle github.com/circonus-labs/circonus-gometrics/api/config -runtime/internal/math -runtime/cgo golang.org/x/sys/internal/unsafeheader golang.org/x/net/internal/iana +crypto/internal/subtle +runtime/cgo +crypto/subtle +container/list +internal/nettrace +unicode/utf16 github.com/hashicorp/consul/lib/stringslice github.com/hashicorp/consul/types google.golang.org/grpc/serviceconfig github.com/aws/aws-sdk-go/aws/client/metadata github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/selection github.com/hashicorp/consul/vendor/k8s.io/utils/integer -github.com/hashicorp/consul/internal/go-sso/oidcauth/internal/strutil -github.com/golang/groupcache/lru -go.opencensus.io internal/bytealg -math internal/testlog +math go.opencensus.io/trace/internal -go.opencensus.io/internal/tagencoding +github.com/hashicorp/consul/internal/go-sso/oidcauth/internal/strutil +github.com/hashicorp/consul/service_os +github.com/golang/groupcache/lru +go.opencensus.io github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/pkg/wellknown github.com/hashicorp/consul/agent/xds/proxysupport -github.com/hashicorp/consul/service_os +go.opencensus.io/internal/tagencoding runtime github.com/aws/aws-sdk-go/internal/sdkmath internal/reflectlite sync internal/singleflight -math/rand google.golang.org/grpc/internal/buffer google.golang.org/grpc/internal/grpcsync github.com/aws/aws-sdk-go/internal/sync/singleflight github.com/hashicorp/consul/agent/token +math/rand errors sort strconv -io -internal/oserror github.com/hashicorp/hcl/hcl/strconv github.com/hashicorp/golang-lru/simplelru +io +internal/oserror vendor/golang.org/x/net/dns/dnsmessage +syscall container/heap github.com/beorn7/perks/quantile github.com/gogo/protobuf/sortkeys -syscall bytes hash text/tabwriter +strings crypto/internal/randutil github.com/aws/aws-sdk-go/internal/sdkio -strings hash/crc32 crypto/hmac -hash/crc64 hash/fnv -vendor/golang.org/x/crypto/hkdf -golang.org/x/crypto/pbkdf2 +hash/crc64 crypto crypto/rc4 encoding/base32 reflect -vendor/golang.org/x/text/transform +vendor/golang.org/x/crypto/hkdf +golang.org/x/crypto/pbkdf2 github.com/kr/text golang.org/x/text/transform +vendor/golang.org/x/text/transform +github.com/armon/go-radix bufio regexp/syntax github.com/hashicorp/go-immutable-radix @@ -2658,19 +2694,15 @@ github.com/aws/aws-sdk-go/internal/strings github.com/ryanuber/go-glob html -github.com/armon/go-radix github.com/aws/aws-sdk-go/internal/sdkuri -internal/syscall/execenv internal/syscall/unix +internal/syscall/execenv time regexp -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/version -github.com/hashicorp/consul/agent/dns -google.golang.org/api/internal/third_party/uritemplates -internal/poll context google.golang.org/grpc/backoff google.golang.org/grpc/internal/grpcrand +internal/poll google.golang.org/grpc/keepalive github.com/aws/aws-sdk-go/internal/sdkrand github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/clock @@ -2678,107 +2710,119 @@ google.golang.org/grpc/tap github.com/hashicorp/consul/lib/semaphore golang.org/x/net/context -internal/fmtsort +github.com/hashicorp/errwrap encoding/binary +internal/fmtsort +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/version +github.com/hashicorp/consul/agent/dns +google.golang.org/api/internal/third_party/uritemplates github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/sets -github.com/hashicorp/errwrap os crypto/md5 encoding/base64 crypto/cipher crypto/sha512 crypto/sha1 -crypto/ed25519/internal/edwards25519 crypto/sha256 +crypto/ed25519/internal/edwards25519 vendor/golang.org/x/crypto/poly1305 github.com/cespare/xxhash -github.com/golang/snappy github.com/pierrec/lz4/internal/xxh32 github.com/google/gofuzz/bytesource +github.com/golang/snappy golang.org/x/sys/unix encoding/pem path/filepath fmt os/signal -net runtime/debug google.golang.org/grpc/internal/envconfig internal/lazyregexp +net crypto/aes crypto/des vendor/golang.org/x/crypto/chacha20 io/ioutil -os/exec github.com/aws/aws-sdk-go/internal/shareddefaults +os/exec golang.org/x/sys/cpu vendor/golang.org/x/sys/cpu github.com/prometheus/procfs/internal/util github.com/hashicorp/consul/vendor/github.com/rboyer/safeio +vendor/golang.org/x/crypto/chacha20poly1305 +golang.org/x/crypto/blake2b github.com/hashicorp/hcl/hcl/token log -compress/flate encoding/hex net/url -vendor/golang.org/x/crypto/curve25519 -math/big encoding/json +compress/flate +vendor/golang.org/x/crypto/curve25519 +mime vendor/golang.org/x/net/http2/hpack +mime/quotedprintable +math/big vendor/golang.org/x/text/unicode/norm -vendor/golang.org/x/crypto/chacha20poly1305 -mime -golang.org/x/crypto/blake2b github.com/hashicorp/hcl/hcl/ast github.com/hashicorp/hcl/hcl/scanner github.com/hashicorp/hcl/json/token vendor/golang.org/x/text/unicode/bidi -mime/quotedprintable -github.com/hashicorp/hcl/json/scanner net/http/internal github.com/pkg/errors -github.com/hashicorp/hcl/hcl/parser github.com/prometheus/procfs/internal/fs github.com/mitchellh/go-testing-interface -github.com/hashicorp/hcl/json/parser github.com/hashicorp/consul/version +github.com/hashicorp/hcl/json/scanner encoding/gob -text/template/parse go/token +text/template/parse +github.com/hashicorp/hcl/hcl/parser compress/lzw -compress/gzip github.com/google/btree github.com/hashicorp/go-multierror golang.org/x/net/bpf -github.com/hashicorp/hcl/hcl/printer -github.com/hashicorp/hcl -vendor/golang.org/x/text/secure/bidirule github.com/mitchellh/reflectwalk github.com/davecgh/go-spew/spew +github.com/hashicorp/hcl/json/parser +compress/gzip +github.com/hashicorp/hcl/hcl/printer github.com/pmezard/go-difflib/difflib gopkg.in/yaml.v3 +github.com/mitchellh/copystructure +github.com/hashicorp/hcl flag golang.org/x/time/rate -github.com/mitchellh/copystructure -github.com/circonus-labs/circonusllhist -vendor/golang.org/x/net/idna -github.com/prometheus/common/model -github.com/golang/protobuf/proto -github.com/stretchr/objx +vendor/golang.org/x/text/secure/bidirule github.com/mattn/go-isatty github.com/hashicorp/go-version -github.com/hashicorp/consul/acl runtime/trace -github.com/mattn/go-colorable github.com/hashicorp/consul/lib/decode github.com/hashicorp/golang-lru +github.com/circonus-labs/circonusllhist +github.com/golang/protobuf/proto +github.com/stretchr/objx +github.com/prometheus/common/model +github.com/mattn/go-colorable +vendor/golang.org/x/net/idna github.com/mitchellh/hashstructure -testing -github.com/bgentry/speakeasy github.com/fatih/color -text/template +github.com/bgentry/speakeasy +github.com/hashicorp/consul/acl os/user -github.com/hashicorp/go-hclog github.com/hashicorp/consul/command/helpers +testing github.com/armon/circbuf +crypto/rand +crypto/elliptic +encoding/asn1 +crypto/dsa +text/template +github.com/hashicorp/go-hclog +crypto/ed25519 +crypto/rsa +github.com/hashicorp/go-uuid +github.com/hashicorp/memberlist/vendor/github.com/sean-/seed +golang.org/x/crypto/ed25519 github.com/hashicorp/consul/agent/exec golang.org/x/net/internal/timeseries google.golang.org/grpc/grpclog @@ -2788,107 +2832,79 @@ golang.org/x/text/unicode/bidi golang.org/x/text/unicode/norm golang.org/x/net/http2/hpack -github.com/aws/aws-sdk-go/aws/awserr google.golang.org/grpc/internal/balancerload +github.com/aws/aws-sdk-go/aws/awserr github.com/jmespath/go-jmespath -google.golang.org/grpc/connectivity -google.golang.org/grpc/internal encoding/xml -crypto/rand -crypto/elliptic -encoding/asn1 -crypto/dsa +vendor/golang.org/x/crypto/cryptobyte +crypto/x509/pkix +google.golang.org/grpc/connectivity +github.com/hashicorp/hil/ast github.com/aws/aws-sdk-go/internal/ini +google.golang.org/grpc/internal github.com/aws/aws-sdk-go/aws/endpoints -crypto/ed25519 -crypto/rsa -github.com/hashicorp/go-uuid -golang.org/x/crypto/ed25519 -github.com/hashicorp/memberlist/vendor/github.com/sean-/seed -golang.org/x/text/secure/bidirule -github.com/hashicorp/hil/ast -html/template github.com/gogo/protobuf/proto github.com/hashicorp/go-memdb -vendor/golang.org/x/crypto/cryptobyte -crypto/x509/pkix -github.com/aws/aws-sdk-go/aws/awsutil -github.com/aws/aws-sdk-go/aws/credentials +html/template github.com/hashicorp/consul/sdk/testutil/retry -golang.org/x/net/idna github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/sdk/helper/consts +golang.org/x/text/secure/bidirule github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/sdk/helper/hclutil -github.com/aws/aws-sdk-go/aws/credentials/processcreds github.com/pierrec/lz4 +crypto/ecdsa github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/sdk/helper/strutil -gopkg.in/square/go-jose.v2/json github.com/hashicorp/go-bexpr +gopkg.in/square/go-jose.v2/json +github.com/aws/aws-sdk-go/aws/credentials github.com/google/gofuzz +github.com/aws/aws-sdk-go/aws/awsutil +gopkg.in/square/go-jose.v2/cipher gopkg.in/inf.v0 -crypto/ecdsa github.com/hashicorp/consul/vendor/k8s.io/apimachinery/third_party/forked/golang/reflect +github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/sdk/helper/compressutil +github.com/aws/aws-sdk-go/aws/credentials/processcreds +golang.org/x/net/idna github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/fields github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/errors go/scanner github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/conversion/queryparams -github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/sdk/helper/compressutil -gopkg.in/square/go-jose.v2/cipher github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/json -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/conversion -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/validation/field github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/naming -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/types github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/sdk/helper/jsonutil +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/validation/field +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/types +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/conversion gopkg.in/yaml.v2 github.com/modern-go/concurrent +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/framer +github.com/hashicorp/consul/vendor/k8s.io/client-go/pkg/version +golang.org/x/term +go/ast +github.com/hashicorp/consul/vendor/k8s.io/client-go/tools/metrics +github.com/hashicorp/consul/vendor/k8s.io/client-go/util/flowcontrol +github.com/patrickmn/go-cache +github.com/modern-go/reflect2 +github.com/boltdb/bolt +golang.org/x/sync/singleflight github.com/golang/protobuf/ptypes/any github.com/golang/protobuf/ptypes/duration github.com/golang/protobuf/ptypes/timestamp github.com/matttproud/golang_protobuf_extensions/pbutil github.com/golang/protobuf/ptypes/struct +github.com/prometheus/client_model/go google.golang.org/grpc/encoding/proto -google.golang.org/genproto/googleapis/rpc/status -go/ast -github.com/modern-go/reflect2 -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/framer -github.com/hashicorp/consul/vendor/k8s.io/client-go/pkg/version github.com/golang/protobuf/ptypes -github.com/prometheus/client_model/go google.golang.org/grpc/binarylog/grpc_binarylog_v1 -golang.org/x/term -github.com/golang/protobuf/jsonpb -github.com/hashicorp/consul/vendor/k8s.io/client-go/tools/metrics -github.com/hashicorp/consul/vendor/k8s.io/client-go/util/flowcontrol -github.com/patrickmn/go-cache -github.com/hashicorp/go-raftchunking/types -google.golang.org/grpc/status -github.com/prometheus/client_golang/prometheus/internal -github.com/googleapis/gnostic/extensions -github.com/hashicorp/consul/proto/pbconfig -golang.org/x/sync/singleflight -github.com/boltdb/bolt +google.golang.org/genproto/googleapis/rpc/status golang.org/x/crypto/ssh/terminal -golang.org/x/oauth2/jws -go.opencensus.io/internal -go.opencensus.io/trace/tracestate -go.opencensus.io/resource -runtime/pprof -github.com/shirou/gopsutil/internal/common -go.opencensus.io/trace -go.opencensus.io/metric/metricdata -go.opencensus.io/metric/metricproducer -github.com/golang/protobuf/protoc-gen-go/descriptor -go/parser -go/doc -github.com/golang/protobuf/ptypes/wrappers -github.com/golang/protobuf/ptypes/empty -github.com/shirou/gopsutil/cpu -github.com/shirou/gopsutil/disk +github.com/hashicorp/go-raftchunking/types github.com/posener/complete/cmd/install +github.com/golang/protobuf/jsonpb github.com/hashicorp/consul/vendor/k8s.io/klog -archive/tar -github.com/shirou/gopsutil/host -go.opencensus.io/tag +github.com/googleapis/gnostic/extensions +google.golang.org/grpc/status +github.com/hashicorp/consul/proto/pbconfig +github.com/prometheus/client_golang/prometheus/internal crypto/x509 net/textproto vendor/golang.org/x/net/http/httpproxy @@ -2896,75 +2912,90 @@ github.com/prometheus/procfs github.com/hashicorp/consul/ipaddr github.com/hashicorp/consul/sdk/freeport -golang.org/x/net/internal/socket github.com/hashicorp/go-sockaddr -github.com/hashicorp/yamux -github.com/mitchellh/mapstructure +golang.org/x/net/internal/socket vendor/golang.org/x/net/http/httpguts mime/multipart +github.com/hashicorp/yamux +github.com/mitchellh/mapstructure log/syslog github.com/posener/complete/cmd golang.org/x/net/internal/socks google.golang.org/grpc/credentials/internal +github.com/posener/complete google.golang.org/grpc/internal/binarylog github.com/hashicorp/go-syslog -github.com/posener/complete +golang.org/x/net/ipv4 +golang.org/x/net/ipv6 +golang.org/x/net/proxy golang.org/x/net/http/httpguts +github.com/mitchellh/cli google.golang.org/grpc/internal/syscall google.golang.org/grpc/stats google.golang.org/grpc/naming -golang.org/x/net/ipv4 -golang.org/x/net/ipv6 -golang.org/x/net/proxy -github.com/hashicorp/consul/logging github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/validation -github.com/json-iterator/go +github.com/hashicorp/consul/logging +go/doc +go/parser sigs.k8s.io/yaml -github.com/mitchellh/cli github.com/hashicorp/consul/vendor/k8s.io/client-go/util/connrotation -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/labels -github.com/hashicorp/hil -github.com/hashicorp/consul/vendor/github.com/mitchellh/pointerstructure -go.opencensus.io/stats/internal -github.com/shirou/gopsutil/mem -go.opencensus.io/stats crypto/tls -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/yaml +github.com/hashicorp/hil gopkg.in/square/go-jose.v2 +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/labels github.com/hashicorp/consul/vendor/k8s.io/client-go/util/keyutil +github.com/json-iterator/go +github.com/hashicorp/consul/vendor/github.com/mitchellh/pointerstructure +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/yaml +archive/tar +github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/sdk/helper/parseutil +github.com/hashicorp/consul/vendor/k8s.io/client-go/util/cert +github.com/hashicorp/go-sockaddr/template +golang.org/x/oauth2/jws +go.opencensus.io/internal +go.opencensus.io/trace/tracestate +go.opencensus.io/resource +github.com/shirou/gopsutil/internal/common +runtime/pprof +go.opencensus.io/trace +github.com/hashicorp/consul/agent/systemd github.com/gogo/protobuf/protoc-gen-gogo/descriptor github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/api/resource -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/intstr github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/schema +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/intstr +go.opencensus.io/metric/metricdata github.com/gogo/protobuf/types -github.com/hashicorp/consul/vendor/k8s.io/client-go/util/cert -go.opencensus.io/stats/view -github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/sdk/helper/parseutil -github.com/hashicorp/go-sockaddr/template -github.com/hashicorp/consul/agent/debug -github.com/hashicorp/consul/agent/systemd +github.com/golang/protobuf/protoc-gen-go/descriptor +go.opencensus.io/metric/metricproducer net/mail -github.com/hashicorp/consul/vendor/github.com/envoyproxy/protoc-gen-validate/validate -google.golang.org/genproto/googleapis/api/annotations github.com/hashicorp/consul/lib/template +github.com/golang/protobuf/ptypes/wrappers +github.com/golang/protobuf/ptypes/empty github.com/census-instrumentation/opencensus-proto/gen-go/resource/v1 github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/pkg/conversion github.com/hashicorp/consul/lib/file github.com/hashicorp/consul/logging/monitor -github.com/census-instrumentation/opencensus-proto/gen-go/trace/v1 github.com/imdario/mergo internal/profile github.com/ryanuber/columnize -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/annotations -github.com/hashicorp/consul/vendor/github.com/cncf/udpa/go/udpa/annotations -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/listener/v2 +github.com/census-instrumentation/opencensus-proto/gen-go/trace/v1 +go.opencensus.io/tag gopkg.in/square/go-jose.v2/jwt +github.com/shirou/gopsutil/cpu +github.com/shirou/gopsutil/host +github.com/shirou/gopsutil/mem +github.com/shirou/gopsutil/disk +go.opencensus.io/stats/internal +github.com/hashicorp/consul/vendor/github.com/cncf/udpa/go/udpa/annotations +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/annotations +github.com/hashicorp/consul/vendor/github.com/envoyproxy/protoc-gen-validate/validate +go.opencensus.io/stats +google.golang.org/genproto/googleapis/api/annotations +go.opencensus.io/stats/view github.com/gogo/protobuf/gogoproto -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/type/metadata/v2 -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/type +github.com/hashicorp/consul/agent/debug +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/listener/v2 github.com/hashicorp/consul/proto/pbacl -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/type/tracing/v2 -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/type/matcher net/http/httptrace github.com/hashicorp/go-rootcerts google.golang.org/grpc/credentials @@ -2972,28 +3003,30 @@ github.com/miekg/dns net/http google.golang.org/grpc/resolver -google.golang.org/grpc/peer google.golang.org/grpc/internal/channelz +google.golang.org/grpc/peer +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/type +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/type/metadata/v2 google.golang.org/grpc/balancer -google.golang.org/grpc/internal/resolver/dns google.golang.org/grpc/internal/resolver/passthrough -github.com/hashicorp/consul/proto -github.com/hashicorp/consul/proto/pbcommon +google.golang.org/grpc/internal/resolver/dns google.golang.org/grpc/balancer/base -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2/core google.golang.org/grpc/balancer/roundrobin +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/type/tracing/v2 +github.com/hashicorp/consul/proto +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/type/matcher +github.com/hashicorp/consul/proto/pbcommon github.com/hashicorp/consul/proto/pbconnect +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2/core github.com/hashicorp/consul/proto/pbautoconf -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/filter/network/ext_authz/v2 -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/trace/v2 +github.com/hashicorp/consul/vendor/github.com/coredns/coredns/plugin/pkg/dnsutil +github.com/hashicorp/mdns github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2/cluster github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2/auth +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/filter/network/ext_authz/v2 +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/trace/v2 github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2/route -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2/listener -github.com/hashicorp/consul/vendor/github.com/coredns/coredns/plugin/pkg/dnsutil -github.com/hashicorp/mdns -github.com/armon/go-metrics github.com/tv42/httpunix github.com/hashicorp/go-cleanhttp expvar @@ -3002,186 +3035,186 @@ net/http/httptest github.com/NYTimes/gziphandler github.com/elazarl/go-bindata-assetfs +github.com/armon/go-metrics github.com/docker/go-connections/sockets golang.org/x/net/trace -golang.org/x/net/http2 -net/http/httputil github.com/hashicorp/go-retryablehttp +net/http/httputil +golang.org/x/net/http2 github.com/aws/aws-sdk-go/aws github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/runtime github.com/googleapis/gnostic/compiler golang.org/x/net/context/ctxhttp -github.com/pquerna/cachecontrol/cacheobject -github.com/stretchr/testify/assert github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime +github.com/stretchr/testify/assert +github.com/pquerna/cachecontrol/cacheobject golang.org/x/oauth2/internal github.com/hashicorp/go-msgpack/codec -github.com/armon/go-metrics/datadog -github.com/hashicorp/serf/coordinate +github.com/circonus-labs/circonus-gometrics/api github.com/googleapis/gnostic/OpenAPIv2 github.com/hashicorp/consul/vendor/github.com/hashicorp/go-connlimit -github.com/pquerna/cachecontrol -github.com/circonus-labs/circonus-gometrics/api golang.org/x/oauth2 +github.com/aws/aws-sdk-go/aws/request +github.com/pquerna/cachecontrol +github.com/prometheus/client_golang/prometheus github.com/denverdino/aliyungo/util cloud.google.com/go/compute/metadata -github.com/prometheus/client_golang/prometheus -github.com/hashicorp/consul/api +github.com/armon/go-metrics/datadog +github.com/hashicorp/serf/coordinate google.golang.org/api/googleapi go.opencensus.io/trace/propagation -github.com/aws/aws-sdk-go/aws/request google.golang.org/api/googleapi/transport +go.opencensus.io/plugin/ochttp/propagation/b3 +github.com/hashicorp/consul/api +google.golang.org/api/transport/http/internal/propagation +github.com/hashicorp/go-discover/provider/mdns github.com/coreos/go-oidc golang.org/x/oauth2/jwt -go.opencensus.io/plugin/ochttp/propagation/b3 github.com/denverdino/aliyungo/common -google.golang.org/api/transport/http/internal/propagation go.opencensus.io/plugin/ochttp -github.com/hashicorp/go-discover/provider/mdns -golang.org/x/oauth2/google github.com/gophercloud/gophercloud github.com/packethost/packngo -github.com/hashicorp/consul/internal/go-sso/oidcauth -github.com/denverdino/aliyungo/ecs -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/filter/accesslog/v2 -github.com/hashicorp/consul/sdk/testutil -github.com/circonus-labs/circonus-gometrics/checkmgr -net/http/pprof -github.com/circonus-labs/circonus-gometrics -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/recognizer -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming -github.com/hashicorp/consul/vendor/k8s.io/client-go/tools/clientcmd/api -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/filter/network/tcp_proxy/v2 -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/json -github.com/hashicorp/go-checkpoint +golang.org/x/oauth2/google github.com/stretchr/testify/mock github.com/stretchr/testify/require -github.com/armon/go-metrics/circonus -github.com/hashicorp/go-discover/provider/aliyun github.com/aws/aws-sdk-go/aws/client github.com/aws/aws-sdk-go/aws/corehandlers github.com/hashicorp/memberlist +github.com/aws/aws-sdk-go/private/protocol github.com/hashicorp/raft +github.com/circonus-labs/circonus-gometrics/checkmgr github.com/aws/aws-sdk-go/aws/csm -github.com/hashicorp/net-rpc-msgpackrpc -github.com/aws/aws-sdk-go/private/protocol -github.com/gophercloud/gophercloud/pagination -github.com/gophercloud/gophercloud/openstack/utils -github.com/gophercloud/gophercloud/openstack/identity/v3/tokens +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/recognizer +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/json github.com/aws/aws-sdk-go/aws/ec2metadata -github.com/hashicorp/consul/internal/go-sso/oidcauth/oidcauthtest +github.com/hashicorp/consul/vendor/k8s.io/client-go/tools/clientcmd/api +github.com/hashicorp/consul/internal/go-sso/oidcauth github.com/aws/aws-sdk-go/private/protocol/rest -github.com/aws/aws-sdk-go/private/protocol/query/queryutil github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil +github.com/aws/aws-sdk-go/private/protocol/query/queryutil github.com/aws/aws-sdk-go/private/protocol/json/jsonutil -github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/ec2tokens -github.com/gophercloud/gophercloud/openstack/identity/v2/tenants -github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/oauth1 github.com/prometheus/client_golang/prometheus/push -github.com/gophercloud/gophercloud/openstack/compute/v2/servers -github.com/hashicorp/go-discover/provider/packet +google.golang.org/grpc/internal/transport +github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/api github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds -github.com/aws/aws-sdk-go/aws/credentials/endpointcreds +github.com/circonus-labs/circonus-gometrics +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/net +github.com/hashicorp/net-rpc-msgpackrpc github.com/armon/go-metrics/prometheus -google.golang.org/grpc/internal/transport +github.com/aws/aws-sdk-go/aws/credentials/endpointcreds +github.com/denverdino/aliyungo/ecs +github.com/gophercloud/gophercloud/pagination github.com/aws/aws-sdk-go/aws/signer/v4 -github.com/aws/aws-sdk-go/private/protocol/jsonrpc github.com/aws/aws-sdk-go/private/protocol/query -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/util/net -github.com/hashicorp/consul/vendor/github.com/hashicorp/vault/api -github.com/hashicorp/consul/command/flags +github.com/aws/aws-sdk-go/private/protocol/jsonrpc github.com/aws/aws-sdk-go/private/protocol/ec2query +github.com/gophercloud/gophercloud/openstack/identity/v3/tokens github.com/aws/aws-sdk-go/aws/defaults -github.com/gophercloud/gophercloud/openstack/identity/v2/tokens -github.com/hashicorp/consul/api/watch -github.com/prometheus/client_golang/prometheus/promhttp -github.com/hashicorp/consul/command/intention/finder -github.com/hashicorp/consul/command/kv/impexp +github.com/hashicorp/consul/command/flags github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/watch +github.com/armon/go-metrics/circonus github.com/hashicorp/consul/vendor/k8s.io/client-go/transport -github.com/hashicorp/consul/sentinel -github.com/hashicorp/serf/serf +github.com/gophercloud/gophercloud/openstack/utils +github.com/hashicorp/go-discover/provider/packet +github.com/gophercloud/gophercloud/openstack/identity/v2/tenants +github.com/gophercloud/gophercloud/openstack/compute/v2/servers github.com/aws/aws-sdk-go/service/sts +github.com/hashicorp/serf/serf github.com/aws/aws-sdk-go/service/acmpca -github.com/gophercloud/gophercloud/openstack +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2/listener +github.com/aws/aws-sdk-go/service/ec2 github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/apis/meta/v1 +github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/ec2tokens github.com/hashicorp/consul/command/acl/agenttokens github.com/hashicorp/consul/command/acl/authmethod -github.com/aws/aws-sdk-go/service/ec2 github.com/hashicorp/consul/command/acl/authmethod/delete -github.com/hashicorp/consul/command/acl/bindingrule -github.com/hashicorp/consul/command/acl/token github.com/hashicorp/consul/command/acl/authmethod/create github.com/hashicorp/consul/command/acl/authmethod/list github.com/hashicorp/consul/command/acl/authmethod/read github.com/hashicorp/consul/command/acl/authmethod/update +github.com/hashicorp/consul/command/acl/bindingrule +github.com/hashicorp/consul/command/acl/token +github.com/hashicorp/consul/command/acl/policy github.com/hashicorp/consul/command/acl/bindingrule/create github.com/hashicorp/consul/command/acl/bindingrule/list -github.com/hashicorp/consul/command/acl/bootstrap -github.com/hashicorp/consul/command/acl/policy github.com/hashicorp/consul/command/acl/role +github.com/hashicorp/consul/command/acl/bootstrap +github.com/hashicorp/consul/command/acl/policy/list github.com/hashicorp/consul/command/acl/token/list +github.com/aws/aws-sdk-go/service/sts/stsiface +github.com/gophercloud/gophercloud/openstack/identity/v2/tokens +github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/oauth1 github.com/hashicorp/go-raftchunking +github.com/hashicorp/consul/command/acl/role/list github.com/hashicorp/consul/snapshot -github.com/aws/aws-sdk-go/service/sts/stsiface github.com/hashicorp/raft-boltdb -github.com/hashicorp/consul/command/acl/policy/list -github.com/hashicorp/go-discover/provider/os +github.com/hashicorp/go-discover/provider/aliyun +github.com/aws/aws-sdk-go/aws/credentials/stscreds +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/filter/accesslog/v2 +github.com/hashicorp/consul/lib +github.com/hashicorp/consul/agent/consul/autopilot +github.com/hashicorp/consul/api/watch +github.com/hashicorp/consul/sdk/testutil +github.com/prometheus/client_golang/prometheus/promhttp +net/http/pprof +github.com/hashicorp/go-checkpoint github.com/hashicorp/consul/command/catalog -github.com/hashicorp/consul/command/acl/role/list +github.com/aws/aws-sdk-go/aws/session github.com/hashicorp/consul/command/catalog/list/dc +google.golang.org/grpc github.com/hashicorp/consul/command/catalog/list/nodes github.com/hashicorp/consul/command/catalog/list/services -github.com/hashicorp/consul/lib -github.com/hashicorp/consul/agent/consul/autopilot -github.com/aws/aws-sdk-go/aws/credentials/stscreds +github.com/hashicorp/consul/agent/cache +github.com/hashicorp/consul/agent/ae github.com/hashicorp/consul/command/config github.com/hashicorp/consul/command/config/delete github.com/hashicorp/consul/command/config/list github.com/hashicorp/consul/command/config/read +github.com/gophercloud/gophercloud/openstack github.com/hashicorp/consul/command/config/write github.com/hashicorp/consul/command/connect github.com/hashicorp/consul/command/connect/ca github.com/hashicorp/consul/command/connect/ca/get github.com/hashicorp/consul/command/connect/ca/set +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/filter/network/tcp_proxy/v2 github.com/hashicorp/consul/command/connect/envoy/pipe-bootstrap -github.com/aws/aws-sdk-go/aws/session -github.com/hashicorp/consul/command/intention/create -github.com/hashicorp/consul/command/debug +github.com/hashicorp/consul/command/intention/finder github.com/hashicorp/consul/command/event +github.com/hashicorp/consul/command/debug github.com/hashicorp/consul/command/exec -github.com/hashicorp/consul/command/forceleave github.com/hashicorp/consul/command/info +github.com/hashicorp/consul/command/forceleave github.com/hashicorp/consul/command/intention +github.com/hashicorp/consul/agent/structs github.com/hashicorp/consul/command/intention/check +github.com/hashicorp/consul/command/intention/match +github.com/hashicorp/consul/command/intention/create github.com/hashicorp/consul/command/intention/delete github.com/hashicorp/consul/command/intention/get -github.com/hashicorp/consul/command/intention/match -github.com/hashicorp/consul/agent/cache -github.com/hashicorp/consul/agent/ae +github.com/hashicorp/go-discover/provider/os github.com/hashicorp/consul/command/join github.com/hashicorp/consul/command/keygen github.com/hashicorp/consul/command/kv -google.golang.org/grpc github.com/hashicorp/consul/command/kv/del -github.com/hashicorp/consul/command/kv/exp +github.com/hashicorp/consul/command/kv/impexp github.com/hashicorp/consul/command/kv/get -github.com/hashicorp/consul/command/kv/imp github.com/hashicorp/consul/command/kv/put github.com/hashicorp/consul/command/leave github.com/hashicorp/consul/command/login github.com/hashicorp/consul/command/logout github.com/hashicorp/consul/command/maint -github.com/hashicorp/consul/command/members github.com/hashicorp/consul/command/monitor +github.com/hashicorp/consul/command/members github.com/hashicorp/consul/command/operator +github.com/hashicorp/consul/command/kv/exp +github.com/hashicorp/consul/command/kv/imp github.com/hashicorp/consul/command/operator/autopilot github.com/hashicorp/consul/command/operator/autopilot/get github.com/hashicorp/consul/command/operator/autopilot/set github.com/hashicorp/consul/command/operator/raft -github.com/hashicorp/consul/agent/structs -github.com/hashicorp/consul/command/operator/raft/removepeer github.com/hashicorp/consul/command/operator/raft/listpeers +github.com/hashicorp/consul/command/operator/raft/removepeer github.com/hashicorp/consul/command/reload github.com/hashicorp/consul/command/rtt github.com/hashicorp/consul/command/snapshot @@ -3191,52 +3224,59 @@ github.com/hashicorp/consul/command/tls github.com/hashicorp/consul/command/tls/ca github.com/hashicorp/consul/command/tls/cert +github.com/hashicorp/consul/internal/go-sso/oidcauth/oidcauthtest +github.com/hashicorp/consul/sentinel +google.golang.org/grpc/health/grpc_health_v1 +github.com/googleapis/gax-go/v2 +google.golang.org/api/internal +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/service/auth/v2 +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2 +google.golang.org/api/internal/gensupport +google.golang.org/api/option +google.golang.org/api/option/internaloption +google.golang.org/api/transport/http +github.com/hashicorp/consul/vendor/k8s.io/api/authentication/v1 github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/api/errors github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/protobuf github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured -github.com/hashicorp/consul/vendor/k8s.io/api/admissionregistration/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/api/admissionregistration/v1 +github.com/hashicorp/consul/vendor/k8s.io/api/admissionregistration/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/api/auditregistration/v1alpha1 github.com/hashicorp/consul/vendor/k8s.io/api/authentication/v1beta1 +github.com/hashicorp/consul/vendor/k8s.io/api/core/v1 github.com/hashicorp/consul/vendor/k8s.io/api/authorization/v1 github.com/hashicorp/consul/vendor/k8s.io/api/authorization/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/api/certificates/v1beta1 -github.com/hashicorp/consul/vendor/k8s.io/api/authentication/v1 -github.com/hashicorp/consul/vendor/k8s.io/api/core/v1 -google.golang.org/grpc/health/grpc_health_v1 github.com/hashicorp/consul/vendor/k8s.io/api/coordination/v1 github.com/hashicorp/consul/vendor/k8s.io/api/coordination/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/api/rbac/v1 -github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/versioning github.com/hashicorp/consul/vendor/k8s.io/api/rbac/v1alpha1 +github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer/versioning github.com/hashicorp/consul/vendor/k8s.io/api/rbac/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/client-go/pkg/apis/clientauthentication github.com/hashicorp/consul/vendor/k8s.io/client-go/rest/watch github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/api/meta github.com/hashicorp/consul/vendor/k8s.io/apimachinery/pkg/runtime/serializer -github.com/googleapis/gax-go/v2 -google.golang.org/api/internal +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/service/auth/v2alpha +google.golang.org/api/compute/v1 github.com/hashicorp/consul/vendor/k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1 github.com/hashicorp/consul/vendor/k8s.io/client-go/pkg/apis/clientauthentication/v1beta1 -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/api/v2 -google.golang.org/api/internal/gensupport -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/service/auth/v2 -google.golang.org/api/option github.com/hashicorp/consul/vendor/k8s.io/client-go/plugin/pkg/client/auth/exec -google.golang.org/api/option/internaloption -google.golang.org/api/transport/http github.com/hashicorp/consul/vendor/k8s.io/client-go/rest -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/service/auth/v2alpha -google.golang.org/api/compute/v1 +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2 +github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2 github.com/hashicorp/consul/command/acl -github.com/hashicorp/consul/agent/connect github.com/hashicorp/consul/agent/consul/prepared_query -github.com/hashicorp/consul/agent/checks +github.com/hashicorp/consul/agent/connect github.com/hashicorp/consul/agent/consul/authmethod -github.com/hashicorp/consul/agent/local +github.com/hashicorp/consul/agent/metadata +github.com/hashicorp/consul/agent/checks github.com/hashicorp/consul/agent/mock +github.com/hashicorp/consul/agent/local github.com/hashicorp/consul/testrpc -github.com/hashicorp/consul/agent/metadata +github.com/hashicorp/consul/agent/router +github.com/hashicorp/consul/agent/consul/authmethod/ssoauth +github.com/hashicorp/consul/agent/consul/authmethod/testauth github.com/hashicorp/consul/command/acl/bindingrule/delete github.com/hashicorp/consul/command/acl/bindingrule/read github.com/hashicorp/consul/command/acl/bindingrule/update @@ -3244,8 +3284,8 @@ github.com/hashicorp/consul/command/acl/policy/delete github.com/hashicorp/consul/command/acl/policy/read github.com/hashicorp/consul/command/acl/policy/update -github.com/hashicorp/consul/command/acl/role/create github.com/hashicorp/consul/command/acl/role/delete +github.com/hashicorp/consul/command/acl/role/create github.com/hashicorp/consul/command/acl/role/read github.com/hashicorp/consul/command/acl/role/update github.com/hashicorp/consul/command/acl/rules @@ -3255,71 +3295,68 @@ github.com/hashicorp/consul/command/acl/token/read github.com/hashicorp/consul/command/acl/token/update github.com/hashicorp/consul/agent/consul/discoverychain -github.com/hashicorp/consul/agent/consul/authmethod/ssoauth github.com/hashicorp/consul/tlsutil -github.com/hashicorp/consul/agent/router github.com/hashicorp/consul/connect -github.com/hashicorp/consul/agent/consul/authmethod/testauth github.com/hashicorp/consul/connect/certgen +github.com/hashicorp/consul/connect/proxy github.com/hashicorp/consul/agent/consul/state github.com/hashicorp/consul/command/tls/ca/create github.com/hashicorp/consul/command/tls/cert/create github.com/hashicorp/consul/agent/pool -github.com/hashicorp/consul/connect/proxy -github.com/hashicorp/consul/agent/consul/wanfed github.com/hashicorp/consul/command/connect/proxy -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2 -github.com/hashicorp/consul/vendor/github.com/envoyproxy/go-control-plane/envoy/config/filter/network/http_connection_manager/v2 -github.com/hashicorp/consul/agent/connect/ca +github.com/hashicorp/consul/agent/consul/wanfed github.com/hashicorp/consul/agent/consul/fsm -github.com/hashicorp/consul/vendor/k8s.io/api/apps/v1 +github.com/hashicorp/consul/agent/connect/ca github.com/hashicorp/consul/vendor/k8s.io/api/apps/v1beta1 +github.com/hashicorp/consul/vendor/k8s.io/api/apps/v1 +github.com/hashicorp/consul/vendor/k8s.io/api/autoscaling/v1 github.com/hashicorp/consul/vendor/k8s.io/api/apps/v1beta2 -github.com/hashicorp/consul/vendor/k8s.io/api/batch/v1 github.com/hashicorp/consul/vendor/k8s.io/api/autoscaling/v2beta1 -github.com/hashicorp/consul/vendor/k8s.io/api/discovery/v1alpha1 +github.com/hashicorp/consul/vendor/k8s.io/api/batch/v1 +github.com/hashicorp/consul/vendor/k8s.io/api/node/v1beta1 +github.com/hashicorp/consul/vendor/k8s.io/api/networking/v1 github.com/hashicorp/consul/vendor/k8s.io/api/autoscaling/v2beta2 github.com/hashicorp/consul/vendor/k8s.io/api/events/v1beta1 -github.com/hashicorp/consul/vendor/k8s.io/api/networking/v1 -github.com/hashicorp/consul/vendor/k8s.io/api/node/v1alpha1 github.com/hashicorp/consul/vendor/k8s.io/api/networking/v1beta1 -github.com/hashicorp/consul/vendor/k8s.io/api/node/v1beta1 -github.com/hashicorp/consul/vendor/k8s.io/api/autoscaling/v1 +github.com/hashicorp/consul/vendor/k8s.io/api/node/v1alpha1 +github.com/hashicorp/consul/vendor/k8s.io/api/discovery/v1alpha1 github.com/hashicorp/consul/vendor/k8s.io/api/policy/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/api/scheduling/v1 github.com/hashicorp/consul/vendor/k8s.io/api/scheduling/v1alpha1 github.com/hashicorp/consul/vendor/k8s.io/api/scheduling/v1beta1 -github.com/hashicorp/consul/vendor/k8s.io/api/batch/v1beta1 -github.com/hashicorp/consul/vendor/k8s.io/api/batch/v2alpha1 github.com/hashicorp/consul/vendor/k8s.io/api/settings/v1alpha1 github.com/hashicorp/consul/vendor/k8s.io/api/storage/v1 github.com/hashicorp/consul/vendor/k8s.io/api/storage/v1alpha1 +github.com/hashicorp/consul/vendor/k8s.io/api/batch/v1beta1 +github.com/hashicorp/consul/vendor/k8s.io/api/batch/v2alpha1 github.com/hashicorp/consul/vendor/k8s.io/api/storage/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/client-go/tools/reference github.com/hashicorp/consul/vendor/k8s.io/api/extensions/v1beta1 +github.com/hashicorp/go-discover/provider/gce github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/scheme +github.com/hashicorp/go-discover/provider/aws +github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/admissionregistration/v1 +github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1 +github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/apps/v1 +github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/apps/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/client-go/discovery +github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/batch/v1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/authentication/v1beta1 -github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1 -github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/authorization/v1beta1 -github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/admissionregistration/v1 -github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/autoscaling/v1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/autoscaling/v2beta1 -github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/apps/v1beta1 -github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/apps/v1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/authorization/v1 -github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/admissionregistration/v1beta1 -github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/apps/v1beta2 -github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/authentication/v1 +github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/autoscaling/v1 +github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/auditregistration/v1alpha1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/autoscaling/v2beta2 -github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/batch/v1 +github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/authentication/v1 +github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/authorization/v1beta1 +github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/apps/v1beta2 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/batch/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/batch/v2alpha1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/certificates/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/coordination/v1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/coordination/v1beta1 -github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/discovery/v1alpha1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/core/v1 +github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/discovery/v1alpha1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/events/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/extensions/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/networking/v1 @@ -3336,30 +3373,28 @@ github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/settings/v1alpha1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/storage/v1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/storage/v1alpha1 -github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/storage/v1beta1 -github.com/hashicorp/go-discover/provider/aws -github.com/hashicorp/go-discover/provider/gce github.com/hashicorp/go-discover +github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes/typed/storage/v1beta1 github.com/hashicorp/consul/vendor/k8s.io/client-go/kubernetes github.com/hashicorp/consul/agent/consul/authmethod/kubeauth github.com/hashicorp/consul/agent/consul -github.com/hashicorp/consul/agent/cache-types github.com/hashicorp/consul/command/version +github.com/hashicorp/consul/agent/cache-types github.com/hashicorp/consul/agent/config github.com/hashicorp/consul/agent/proxycfg -github.com/hashicorp/consul/command/services github.com/hashicorp/consul/command/validate +github.com/hashicorp/consul/command/services github.com/hashicorp/consul/agent/auto-config -github.com/hashicorp/consul/agent/xds -github.com/hashicorp/consul/command/services/register github.com/hashicorp/consul/command/services/deregister +github.com/hashicorp/consul/command/services/register +github.com/hashicorp/consul/agent/xds github.com/hashicorp/consul/command/connect/envoy github.com/hashicorp/consul/agent github.com/hashicorp/consul/command/agent -github.com/hashicorp/consul/command/watch -github.com/hashicorp/consul/command/keyring github.com/hashicorp/consul/command/connect/expose +github.com/hashicorp/consul/command/keyring github.com/hashicorp/consul/command/lock +github.com/hashicorp/consul/command/watch github.com/hashicorp/consul/command github.com/hashicorp/consul make[1]: Leaving directory '/build/consul-1.8.7+dfsg1' @@ -3371,7 +3406,7 @@ cd _build && go test -vet=off -v -p 4 -short -failfast -timeout 8m github.com/hashicorp/consul github.com/hashicorp/consul/acl github.com/hashicorp/consul/agent github.com/hashicorp/consul/agent/ae github.com/hashicorp/consul/agent/auto-config github.com/hashicorp/consul/agent/config github.com/hashicorp/consul/agent/debug github.com/hashicorp/consul/agent/dns github.com/hashicorp/consul/agent/exec github.com/hashicorp/consul/agent/local github.com/hashicorp/consul/agent/metadata github.com/hashicorp/consul/agent/mock github.com/hashicorp/consul/agent/pool github.com/hashicorp/consul/agent/router github.com/hashicorp/consul/agent/structs github.com/hashicorp/consul/agent/systemd github.com/hashicorp/consul/agent/token github.com/hashicorp/consul/agent/xds github.com/hashicorp/consul/agent/xds/proxysupport github.com/hashicorp/consul/command github.com/hashicorp/consul/command/acl github.com/hashicorp/consul/command/acl/agenttokens github.com/hashicorp/consul/command/acl/authmethod github.com/hashicorp/consul/command/acl/authmethod/create github.com/hashicorp/consul/command/acl/authmethod/delete github.com/hashicorp/consul/command/acl/authmethod/list github.com/hashicorp/consul/command/acl/authmethod/read github.com/hashicorp/consul/command/acl/authmethod/update github.com/hashicorp/consul/command/acl/bindingrule github.com/hashicorp/consul/command/acl/bindingrule/create github.com/hashicorp/consul/command/acl/bindingrule/delete github.com/hashicorp/consul/command/acl/bindingrule/list github.com/hashicorp/consul/command/acl/bindingrule/read github.com/hashicorp/consul/command/acl/bindingrule/update github.com/hashicorp/consul/command/acl/bootstrap github.com/hashicorp/consul/command/acl/policy github.com/hashicorp/consul/command/acl/policy/create github.com/hashicorp/consul/command/acl/policy/delete github.com/hashicorp/consul/command/acl/policy/list github.com/hashicorp/consul/command/acl/policy/read github.com/hashicorp/consul/command/acl/policy/update github.com/hashicorp/consul/command/acl/role github.com/hashicorp/consul/command/acl/role/create github.com/hashicorp/consul/command/acl/role/delete github.com/hashicorp/consul/command/acl/role/list github.com/hashicorp/consul/command/acl/role/read github.com/hashicorp/consul/command/acl/role/update github.com/hashicorp/consul/command/acl/rules github.com/hashicorp/consul/command/acl/token github.com/hashicorp/consul/command/acl/token/clone github.com/hashicorp/consul/command/acl/token/create github.com/hashicorp/consul/command/acl/token/delete github.com/hashicorp/consul/command/acl/token/list github.com/hashicorp/consul/command/acl/token/read github.com/hashicorp/consul/command/acl/token/update github.com/hashicorp/consul/command/agent github.com/hashicorp/consul/command/catalog github.com/hashicorp/consul/command/catalog/list/dc github.com/hashicorp/consul/command/catalog/list/nodes github.com/hashicorp/consul/command/catalog/list/services github.com/hashicorp/consul/command/config github.com/hashicorp/consul/command/config/delete github.com/hashicorp/consul/command/config/list github.com/hashicorp/consul/command/config/read github.com/hashicorp/consul/command/config/write github.com/hashicorp/consul/command/connect github.com/hashicorp/consul/command/connect/ca github.com/hashicorp/consul/command/connect/ca/get github.com/hashicorp/consul/command/connect/ca/set github.com/hashicorp/consul/command/connect/envoy github.com/hashicorp/consul/command/connect/envoy/pipe-bootstrap github.com/hashicorp/consul/command/connect/expose github.com/hashicorp/consul/command/connect/proxy github.com/hashicorp/consul/command/event github.com/hashicorp/consul/command/exec github.com/hashicorp/consul/command/forceleave github.com/hashicorp/consul/command/helpers github.com/hashicorp/consul/command/info github.com/hashicorp/consul/command/intention github.com/hashicorp/consul/command/intention/check github.com/hashicorp/consul/command/intention/create github.com/hashicorp/consul/command/intention/delete github.com/hashicorp/consul/command/intention/finder github.com/hashicorp/consul/command/intention/get github.com/hashicorp/consul/command/intention/match github.com/hashicorp/consul/command/join github.com/hashicorp/consul/command/keygen github.com/hashicorp/consul/command/keyring github.com/hashicorp/consul/command/kv github.com/hashicorp/consul/command/kv/del github.com/hashicorp/consul/command/kv/exp github.com/hashicorp/consul/command/kv/get github.com/hashicorp/consul/command/kv/imp github.com/hashicorp/consul/command/kv/impexp github.com/hashicorp/consul/command/kv/put github.com/hashicorp/consul/command/leave github.com/hashicorp/consul/command/lock github.com/hashicorp/consul/command/login github.com/hashicorp/consul/command/logout github.com/hashicorp/consul/command/maint github.com/hashicorp/consul/command/members github.com/hashicorp/consul/command/operator github.com/hashicorp/consul/command/operator/autopilot github.com/hashicorp/consul/command/operator/autopilot/get github.com/hashicorp/consul/command/operator/autopilot/set github.com/hashicorp/consul/command/operator/raft github.com/hashicorp/consul/command/operator/raft/listpeers github.com/hashicorp/consul/command/operator/raft/removepeer github.com/hashicorp/consul/command/reload github.com/hashicorp/consul/command/rtt github.com/hashicorp/consul/command/services github.com/hashicorp/consul/command/services/deregister github.com/hashicorp/consul/command/services/register github.com/hashicorp/consul/command/snapshot github.com/hashicorp/consul/command/snapshot/inspect github.com/hashicorp/consul/command/snapshot/restore github.com/hashicorp/consul/command/snapshot/save github.com/hashicorp/consul/command/validate github.com/hashicorp/consul/command/version github.com/hashicorp/consul/command/watch github.com/hashicorp/consul/connect github.com/hashicorp/consul/connect/certgen github.com/hashicorp/consul/connect/proxy github.com/hashicorp/consul/internal/go-sso/oidcauth github.com/hashicorp/consul/internal/go-sso/oidcauth/internal/strutil github.com/hashicorp/consul/internal/go-sso/oidcauth/oidcauthtest github.com/hashicorp/consul/ipaddr github.com/hashicorp/consul/lib github.com/hashicorp/consul/lib/decode github.com/hashicorp/consul/lib/file github.com/hashicorp/consul/lib/semaphore github.com/hashicorp/consul/lib/stringslice github.com/hashicorp/consul/lib/template github.com/hashicorp/consul/logging github.com/hashicorp/consul/logging/monitor github.com/hashicorp/consul/proto github.com/hashicorp/consul/proto/pbacl github.com/hashicorp/consul/proto/pbautoconf github.com/hashicorp/consul/proto/pbcommon github.com/hashicorp/consul/proto/pbconfig github.com/hashicorp/consul/proto/pbconnect github.com/hashicorp/consul/sdk/freeport github.com/hashicorp/consul/sdk/testutil github.com/hashicorp/consul/sdk/testutil/retry github.com/hashicorp/consul/sentinel github.com/hashicorp/consul/service_os github.com/hashicorp/consul/snapshot github.com/hashicorp/consul/testrpc github.com/hashicorp/consul/tlsutil github.com/hashicorp/consul/types github.com/hashicorp/consul/version testing: warning: no tests to run PASS -ok github.com/hashicorp/consul 0.087s [no tests to run] +ok github.com/hashicorp/consul 0.055s [no tests to run] === RUN TestACL === RUN TestACL/DenyAll === RUN TestACL/DenyAll/DenyACLRead @@ -3977,7 +4012,7 @@ === RUN TestACL/KeyWritePrefixDefaultAllow === RUN TestACL/KeyWritePrefixDefaultAllow/KeyWritePrefixDenied.Prefix(foo) === RUN TestACL/KeyWritePrefixDefaultAllow/KeyWritePrefixAllowed.Prefix(bar) ---- PASS: TestACL (0.02s) +--- PASS: TestACL (0.04s) --- PASS: TestACL/DenyAll (0.00s) --- PASS: TestACL/DenyAll/DenyACLRead (0.00s) --- PASS: TestACL/DenyAll/DenyACLWrite (0.00s) @@ -4149,7 +4184,7 @@ --- PASS: TestACL/KeyringDefaultAllowPolicyDeny (0.00s) --- PASS: TestACL/KeyringDefaultAllowPolicyDeny/ReadDenied (0.00s) --- PASS: TestACL/KeyringDefaultAllowPolicyDeny/WriteDenied (0.00s) - --- PASS: TestACL/KeyringDefaultAllowPolicyRead (0.00s) + --- PASS: TestACL/KeyringDefaultAllowPolicyRead (0.01s) --- PASS: TestACL/KeyringDefaultAllowPolicyRead/ReadAllowed (0.00s) --- PASS: TestACL/KeyringDefaultAllowPolicyRead/WriteDenied (0.00s) --- PASS: TestACL/KeyringDefaultAllowPolicyWrite (0.00s) @@ -4310,7 +4345,7 @@ --- PASS: TestACL/SessionDefaultAllow/ChildRWSuffixWriteAllowed.Prefix(child-rw-prefix) (0.00s) --- PASS: TestACL/SessionDefaultAllow/ChildOverrideReadAllowed.Prefix(override) (0.00s) --- PASS: TestACL/SessionDefaultAllow/ChildOverrideWriteAllowed.Prefix(override) (0.00s) - --- PASS: TestACL/Parent (0.00s) + --- PASS: TestACL/Parent (0.01s) --- PASS: TestACL/Parent/KeyReadDenied.Prefix(other) (0.00s) --- PASS: TestACL/Parent/KeyWriteDenied.Prefix(other) (0.00s) --- PASS: TestACL/Parent/KeyWritePrefixDenied.Prefix(other) (0.00s) @@ -4651,7 +4686,7 @@ === RUN TestPolicySourceParse/Bad_Policy_-_Operator === RUN TestPolicySourceParse/Keyring_Empty === RUN TestPolicySourceParse/Operator_Empty ---- PASS: TestPolicySourceParse (0.00s) +--- PASS: TestPolicySourceParse (0.01s) --- PASS: TestPolicySourceParse/Legacy_Basic (0.00s) --- PASS: TestPolicySourceParse/Legacy_(JSON) (0.00s) --- PASS: TestPolicySourceParse/Service_No_Intentions_(Legacy) (0.00s) @@ -4687,7 +4722,7 @@ === RUN TestMergePolicies/Write_Precedence === RUN TestMergePolicies/Deny_Precedence === RUN TestMergePolicies/Read_Precedence ---- PASS: TestMergePolicies (0.00s) +--- PASS: TestMergePolicies (0.01s) --- PASS: TestMergePolicies/Agents (0.00s) --- PASS: TestMergePolicies/Events (0.00s) --- PASS: TestMergePolicies/Node (0.00s) @@ -4696,7 +4731,7 @@ --- PASS: TestMergePolicies/Sessions (0.00s) --- PASS: TestMergePolicies/Prepared_Queries (0.00s) --- PASS: TestMergePolicies/Write_Precedence (0.00s) - --- PASS: TestMergePolicies/Deny_Precedence (0.00s) + --- PASS: TestMergePolicies/Deny_Precedence (0.01s) --- PASS: TestMergePolicies/Read_Precedence (0.00s) === RUN TestRulesTranslate --- PASS: TestRulesTranslate (0.00s) @@ -4748,126 +4783,54 @@ === PAUSE TestStaticAuthorizer === CONT TestACL_Enforce === RUN TestACL_Enforce/acl/read/Deny -=== CONT TestStaticAuthorizer -=== RUN TestStaticAuthorizer/AllowAll -=== CONT TestACL_Enforce/acl/read/Deny authorizer_test.go:618: PASS: ACLRead(*acl.AuthorizerContext) === RUN TestACL_Enforce/acl/read/Allow - authorizer_test.go:618: PASS: ACLRead(*acl.AuthorizerContext) -=== RUN TestACL_Enforce/acl/write/Deny - authorizer_test.go:618: PASS: ACLWrite(*acl.AuthorizerContext) -=== RUN TestACL_Enforce/acl/write/Allow - authorizer_test.go:618: PASS: ACLWrite(*acl.AuthorizerContext) -=== RUN TestACL_Enforce/acl/list/Deny -=== RUN TestACL_Enforce/operator/read/Deny - authorizer_test.go:618: PASS: OperatorRead(*acl.AuthorizerContext) -=== RUN TestACL_Enforce/operator/read/Allow - authorizer_test.go:618: PASS: OperatorRead(*acl.AuthorizerContext) -=== RUN TestACL_Enforce/operator/write/Deny - authorizer_test.go:618: PASS: OperatorWrite(*acl.AuthorizerContext) -=== RUN TestACL_Enforce/operator/write/Allow - authorizer_test.go:618: PASS: OperatorWrite(*acl.AuthorizerContext) -=== RUN TestACL_Enforce/operator/list/Deny -=== RUN TestACL_Enforce/keyring/read/Deny - authorizer_test.go:618: PASS: KeyringRead(*acl.AuthorizerContext) -=== RUN TestACL_Enforce/keyring/read/Allow - authorizer_test.go:618: PASS: KeyringRead(*acl.AuthorizerContext) -=== RUN TestACL_Enforce/keyring/write/Deny - authorizer_test.go:618: PASS: KeyringWrite(*acl.AuthorizerContext) -=== RUN TestACL_Enforce/keyring/write/Allow - authorizer_test.go:618: PASS: KeyringWrite(*acl.AuthorizerContext) -=== RUN TestACL_Enforce/keyring/list/Deny -=== RUN TestACL_Enforce/agent/foo/read/Deny - authorizer_test.go:618: PASS: AgentRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/agent/foo/read/Allow - authorizer_test.go:618: PASS: AgentRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/agent/foo/write/Deny - authorizer_test.go:618: PASS: AgentWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/agent/foo/write/Allow - authorizer_test.go:618: PASS: AgentWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/agent/foo/list/Deny -=== RUN TestACL_Enforce/event/foo/read/Deny - authorizer_test.go:618: PASS: EventRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/event/foo/read/Allow - authorizer_test.go:618: PASS: EventRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/event/foo/write/Deny - authorizer_test.go:618: PASS: EventWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/event/foo/write/Allow - authorizer_test.go:618: PASS: EventWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/event/foo/list/Deny -=== RUN TestACL_Enforce/intention/foo/read/Deny - authorizer_test.go:618: PASS: IntentionRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/intention/foo/read/Allow - authorizer_test.go:618: PASS: IntentionRead(string,*acl.AuthorizerContext) -=== PAUSE TestStaticAuthorizer/AllowAll -=== RUN TestACL_Enforce/intention/foo/write/Deny -=== RUN TestStaticAuthorizer/DenyAll -=== PAUSE TestStaticAuthorizer/DenyAll -=== RUN TestStaticAuthorizer/ManageAll -=== PAUSE TestStaticAuthorizer/ManageAll -=== CONT TestStaticAuthorizer/AllowAll -=== CONT TestACL_Enforce/intention/foo/write/Deny - authorizer_test.go:618: PASS: IntentionWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/intention/foo/write/Allow - authorizer_test.go:618: PASS: IntentionWrite(string,*acl.AuthorizerContext) === CONT TestAllAllowed -=== RUN TestACL_Enforce/intention/foo/list/Deny -=== RUN TestAllAllowed/prefix-allow-other-write-prefix -=== RUN TestACL_Enforce/node/foo/read/Deny -=== RUN TestAllAllowed/prefix-allow-other-write-exact -=== RUN TestAllAllowed/prefix-allow-other-list-prefix -=== RUN TestAllAllowed/no-rules-default -=== CONT TestACL_Enforce/node/foo/read/Deny - authorizer_test.go:618: PASS: NodeRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/node/foo/read/Allow -=== RUN TestAllAllowed/prefix-list-allowed -=== RUN TestAllAllowed/prefix-deny -=== RUN TestAllAllowed/prefix-allow-other-read-prefix -=== CONT TestACL_Enforce/node/foo/read/Allow - authorizer_test.go:618: PASS: NodeRead(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/node/foo/write/Deny -=== RUN TestAllAllowed/prefix-allow-other-read-exact -=== RUN TestAllAllowed/prefix-allow-other-deny-prefix -=== CONT TestStaticAuthorizer/DenyAll -=== RUN TestAllAllowed/prefix-allow-other-deny-exact -=== RUN TestAllAllowed/prefix-write-allowed +=== CONT TestACL_Enforce/acl/read/Allow + authorizer_test.go:618: PASS: ACLRead(*acl.AuthorizerContext) === CONT TestAnyAllowed -=== RUN TestAllAllowed/prefix-read-allowed -=== RUN TestAllAllowed/prefix-allow-other-list-exact -=== RUN TestAnyAllowed/prefix-list-allowed ---- PASS: TestAllAllowed (0.00s) - --- PASS: TestAllAllowed/prefix-allow-other-write-prefix (0.00s) - --- PASS: TestAllAllowed/prefix-allow-other-write-exact (0.00s) - --- PASS: TestAllAllowed/prefix-allow-other-list-prefix (0.00s) - --- PASS: TestAllAllowed/no-rules-default (0.00s) - --- PASS: TestAllAllowed/prefix-list-allowed (0.00s) - --- PASS: TestAllAllowed/prefix-deny (0.00s) - --- PASS: TestAllAllowed/prefix-allow-other-read-prefix (0.00s) - --- PASS: TestAllAllowed/prefix-allow-other-read-exact (0.00s) - --- PASS: TestAllAllowed/prefix-allow-other-deny-prefix (0.00s) - --- PASS: TestAllAllowed/prefix-allow-other-deny-exact (0.00s) - --- PASS: TestAllAllowed/prefix-write-allowed (0.00s) - --- PASS: TestAllAllowed/prefix-read-allowed (0.00s) - --- PASS: TestAllAllowed/prefix-allow-other-list-exact (0.00s) === CONT TestPolicyAuthorizer -=== RUN TestAnyAllowed/prefix-read-allowed +=== CONT TestChainedAuthorizer +=== CONT TestStaticAuthorizer +=== RUN TestAllAllowed/prefix-allow-other-deny-prefix +=== RUN TestAnyAllowed/prefix-deny-other-list-exact +=== RUN TestAnyAllowed/prefix-deny-other-read-exact +=== RUN TestAnyAllowed/prefix-deny-other-deny-prefix +=== RUN TestAnyAllowed/no-rules-default +=== RUN TestAnyAllowed/prefix-list-allowed +=== RUN TestAnyAllowed/prefix-deny-other-list-prefix === RUN TestAnyAllowed/prefix-deny-other-write-prefix +=== RUN TestAnyAllowed/prefix-deny-other-write-exact +=== RUN TestAnyAllowed/prefix-deny-other-read-prefix +=== RUN TestAnyAllowed/prefix-deny-other-deny-exact +=== RUN TestAnyAllowed/prefix-write-allowed +=== RUN TestAnyAllowed/prefix-read-allowed +=== RUN TestAnyAllowed/prefix-deny +--- PASS: TestAnyAllowed (0.03s) + --- PASS: TestAnyAllowed/prefix-deny-other-list-exact (0.00s) + --- PASS: TestAnyAllowed/prefix-deny-other-read-exact (0.00s) + --- PASS: TestAnyAllowed/prefix-deny-other-deny-prefix (0.00s) + --- PASS: TestAnyAllowed/no-rules-default (0.00s) + --- PASS: TestAnyAllowed/prefix-list-allowed (0.00s) + --- PASS: TestAnyAllowed/prefix-deny-other-list-prefix (0.00s) + --- PASS: TestAnyAllowed/prefix-deny-other-write-prefix (0.00s) + --- PASS: TestAnyAllowed/prefix-deny-other-write-exact (0.00s) + --- PASS: TestAnyAllowed/prefix-deny-other-read-prefix (0.00s) + --- PASS: TestAnyAllowed/prefix-deny-other-deny-exact (0.00s) + --- PASS: TestAnyAllowed/prefix-write-allowed (0.00s) + --- PASS: TestAnyAllowed/prefix-read-allowed (0.00s) + --- PASS: TestAnyAllowed/prefix-deny (0.00s) +=== RUN TestACL_Enforce/acl/write/Deny + authorizer_test.go:618: PASS: ACLWrite(*acl.AuthorizerContext) === RUN TestPolicyAuthorizer/Intention_Wildcards_-_any_default === PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_any_default === RUN TestPolicyAuthorizer/Defaults -=== RUN TestAnyAllowed/prefix-deny-other-write-exact === PAUSE TestPolicyAuthorizer/Defaults === RUN TestPolicyAuthorizer/Prefer_Exact_Matches -=== RUN TestAnyAllowed/prefix-deny-other-list-prefix === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches === RUN TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied === PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied === RUN TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed -=== RUN TestAnyAllowed/prefix-deny-other-read-prefix -=== CONT TestACL_Enforce/node/foo/write/Deny - authorizer_test.go:618: PASS: NodeWrite(string,*acl.AuthorizerContext) -=== RUN TestACL_Enforce/node/foo/write/Allow -=== RUN TestAnyAllowed/prefix-deny-other-read-exact === PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed === RUN TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed === PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed @@ -4879,7 +4842,6 @@ === RUN TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AllDenied.Prefix(*) === PAUSE TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AllDenied.Prefix(*) === CONT TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AnyDefault.Prefix(*) -=== CONT TestChainedAuthorizer === RUN TestChainedAuthorizer/No_Authorizers === PAUSE TestChainedAuthorizer/No_Authorizers === RUN TestChainedAuthorizer/Authorizer_Defaults @@ -5078,6 +5040,14 @@ === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(food) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(food) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(food) +=== CONT TestChainedAuthorizer/Authorizer_No_Defaults +=== RUN TestStaticAuthorizer/AllowAll +=== PAUSE TestStaticAuthorizer/AllowAll +=== RUN TestStaticAuthorizer/DenyAll +=== PAUSE TestStaticAuthorizer/DenyAll +=== RUN TestStaticAuthorizer/ManageAll +=== PAUSE TestStaticAuthorizer/ManageAll +=== CONT TestStaticAuthorizer/AllowAll === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadDenied.Prefix(football) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadDenied.Prefix(football) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWriteDenied.Prefix(football) @@ -5088,14 +5058,103 @@ === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(fo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(for) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(for) +=== CONT TestStaticAuthorizer/ManageAll +=== RUN TestACL_Enforce/acl/write/Allow +=== CONT TestStaticAuthorizer/DenyAll +=== CONT TestChainedAuthorizer/Authorizer_Defaults +=== CONT TestACL_Enforce/acl/write/Allow + authorizer_test.go:618: PASS: ACLWrite(*acl.AuthorizerContext) +=== RUN TestACL_Enforce/acl/list/Deny +--- PASS: TestStaticAuthorizer (0.04s) + --- PASS: TestStaticAuthorizer/AllowAll (0.00s) + --- PASS: TestStaticAuthorizer/ManageAll (0.00s) + --- PASS: TestStaticAuthorizer/DenyAll (0.00s) +=== CONT TestChainedAuthorizer/First_Found +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AllDefault.Prefix(*) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(for) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(for) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadAllowed.Prefix(foo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadAllowed.Prefix(foo) +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AllAllowed.Prefix(*) +=== RUN TestAllAllowed/prefix-list-allowed +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AllDenied.Prefix(*) +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AllDenied.Prefix(*) +=== RUN TestAllAllowed/prefix-deny +=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AllDenied.Prefix(*) +=== RUN TestAllAllowed/prefix-allow-other-write-prefix +=== CONT TestPolicyAuthorizer/Defaults +=== RUN TestAllAllowed/prefix-allow-other-read-exact +=== RUN TestPolicyAuthorizer/Defaults/DefaultACLRead.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultACLRead.Prefix(foo) +--- PASS: TestChainedAuthorizer (0.04s) + --- PASS: TestChainedAuthorizer/No_Authorizers (0.00s) + --- PASS: TestChainedAuthorizer/Authorizer_No_Defaults (0.00s) + --- PASS: TestChainedAuthorizer/Authorizer_Defaults (0.00s) + --- PASS: TestChainedAuthorizer/First_Found (0.00s) +=== RUN TestPolicyAuthorizer/Defaults/DefaultACLWrite.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultACLWrite.Prefix(foo) +=== RUN TestAllAllowed/prefix-allow-other-list-prefix +=== RUN TestPolicyAuthorizer/Defaults/DefaultAgentRead.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultAgentRead.Prefix(foo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultAgentWrite.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultAgentWrite.Prefix(foo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultEventRead.Prefix(foo) +=== RUN TestAllAllowed/prefix-allow-other-list-exact +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultEventRead.Prefix(foo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultEventWrite.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultEventWrite.Prefix(foo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultIntentionDefaultAllow.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultIntentionDefaultAllow.Prefix(foo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultIntentionRead.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultIntentionRead.Prefix(foo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultIntentionWrite.Prefix(foo) +=== RUN TestAllAllowed/prefix-allow-other-read-prefix +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultIntentionWrite.Prefix(foo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultKeyRead.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultKeyRead.Prefix(foo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultKeyList.Prefix(foo) +=== RUN TestAllAllowed/prefix-allow-other-deny-exact +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultKeyList.Prefix(foo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultKeyringRead.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultKeyringRead.Prefix(foo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultKeyringWrite.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultKeyringWrite.Prefix(foo) +=== RUN TestAllAllowed/no-rules-default +=== RUN TestPolicyAuthorizer/Defaults/DefaultKeyWrite.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultKeyWrite.Prefix(foo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultKeyWritePrefix.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultKeyWritePrefix.Prefix(foo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultNodeRead.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultNodeRead.Prefix(foo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteAllowed.Prefix(foo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteAllowed.Prefix(foo) +=== RUN TestACL_Enforce/operator/read/Deny +=== RUN TestAllAllowed/prefix-write-allowed +=== RUN TestAllAllowed/prefix-read-allowed +=== RUN TestAllAllowed/prefix-allow-other-write-exact +--- PASS: TestAllAllowed (0.04s) + --- PASS: TestAllAllowed/prefix-allow-other-deny-prefix (0.00s) + --- PASS: TestAllAllowed/prefix-list-allowed (0.00s) + --- PASS: TestAllAllowed/prefix-deny (0.00s) + --- PASS: TestAllAllowed/prefix-allow-other-write-prefix (0.00s) + --- PASS: TestAllAllowed/prefix-allow-other-read-exact (0.00s) + --- PASS: TestAllAllowed/prefix-allow-other-list-prefix (0.00s) + --- PASS: TestAllAllowed/prefix-allow-other-list-exact (0.00s) + --- PASS: TestAllAllowed/prefix-allow-other-read-prefix (0.00s) + --- PASS: TestAllAllowed/prefix-allow-other-deny-exact (0.00s) + --- PASS: TestAllAllowed/no-rules-default (0.00s) + --- PASS: TestAllAllowed/prefix-write-allowed (0.00s) + --- PASS: TestAllAllowed/prefix-read-allowed (0.00s) + --- PASS: TestAllAllowed/prefix-allow-other-write-exact (0.00s) +=== RUN TestPolicyAuthorizer/Defaults/DefaultNodeWrite.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultNodeWrite.Prefix(foo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultOperatorRead.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultOperatorRead.Prefix(foo) +=== CONT TestACL_Enforce/operator/read/Deny + authorizer_test.go:618: PASS: OperatorRead(*acl.AuthorizerContext) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot) +=== RUN TestACL_Enforce/operator/read/Allow === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot2) @@ -5109,10 +5168,22 @@ === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadDenied.Prefix(football) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadDenied.Prefix(football) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteDenied.Prefix(football) +=== CONT TestACL_Enforce/operator/read/Allow + authorizer_test.go:618: PASS: OperatorRead(*acl.AuthorizerContext) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteDenied.Prefix(football) +=== RUN TestPolicyAuthorizer/Defaults/DefaultOperatorWrite.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultOperatorWrite.Prefix(foo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(fo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultPreparedQueryRead.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultPreparedQueryRead.Prefix(foo) +=== RUN TestACL_Enforce/operator/write/Deny +=== RUN TestPolicyAuthorizer/Defaults/DefaultPreparedQueryWrite.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultPreparedQueryWrite.Prefix(foo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(fo) +=== CONT TestACL_Enforce/operator/write/Deny + authorizer_test.go:618: PASS: OperatorWrite(*acl.AuthorizerContext) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(fo) +=== RUN TestACL_Enforce/operator/write/Allow === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(fo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(for) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(for) @@ -5123,34 +5194,17 @@ === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteAllowed.Prefix(foo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteAllowed.Prefix(foo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot) +=== CONT TestACL_Enforce/operator/write/Allow + authorizer_test.go:618: PASS: OperatorWrite(*acl.AuthorizerContext) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot) -=== CONT TestStaticAuthorizer/ManageAll -=== CONT TestChainedAuthorizer/Authorizer_No_Defaults -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AllDenied.Prefix(*) -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AllDenied.Prefix(*) -=== CONT TestChainedAuthorizer/Authorizer_Defaults -=== CONT TestChainedAuthorizer/First_Found -=== CONT TestACL_Enforce/node/foo/write/Allow - authorizer_test.go:618: PASS: NodeWrite(string,*acl.AuthorizerContext) -=== CONT TestPolicyAuthorizer/Defaults -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AllAllowed.Prefix(*) -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AllDefault.Prefix(*) ---- PASS: TestStaticAuthorizer (0.01s) - --- PASS: TestStaticAuthorizer/AllowAll (0.00s) - --- PASS: TestStaticAuthorizer/DenyAll (0.00s) - --- PASS: TestStaticAuthorizer/ManageAll (0.00s) -=== CONT TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AllDenied.Prefix(*) ---- PASS: TestChainedAuthorizer (0.00s) - --- PASS: TestChainedAuthorizer/No_Authorizers (0.00s) - --- PASS: TestChainedAuthorizer/Authorizer_No_Defaults (0.00s) - --- PASS: TestChainedAuthorizer/Authorizer_Defaults (0.00s) - --- PASS: TestChainedAuthorizer/First_Found (0.00s) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot) +=== RUN TestACL_Enforce/operator/list/Deny === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot2) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot2) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot2) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot2) +=== RUN TestACL_Enforce/keyring/read/Deny === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(food) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(food) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(food) @@ -5165,8 +5219,26 @@ === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(fo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(for) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(for) +=== CONT TestACL_Enforce/keyring/read/Deny + authorizer_test.go:618: PASS: KeyringRead(*acl.AuthorizerContext) +=== RUN TestPolicyAuthorizer/Defaults/DefaultServiceRead.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultServiceRead.Prefix(foo) +=== RUN TestACL_Enforce/keyring/read/Allow +=== RUN TestPolicyAuthorizer/Defaults/DefaultServiceWrite.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultServiceWrite.Prefix(foo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultSessionRead.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultSessionRead.Prefix(foo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultSessionWrite.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultSessionWrite.Prefix(foo) +=== RUN TestPolicyAuthorizer/Defaults/DefaultSnapshot.Prefix(foo) +=== PAUSE TestPolicyAuthorizer/Defaults/DefaultSnapshot.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultACLRead.Prefix(foo) +=== CONT TestACL_Enforce/keyring/read/Allow + authorizer_test.go:618: PASS: KeyringRead(*acl.AuthorizerContext) +=== CONT TestPolicyAuthorizer/Defaults/DefaultSnapshot.Prefix(foo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(for) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(for) +=== RUN TestACL_Enforce/keyring/write/Deny === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadAllowed.Prefix(foo) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadAllowed.Prefix(foo) === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWriteAllowed.Prefix(foo) @@ -5188,109 +5260,90 @@ === RUN TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWriteDenied.Prefix(football) === PAUSE TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWriteDenied.Prefix(football) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(fo) -=== RUN TestAnyAllowed/no-rules-default -=== RUN TestAnyAllowed/prefix-write-allowed -=== RUN TestAnyAllowed/prefix-deny -=== RUN TestAnyAllowed/prefix-deny-other-list-exact -=== RUN TestAnyAllowed/prefix-deny-other-deny-prefix -=== RUN TestAnyAllowed/prefix-deny-other-deny-exact ---- PASS: TestAnyAllowed (0.01s) - --- PASS: TestAnyAllowed/prefix-list-allowed (0.00s) - --- PASS: TestAnyAllowed/prefix-read-allowed (0.00s) - --- PASS: TestAnyAllowed/prefix-deny-other-write-prefix (0.00s) - --- PASS: TestAnyAllowed/prefix-deny-other-write-exact (0.00s) - --- PASS: TestAnyAllowed/prefix-deny-other-list-prefix (0.00s) - --- PASS: TestAnyAllowed/prefix-deny-other-read-prefix (0.00s) - --- PASS: TestAnyAllowed/prefix-deny-other-read-exact (0.00s) - --- PASS: TestAnyAllowed/no-rules-default (0.00s) - --- PASS: TestAnyAllowed/prefix-write-allowed (0.00s) - --- PASS: TestAnyAllowed/prefix-deny (0.00s) - --- PASS: TestAnyAllowed/prefix-deny-other-list-exact (0.00s) - --- PASS: TestAnyAllowed/prefix-deny-other-deny-prefix (0.00s) - --- PASS: TestAnyAllowed/prefix-deny-other-deny-exact (0.00s) +=== CONT TestACL_Enforce/keyring/write/Deny + authorizer_test.go:618: PASS: KeyringWrite(*acl.AuthorizerContext) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWriteDenied.Prefix(football) +=== RUN TestACL_Enforce/keyring/write/Allow === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadDenied.Prefix(football) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(food) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadAllowed.Prefix(foo)#01 -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteAllowed.Prefix(foo)#01 -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for)#01 === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(food) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(for)#01 -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo)#01 === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(foot2) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo)#01 === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(foot2) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(food) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(food) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWriteDenied.Prefix(football) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot2) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadDenied.Prefix(football) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot2) +=== CONT TestACL_Enforce/keyring/write/Allow + authorizer_test.go:618: PASS: KeyringWrite(*acl.AuthorizerContext) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(foot) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot) +=== RUN TestACL_Enforce/keyring/list/Deny === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(foot) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(food) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(food) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWriteAllowed.Prefix(foo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot2) +=== RUN TestACL_Enforce/agent/foo/read/Deny === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadAllowed.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(for) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot2) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(for) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(fo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(fo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteAllowed.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteDenied.Prefix(football) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadAllowed.Prefix(foo) +=== CONT TestACL_Enforce/agent/foo/read/Deny + authorizer_test.go:618: PASS: AgentRead(string,*acl.AuthorizerContext) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadDenied.Prefix(football) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(for) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWriteAllowed.Prefix(foo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(for) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadAllowed.Prefix(foo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(fo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultSessionRead.Prefix(foo) +=== RUN TestACL_Enforce/agent/foo/read/Allow +=== CONT TestPolicyAuthorizer/Defaults/DefaultServiceWrite.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultServiceRead.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultPreparedQueryWrite.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultPreparedQueryRead.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultOperatorWrite.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultOperatorRead.Prefix(foo) +=== CONT TestACL_Enforce/agent/foo/read/Allow + authorizer_test.go:618: PASS: AgentRead(string,*acl.AuthorizerContext) +=== CONT TestPolicyAuthorizer/Defaults/DefaultNodeRead.Prefix(foo) +=== RUN TestACL_Enforce/agent/foo/write/Deny +=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyringRead.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyList.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyRead.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultIntentionWrite.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultIntentionRead.Prefix(foo) +=== CONT TestPolicyAuthorizer/Defaults/DefaultIntentionDefaultAllow.Prefix(foo) +=== CONT TestACL_Enforce/agent/foo/write/Deny + authorizer_test.go:618: PASS: AgentWrite(string,*acl.AuthorizerContext) +=== CONT TestPolicyAuthorizer/Defaults/DefaultAgentRead.Prefix(foo) +=== RUN TestACL_Enforce/agent/foo/write/Allow +=== CONT TestPolicyAuthorizer/Defaults/DefaultACLWrite.Prefix(foo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(food) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(food) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(food) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot2) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot2) +=== CONT TestACL_Enforce/agent/foo/write/Allow + authorizer_test.go:618: PASS: AgentWrite(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/agent/foo/list/Deny +=== RUN TestACL_Enforce/event/foo/read/Deny +=== CONT TestPolicyAuthorizer/Defaults/DefaultNodeWrite.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(fo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(for) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteDenied.Prefix(football) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(for) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadDenied.Prefix(football) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(fo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(fo) +=== CONT TestACL_Enforce/event/foo/read/Deny + authorizer_test.go:618: PASS: EventRead(string,*acl.AuthorizerContext) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(food) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteDenied.Prefix(football) +=== RUN TestACL_Enforce/event/foo/read/Allow === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(food) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadDenied.Prefix(football) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(food) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot2) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(food) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot2) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot2) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot2) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteAllowed.Prefix(foo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot) +=== CONT TestACL_Enforce/event/foo/read/Allow + authorizer_test.go:618: PASS: EventRead(string,*acl.AuthorizerContext) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadAllowed.Prefix(foo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteAllowed.Prefix(foo) +=== RUN TestACL_Enforce/event/foo/write/Deny === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(for) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(for) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadAllowed.Prefix(foo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(fo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(for) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(fo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWriteDenied.Prefix(football) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadDenied.Prefix(football) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteDenied.Prefix(football) +=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyWritePrefix.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(food) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadDenied.Prefix(football) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(food) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(food) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(foot2) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(foot2) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(foot) @@ -5305,12 +5358,54 @@ === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadDenied.Prefix(football)#01 === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(food)#01 === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(food)#01 -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot2)#01 +=== CONT TestPolicyAuthorizer/Defaults/DefaultEventWrite.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot2)#01 === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot)#01 === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot)#01 -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadDenied.Prefix(football) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteAllowed.Prefix(foo)#01 +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadAllowed.Prefix(foo)#01 +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for)#01 +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(for)#01 +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo)#01 +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo)#01 +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWriteDenied.Prefix(football) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadDenied.Prefix(football) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(food) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(food) +=== CONT TestPolicyAuthorizer/Defaults/DefaultSessionWrite.Prefix(foo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWriteAllowed.Prefix(foo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadAllowed.Prefix(foo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(for) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(for) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(fo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(fo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteDenied.Prefix(football) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadDenied.Prefix(football) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(food) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(food) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot2) +=== CONT TestACL_Enforce/event/foo/write/Deny + authorizer_test.go:618: PASS: EventWrite(string,*acl.AuthorizerContext) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot) +=== RUN TestACL_Enforce/event/foo/write/Allow +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteAllowed.Prefix(foo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadAllowed.Prefix(foo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(for) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo) +=== CONT TestACL_Enforce/event/foo/write/Allow + authorizer_test.go:618: PASS: EventWrite(string,*acl.AuthorizerContext) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteDenied.Prefix(football) +=== RUN TestACL_Enforce/event/foo/list/Deny +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadDenied.Prefix(football) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(food) +=== RUN TestACL_Enforce/intention/foo/read/Deny +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(food) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(foot2) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(foot2) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(foot) @@ -5318,74 +5413,49 @@ === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteAllowed.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadAllowed.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(for) +=== CONT TestACL_Enforce/intention/foo/read/Deny + authorizer_test.go:618: PASS: IntentionRead(string,*acl.AuthorizerContext) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(for) +=== RUN TestACL_Enforce/intention/foo/read/Allow === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(fo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(fo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteDenied.Prefix(football) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadDenied.Prefix(football) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(food) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(food) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(for) +=== CONT TestACL_Enforce/intention/foo/read/Allow + authorizer_test.go:618: PASS: IntentionRead(string,*acl.AuthorizerContext) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(foot2) +=== RUN TestACL_Enforce/intention/foo/write/Deny === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot2) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(foot) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(for) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteAllowed.Prefix(foo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadAllowed.Prefix(foo) -=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(fo) === CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(for) -=== RUN TestPolicyAuthorizer/Defaults/DefaultACLRead.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultACLRead.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultACLWrite.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultACLWrite.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultAgentRead.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultAgentRead.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultAgentWrite.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultAgentWrite.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultEventRead.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultEventRead.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultEventWrite.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultEventWrite.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultIntentionDefaultAllow.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultIntentionDefaultAllow.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultIntentionRead.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultIntentionRead.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultIntentionWrite.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultIntentionWrite.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultKeyRead.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultKeyRead.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultKeyList.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultKeyList.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultKeyringRead.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultKeyringRead.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultKeyringWrite.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultKeyringWrite.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultKeyWrite.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultKeyWrite.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultKeyWritePrefix.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultKeyWritePrefix.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultNodeRead.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultNodeRead.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultNodeWrite.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultNodeWrite.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultOperatorRead.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultOperatorRead.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultOperatorWrite.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultOperatorWrite.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultPreparedQueryRead.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultPreparedQueryRead.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultPreparedQueryWrite.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultPreparedQueryWrite.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultServiceRead.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultServiceRead.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultServiceWrite.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultServiceWrite.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultSessionRead.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultSessionRead.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultSessionWrite.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultSessionWrite.Prefix(foo) -=== RUN TestPolicyAuthorizer/Defaults/DefaultSnapshot.Prefix(foo) -=== PAUSE TestPolicyAuthorizer/Defaults/DefaultSnapshot.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultACLRead.Prefix(foo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(fo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot2) +=== CONT TestACL_Enforce/intention/foo/write/Deny + authorizer_test.go:618: PASS: IntentionWrite(string,*acl.AuthorizerContext) +=== CONT TestPolicyAuthorizer/Defaults/DefaultAgentWrite.Prefix(foo) +=== RUN TestACL_Enforce/intention/foo/write/Allow +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot2) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadAllowed.Prefix(foo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(for) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(fo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot2) +=== CONT TestACL_Enforce/intention/foo/write/Allow + authorizer_test.go:618: PASS: IntentionWrite(string,*acl.AuthorizerContext) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(for) +=== RUN TestACL_Enforce/intention/foo/list/Deny +=== RUN TestACL_Enforce/node/foo/read/Deny + authorizer_test.go:618: PASS: NodeRead(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/node/foo/read/Allow + authorizer_test.go:618: PASS: NodeRead(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/node/foo/write/Deny + authorizer_test.go:618: PASS: NodeWrite(string,*acl.AuthorizerContext) +=== RUN TestACL_Enforce/node/foo/write/Allow + authorizer_test.go:618: PASS: NodeWrite(string,*acl.AuthorizerContext) === RUN TestACL_Enforce/node/foo/list/Deny === RUN TestACL_Enforce/query/foo/read/Deny authorizer_test.go:618: PASS: PreparedQueryRead(string,*acl.AuthorizerContext) @@ -5395,7 +5465,9 @@ authorizer_test.go:618: PASS: PreparedQueryWrite(string,*acl.AuthorizerContext) === RUN TestACL_Enforce/query/foo/write/Allow authorizer_test.go:618: PASS: PreparedQueryWrite(string,*acl.AuthorizerContext) +=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyringWrite.Prefix(foo) === RUN TestACL_Enforce/query/foo/list/Deny +=== CONT TestPolicyAuthorizer/Defaults/DefaultEventRead.Prefix(foo) === RUN TestACL_Enforce/service/foo/read/Deny authorizer_test.go:618: PASS: ServiceRead(string,*acl.AuthorizerContext) === RUN TestACL_Enforce/service/foo/read/Allow @@ -5403,46 +5475,25 @@ === RUN TestACL_Enforce/service/foo/write/Deny authorizer_test.go:618: PASS: ServiceWrite(string,*acl.AuthorizerContext) === RUN TestACL_Enforce/service/foo/write/Allow -=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyringWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyringRead.Prefix(foo) -=== CONT TestACL_Enforce/service/foo/write/Allow authorizer_test.go:618: PASS: ServiceWrite(string,*acl.AuthorizerContext) -=== CONT TestPolicyAuthorizer/Defaults/DefaultIntentionRead.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultIntentionDefaultAllow.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultEventWrite.Prefix(foo) === RUN TestACL_Enforce/session/foo/list/Deny -=== CONT TestPolicyAuthorizer/Defaults/DefaultEventRead.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultAgentWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultAgentRead.Prefix(foo) === RUN TestACL_Enforce/session/foo/read/Deny -=== CONT TestPolicyAuthorizer/Defaults/DefaultACLWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultPreparedQueryRead.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultSnapshot.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultSessionWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultSessionRead.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultServiceWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultServiceRead.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultPreparedQueryWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultNodeWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultOperatorWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultNodeRead.Prefix(foo) -=== CONT TestACL_Enforce/session/foo/read/Deny authorizer_test.go:618: PASS: SessionRead(string,*acl.AuthorizerContext) -=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyWritePrefix.Prefix(foo) === RUN TestACL_Enforce/session/foo/read/Allow -=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyList.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultIntentionWrite.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultOperatorRead.Prefix(foo) -=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyRead.Prefix(foo) +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot) +=== CONT TestPolicyAuthorizer/Defaults/DefaultKeyWrite.Prefix(foo) === CONT TestACL_Enforce/session/foo/read/Allow authorizer_test.go:618: PASS: SessionRead(string,*acl.AuthorizerContext) === RUN TestACL_Enforce/session/foo/write/Deny +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteAllowed.Prefix(foo) +=== CONT TestACL_Enforce/session/foo/write/Deny authorizer_test.go:618: PASS: SessionWrite(string,*acl.AuthorizerContext) === RUN TestACL_Enforce/session/foo/write/Allow authorizer_test.go:618: PASS: SessionWrite(string,*acl.AuthorizerContext) === RUN TestACL_Enforce/session/foo/list/Deny#01 === RUN TestACL_Enforce/key/foo/read/Deny +=== CONT TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot2)#01 +=== CONT TestACL_Enforce/key/foo/read/Deny authorizer_test.go:618: PASS: KeyRead(string,*acl.AuthorizerContext) === RUN TestACL_Enforce/key/foo/read/Allow authorizer_test.go:618: PASS: KeyRead(string,*acl.AuthorizerContext) @@ -5456,7 +5507,7 @@ authorizer_test.go:618: PASS: KeyList(string,*acl.AuthorizerContext) === RUN TestACL_Enforce/key/foo/deny/Deny === RUN TestACL_Enforce/not-a-real-resource/read/Deny ---- PASS: TestACL_Enforce (0.02s) +--- PASS: TestACL_Enforce (0.05s) --- PASS: TestACL_Enforce/acl/read/Deny (0.00s) --- PASS: TestACL_Enforce/acl/read/Allow (0.00s) --- PASS: TestACL_Enforce/acl/write/Deny (0.00s) @@ -5515,138 +5566,96 @@ --- PASS: TestACL_Enforce/key/foo/list/Allow (0.00s) --- PASS: TestACL_Enforce/key/foo/deny/Deny (0.00s) --- PASS: TestACL_Enforce/not-a-real-resource/read/Deny (0.00s) ---- PASS: TestPolicyAuthorizer (0.00s) +--- PASS: TestPolicyAuthorizer (0.04s) + --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_default (0.00s) + --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AnyAllowed.Prefix(*) (0.00s) + --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AllDefault.Prefix(*) (0.00s) + --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed (0.00s) + --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AnyAllowed.Prefix(*) (0.00s) + --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AllAllowed.Prefix(*) (0.00s) --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed (0.00s) --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AnyAllowed.Prefix(*) (0.00s) --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_allowed/AllDenied.Prefix(*) (0.00s) - --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_any_default (0.00s) - --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AnyDefault.Prefix(*) (0.00s) - --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AllDenied.Prefix(*) (0.00s) --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied (0.00s) --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AnyAllowed.Prefix(*) (0.00s) --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_prefix_denied/AllDenied.Prefix(*) (0.00s) - --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed (0.00s) - --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AnyAllowed.Prefix(*) (0.00s) - --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_allowed/AllAllowed.Prefix(*) (0.00s) - --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_default (0.00s) - --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AnyAllowed.Prefix(*) (0.00s) - --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_all_default/AllDefault.Prefix(*) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults (0.01s) + --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_any_default (0.00s) + --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AnyDefault.Prefix(*) (0.00s) + --- PASS: TestPolicyAuthorizer/Intention_Wildcards_-_any_default/AllDenied.Prefix(*) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultACLRead.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyringWrite.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyWrite.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyringRead.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultIntentionRead.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultIntentionDefaultAllow.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultEventWrite.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultEventRead.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultAgentWrite.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultAgentRead.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultACLWrite.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultPreparedQueryRead.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultSnapshot.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultSessionWrite.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultSessionRead.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultServiceWrite.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultServiceRead.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultPreparedQueryWrite.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultNodeWrite.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultPreparedQueryRead.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultOperatorWrite.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultOperatorRead.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultNodeRead.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyWritePrefix.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyringRead.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyList.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultIntentionWrite.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Defaults/DefaultOperatorRead.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyRead.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches (0.01s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultIntentionWrite.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultIntentionRead.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultIntentionDefaultAllow.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultAgentRead.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultACLWrite.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultNodeWrite.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyWritePrefix.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultEventWrite.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultSessionWrite.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultAgentWrite.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyringWrite.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultEventRead.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Defaults/DefaultKeyWrite.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(fo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWriteDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(food) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadAllowed.Prefix(foo)#01 (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteAllowed.Prefix(foo)#01 (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for)#01 (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(for)#01 (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(food) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo)#01 (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(foot2) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo)#01 (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(food) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(foot2) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(food) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWriteDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot2) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot2) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(foot) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(foot) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(food) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(food) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWriteAllowed.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadAllowed.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot2) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(for) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot2) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(for) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryWritePrefixDenied.Prefix(fo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/PreparedQueryReadPrefixAllowed.Prefix(fo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteAllowed.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadAllowed.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(for) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWriteAllowed.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(for) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(fo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadAllowed.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot2) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot2) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(fo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(for) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(for) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(fo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(fo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(food) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(food) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(food) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot2) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(food) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot2) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot2) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(foot) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot2) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(foot) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWriteAllowed.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadAllowed.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteAllowed.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(for) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadAllowed.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(for) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionWritePrefixDenied.Prefix(fo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(for) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/SessionReadPrefixAllowed.Prefix(fo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWriteDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(food) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(food) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(foot2) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(foot2) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(foot) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(foot) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWriteAllowed.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(foot) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadAllowed.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionWritePrefixDenied.Prefix(for) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/IntentionReadPrefixAllowed.Prefix(for) (0.00s) @@ -5656,12 +5665,45 @@ --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadDenied.Prefix(football)#01 (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(food)#01 (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(food)#01 (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot2)#01 (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot2)#01 (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot)#01 (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot)#01 (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadDenied.Prefix(football) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteAllowed.Prefix(foo)#01 (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for)#01 (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadAllowed.Prefix(foo)#01 (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(for)#01 (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo)#01 (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo)#01 (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWriteDenied.Prefix(football) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWriteAllowed.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadAllowed.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(for) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(for) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(fo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(fo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteDenied.Prefix(football) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadDenied.Prefix(football) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot2) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWriteAllowed.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadAllowed.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(for) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(for) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(fo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeReadPrefixAllowed.Prefix(fo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWriteDenied.Prefix(football) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadDenied.Prefix(football) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(food) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(foot2) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(foot2) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(foot) (0.00s) @@ -5673,20 +5715,29 @@ --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(fo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyReadPrefixAllowed.Prefix(fo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteDenied.Prefix(football) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadDenied.Prefix(football) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(food) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(food) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(for) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(foot2) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot2) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(foot) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(for) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(foot) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWriteAllowed.Prefix(foo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadAllowed.Prefix(foo) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(fo) (0.00s) --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentReadPrefixAllowed.Prefix(for) (0.00s) - --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/KeyWritePrefixDenied.Prefix(food) (0.01s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/AgentWritePrefixDenied.Prefix(fo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceWritePrefixDenied.Prefix(foot2) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot2) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadAllowed.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(for) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(fo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/ServiceReadPrefixAllowed.Prefix(foot2) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventReadPrefixAllowed.Prefix(foot) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWriteAllowed.Prefix(foo) (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/NodeWritePrefixDenied.Prefix(foot2)#01 (0.00s) + --- PASS: TestPolicyAuthorizer/Prefer_Exact_Matches/EventWritePrefixDenied.Prefix(for) (0.00s) PASS -ok github.com/hashicorp/consul/acl 0.119s +ok github.com/hashicorp/consul/acl 0.149s === RUN TestACL_Legacy_Disabled_Response === PAUSE TestACL_Legacy_Disabled_Response === RUN TestACL_Legacy_Update @@ -5778,80 +5829,73 @@ [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:11:35.129Z [WARN] TestAgent_ReloadDoesNotTriggerWatch: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:34.097Z [INFO] TestAgent_ReloadDoesNotTriggerWatch: Watch not yet populated, retrying -2021-10-31T23:11:33.094Z [INFO] TestAgent_ReloadDoesNotTriggerWatch: Consul is now ready -2021-10-31T23:11:32.646Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:32.672Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:32.679 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c Address:127.0.0.1:16132}]" -11:32.680 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c.dc1 127.0.0.1 -11:32.680 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16132 [Follower]" leader= -11:32.680 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c 127.0.0.1 -2021-10-31T23:11:32.680Z [INFO] agent.router: Initializing LAN area manager -11:32.681 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=udp -11:32.681 [INFO] TestAgent.server: Adding LAN server: server="Node-e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c (Addr: tcp/127.0.0.1:16132) (DC: dc1)" -11:32.681 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c.dc1 area=wan -11:32.681 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=tcp -11:32.686 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16128 network=tcp -11:32.686 [INFO] TestAgent: started state syncer -11:32.686 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16133 network=tcp -11:32.723 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:32.723 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16132 [Candidate]" term=2 -11:32.723 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:32.723 [DEBUG] TestAgent.server.raft: vote granted: from=e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c term=2 tally=1 -11:32.723 [INFO] TestAgent.server.raft: election won: tally=1 -11:32.723 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16132 [Leader]" -11:32.723 [INFO] TestAgent.server: cluster leadership acquired -11:32.724 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16132 -11:32.724 [INFO] TestAgent.server: New leader elected: payload=Node-e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c -11:32.724 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:32.726 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:32.726 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:32.726 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:32.726 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:32.726 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:32.726 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c -11:32.727 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c -11:32.728 [INFO] TestAgent.server: federation state anti-entropy synced -11:32.934 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:32.934 [INFO] TestAgent: Synced node info -11:32.934 [INFO] TestAgent: Synced service: service=redis -11:32.934 [DEBUG] TestAgent: Check in sync: check=red-is-dead -11:32.934 [DEBUG] TestAgent: Node info in sync -11:32.934 [DEBUG] TestAgent: Service in sync: service=redis -11:32.934 [DEBUG] TestAgent: Check in sync: check=red-is-dead -11:33.043 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/redis?dc=dc1 from=127.0.0.1:41560 latency=345.176096ms -11:33.094 [DEBUG] TestAgent: Check status updated: check=red-is-dead status=passing -11:33.169 [DEBUG] TestAgent: watch handler output: watch_handler=[bash, -c, "(cat ; echo CONSUL_INDEX $CONSUL_INDEX) | tee '/tmp/rexec382967321.atomic' ; mv '/tmp/rexec382967321.atomic' '/tmp/rexec382967321'"] output="[{"Node":{"ID":"e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c","Node":"Node-e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c","Address":"127.0.0.1","Datacenter":"dc1","TaggedAddresses":{"lan":"127.0.0.1","lan_ipv4":"127.0.0.1","wan":"127.0.0.1","wan_ipv4":"127.0.0.1"},"Meta":{"consul-network-segment":""},"CreateIndex":10,"ModifyIndex":12},"Service":{"ID":"redis","Service":"redis","Tags":[],"Meta":null,"Port":0,"Address":"","Weights":{"Passing":1,"Warning":1},"EnableTagOverride":false,"CreateIndex":13,"ModifyIndex":13,"Proxy":{"MeshGateway":{},"Expose":{}},"Connect":{}},"Checks":[{"Node":"Node-e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c","CheckID":"serfHealth","Name":"Serf Health Status","Status":"passing","Notes":"","Output":"Agent alive and reachable","ServiceID":"","ServiceName":"","ServiceTags":[],"Type":"","Definition":{"Interval":"0s","Timeout":"0s","DeregisterCriticalServiceAfter":"0s","HTTP":"","Header":null,"Method":"","Body":"","TLSSkipVerify":false,"TCP":""},"CreateIndex":10,"ModifyIndex":10},{"Node":"Node-e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c","CheckID":"red-is-dead","Name":"Service 'redis' check","Status":"critical","Notes":"initial check","Output":"","ServiceID":"redis","ServiceName":"redis","ServiceTags":[],"Type":"ttl","Definition":{"Interval":"0s","Timeout":"0s","DeregisterCriticalServiceAfter":"0s","HTTP":"","Header":null,"Method":"","Body":"","TLSSkipVerify":false,"TCP":""},"CreateIndex":13,"ModifyIndex":13}]}] +2022-12-04T05:47:52.676Z [WARN] TestAgent_ReloadDoesNotTriggerWatch: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:52.648Z [INFO] TestAgent_ReloadDoesNotTriggerWatch: Watch not yet populated, retrying +2022-12-04T05:47:51.644Z [INFO] TestAgent_ReloadDoesNotTriggerWatch: Consul is now ready +2022-12-04T05:47:51.569Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:51.580Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:51.583 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:581dae22-f6b1-fd02-582d-b716d830d120 Address:127.0.0.1:23279}]" +47:51.584 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23279 [Follower]" leader= +47:51.584 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-581dae22-f6b1-fd02-582d-b716d830d120.dc1 127.0.0.1 +47:51.586 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-581dae22-f6b1-fd02-582d-b716d830d120 127.0.0.1 +2022-12-04T05:47:51.586Z [INFO] agent.router: Initializing LAN area manager +47:51.588 [INFO] TestAgent.server: Adding LAN server: server="Node-581dae22-f6b1-fd02-582d-b716d830d120 (Addr: tcp/127.0.0.1:23279) (DC: dc1)" +47:51.588 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-581dae22-f6b1-fd02-582d-b716d830d120.dc1 area=wan +47:51.589 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=tcp +47:51.589 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=udp +47:51.590 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23275 network=tcp +47:51.590 [INFO] TestAgent: started state syncer +47:51.590 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23280 network=tcp +47:51.625 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:51.625 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23279 [Candidate]" term=2 +47:51.625 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:51.625 [DEBUG] TestAgent.server.raft: vote granted: from=581dae22-f6b1-fd02-582d-b716d830d120 term=2 tally=1 +47:51.625 [INFO] TestAgent.server.raft: election won: tally=1 +47:51.625 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23279 [Leader]" +47:51.625 [INFO] TestAgent.server: cluster leadership acquired +47:51.625 [INFO] TestAgent.server: New leader elected: payload=Node-581dae22-f6b1-fd02-582d-b716d830d120 +47:51.625 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23279 +47:51.626 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:51.627 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:51.627 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:51.627 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:51.627 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:51.627 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:51.627 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-581dae22-f6b1-fd02-582d-b716d830d120 +47:51.627 [INFO] TestAgent.server: member joined, marking health alive: member=Node-581dae22-f6b1-fd02-582d-b716d830d120 +47:51.628 [INFO] TestAgent.server: federation state anti-entropy synced +47:51.644 [DEBUG] TestAgent: Check status updated: check=red-is-dead status=passing +47:51.820 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:51.821 [INFO] TestAgent: Synced node info +47:51.821 [INFO] TestAgent: Synced service: service=redis +47:51.821 [DEBUG] TestAgent: Check in sync: check=red-is-dead +47:51.821 [DEBUG] TestAgent: Node info in sync +47:51.821 [DEBUG] TestAgent: Service in sync: service=redis +47:51.821 [DEBUG] TestAgent: Check in sync: check=red-is-dead +47:51.882 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/redis?dc=dc1 from=127.0.0.1:52444 latency=291.411824ms +47:51.918 [DEBUG] TestAgent: watch handler output: watch_handler=[bash, -c, "(cat ; echo CONSUL_INDEX $CONSUL_INDEX) | tee '/tmp/rexec129472104.atomic' ; mv '/tmp/rexec129472104.atomic' '/tmp/rexec129472104'"] output="[{"Node":{"ID":"581dae22-f6b1-fd02-582d-b716d830d120","Node":"Node-581dae22-f6b1-fd02-582d-b716d830d120","Address":"127.0.0.1","Datacenter":"dc1","TaggedAddresses":{"lan":"127.0.0.1","lan_ipv4":"127.0.0.1","wan":"127.0.0.1","wan_ipv4":"127.0.0.1"},"Meta":{"consul-network-segment":""},"CreateIndex":10,"ModifyIndex":12},"Service":{"ID":"redis","Service":"redis","Tags":[],"Meta":null,"Port":0,"Address":"","Weights":{"Passing":1,"Warning":1},"EnableTagOverride":false,"CreateIndex":13,"ModifyIndex":13,"Proxy":{"MeshGateway":{},"Expose":{}},"Connect":{}},"Checks":[{"Node":"Node-581dae22-f6b1-fd02-582d-b716d830d120","CheckID":"serfHealth","Name":"Serf Health Status","Status":"passing","Notes":"","Output":"Agent alive and reachable","ServiceID":"","ServiceName":"","ServiceTags":[],"Type":"","Definition":{"Interval":"0s","Timeout":"0s","DeregisterCriticalServiceAfter":"0s","HTTP":"","Header":null,"Method":"","Body":"","TLSSkipVerify":false,"TCP":""},"CreateIndex":10,"ModifyIndex":10},{"Node":"Node-581dae22-f6b1-fd02-582d-b716d830d120","CheckID":"red-is-dead","Name":"Service 'redis' check","Status":"passing","Notes":"initial check","Output":"testing-agent-reload-001","ServiceID":"redis","ServiceName":"redis","ServiceTags":[],"Type":"ttl","Definition":{"Interval":"0s","Timeout":"0s","DeregisterCriticalServiceAfter":"0s","HTTP":"","Header":null,"Method":"","Body":"","TLSSkipVerify":false,"TCP":""},"CreateIndex":13,"ModifyIndex":13}]}] CONSUL_INDEX 13 " -11:34.319 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:34.319 [DEBUG] TestAgent: Node info in sync -11:34.319 [DEBUG] TestAgent: Service in sync: service=redis -11:34.320 [INFO] TestAgent: Synced check: check=red-is-dead -11:34.320 [DEBUG] TestAgent: Node info in sync -11:34.320 [DEBUG] TestAgent: Service in sync: service=redis -11:34.320 [DEBUG] TestAgent: Check in sync: check=red-is-dead -11:34.321 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/redis?dc=dc1&index=13 from=127.0.0.1:41560 latency=1.147275811s -11:34.351 [DEBUG] TestAgent: watch handler output: watch_handler=[bash, -c, "(cat ; echo CONSUL_INDEX $CONSUL_INDEX) | tee '/tmp/rexec382967321.atomic' ; mv '/tmp/rexec382967321.atomic' '/tmp/rexec382967321'"] output="[{"Node":{"ID":"e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c","Node":"Node-e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c","Address":"127.0.0.1","Datacenter":"dc1","TaggedAddresses":{"lan":"127.0.0.1","lan_ipv4":"127.0.0.1","wan":"127.0.0.1","wan_ipv4":"127.0.0.1"},"Meta":{"consul-network-segment":""},"CreateIndex":10,"ModifyIndex":12},"Service":{"ID":"redis","Service":"redis","Tags":[],"Meta":null,"Port":0,"Address":"","Weights":{"Passing":1,"Warning":1},"EnableTagOverride":false,"CreateIndex":13,"ModifyIndex":13,"Proxy":{"MeshGateway":{},"Expose":{}},"Connect":{}},"Checks":[{"Node":"Node-e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c","CheckID":"serfHealth","Name":"Serf Health Status","Status":"passing","Notes":"","Output":"Agent alive and reachable","ServiceID":"","ServiceName":"","ServiceTags":[],"Type":"","Definition":{"Interval":"0s","Timeout":"0s","DeregisterCriticalServiceAfter":"0s","HTTP":"","Header":null,"Method":"","Body":"","TLSSkipVerify":false,"TCP":""},"CreateIndex":10,"ModifyIndex":10},{"Node":"Node-e4ba1da9-23f4-9765-deaf-c6cc1e5fd24c","CheckID":"red-is-dead","Name":"Service 'redis' check","Status":"passing","Notes":"initial check","Output":"testing-agent-reload-001","ServiceID":"redis","ServiceName":"redis","ServiceTags":[],"Type":"ttl","Definition":{"Interval":"0s","Timeout":"0s","DeregisterCriticalServiceAfter":"0s","HTTP":"","Header":null,"Method":"","Body":"","TLSSkipVerify":false,"TCP":""},"CreateIndex":13,"ModifyIndex":14}]}] -CONSUL_INDEX 14 -" -11:35.130 [INFO] TestAgent: Requesting shutdown -11:35.130 [INFO] TestAgent.server: shutting down server -11:35.130 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:35.131 [INFO] TestAgent: Synced service: service=redis -11:35.131 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -11:35.131 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" -2021-10-31T23:11:35.132Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:35.132Z [INFO] agent.router.manager: shutting down -11:35.132 [INFO] TestAgent: consul server down -11:35.132 [INFO] TestAgent: shutdown complete -11:35.132 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=tcp -11:35.132 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=udp -11:35.132 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16128 network=tcp -11:35.633 [INFO] TestAgent: Waiting for endpoints to shut down -11:35.633 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ReloadDoesNotTriggerWatch (3.11s) +47:52.672 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:52.672 [DEBUG] TestAgent: Node info in sync +47:52.672 [DEBUG] TestAgent: Service in sync: service=redis +47:52.672 [DEBUG] TestAgent: Check in sync: check=red-is-dead +47:52.677 [INFO] TestAgent: Requesting shutdown +47:52.677 [INFO] TestAgent.server: shutting down server +47:52.677 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:52.678 [INFO] TestAgent: Synced service: service=redis +47:52.679 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:52.679 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:52.680Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:52.680Z [INFO] agent.router.manager: shutting down +47:52.680 [INFO] TestAgent: consul server down +47:52.680 [INFO] TestAgent: shutdown complete +47:52.680 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=tcp +47:52.680 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=udp +47:52.680 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23275 network=tcp +47:53.185 [INFO] TestAgent: Waiting for endpoints to shut down +47:53.185 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ReloadDoesNotTriggerWatch (1.65s) === RUN TestAgent_Reload_ACLDeny === PAUSE TestAgent_Reload_ACLDeny === RUN TestAgent_Members @@ -5922,129 +5966,129 @@ === PAUSE TestAgent_RegisterService/service_manager === CONT TestAgent_RegisterService/normal === CONT TestAgent_RegisterService/service_manager -2021-10-31T23:11:35.679Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:35.716Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:35.722 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:da6eda55-6f5b-afd3-b876-d99e08abf244 Address:127.0.0.1:16139}]" -11:35.723 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-da6eda55-6f5b-afd3-b876-d99e08abf244.dc1 127.0.0.1 -11:35.723 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-da6eda55-6f5b-afd3-b876-d99e08abf244 127.0.0.1 -2021-10-31T23:11:35.723Z [INFO] agent.router: Initializing LAN area manager -11:35.724 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=udp -11:35.724 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16139 [Follower]" leader= -11:35.724 [INFO] TestAgent.server: Adding LAN server: server="Node-da6eda55-6f5b-afd3-b876-d99e08abf244 (Addr: tcp/127.0.0.1:16139) (DC: dc1)" -11:35.724 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-da6eda55-6f5b-afd3-b876-d99e08abf244.dc1 area=wan -11:35.724 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=tcp -11:35.724 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16135 network=tcp -11:35.725 [INFO] TestAgent: started state syncer -11:35.725 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16140 network=tcp -11:35.794 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:35.794 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16139 [Candidate]" term=2 -11:35.794 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:35.794 [DEBUG] TestAgent.server.raft: vote granted: from=da6eda55-6f5b-afd3-b876-d99e08abf244 term=2 tally=1 -11:35.794 [INFO] TestAgent.server.raft: election won: tally=1 -11:35.794 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16139 [Leader]" -11:35.794 [INFO] TestAgent.server: cluster leadership acquired -11:35.794 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16139 -11:35.794 [INFO] TestAgent.server: New leader elected: payload=Node-da6eda55-6f5b-afd3-b876-d99e08abf244 -11:35.795 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:35.796 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:35.796 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:35.796 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:35.796 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:35.796 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:35.796 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-da6eda55-6f5b-afd3-b876-d99e08abf244 -11:35.796 [INFO] TestAgent.server: member joined, marking health alive: member=Node-da6eda55-6f5b-afd3-b876-d99e08abf244 -11:35.797 [INFO] TestAgent.server: federation state anti-entropy synced -11:35.815 [INFO] TestAgent: Synced node info -11:35.815 [INFO] TestAgent: Synced service: service=test -11:35.815 [DEBUG] TestAgent: Check in sync: check=service:test:1 -11:35.815 [DEBUG] TestAgent: Check in sync: check=service:test:2 -11:35.815 [DEBUG] TestAgent: Check in sync: check=service:test:3 -11:35.815 [INFO] TestAgent: Requesting shutdown -11:35.815 [INFO] TestAgent.server: shutting down server -11:35.815 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:35.815 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:35.815 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:35.815 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:35.815 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:35.816 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:35.816 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:35.816 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:35.816 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:35.816 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:35.816 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:35.816 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:35.816Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:35.816Z [INFO] agent.router.manager: shutting down -11:35.816 [INFO] TestAgent: consul server down -11:35.816 [INFO] TestAgent: shutdown complete -11:35.816 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=tcp -11:35.816 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=udp -11:35.816 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16135 network=tcp -11:36.316 [INFO] TestAgent: Waiting for endpoints to shut down -11:36.316 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:35.656Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:35.691Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:35.693 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef3c57f7-a44b-a351-4499-1e0e1dba5491 Address:127.0.0.1:16146}]" -11:35.693 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16146 [Follower]" leader= -11:35.703 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef3c57f7-a44b-a351-4499-1e0e1dba5491.dc1 127.0.0.1 -11:35.704 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef3c57f7-a44b-a351-4499-1e0e1dba5491 127.0.0.1 -2021-10-31T23:11:35.704Z [INFO] agent.router: Initializing LAN area manager -11:35.704 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16141 network=udp -11:35.704 [INFO] TestAgent.server: Adding LAN server: server="Node-ef3c57f7-a44b-a351-4499-1e0e1dba5491 (Addr: tcp/127.0.0.1:16146) (DC: dc1)" -11:35.704 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef3c57f7-a44b-a351-4499-1e0e1dba5491.dc1 area=wan -11:35.705 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16141 network=tcp -11:35.705 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16142 network=tcp -11:35.705 [INFO] TestAgent: started state syncer -11:35.706 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16147 network=tcp -11:35.745 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:35.745 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16146 [Candidate]" term=2 -11:35.746 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:35.746 [DEBUG] TestAgent.server.raft: vote granted: from=ef3c57f7-a44b-a351-4499-1e0e1dba5491 term=2 tally=1 -11:35.746 [INFO] TestAgent.server.raft: election won: tally=1 -11:35.746 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16146 [Leader]" -11:35.746 [INFO] TestAgent.server: cluster leadership acquired -11:35.746 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16146 -11:35.747 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:35.748 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:35.748 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:35.748 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:35.748 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:35.748 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:35.748 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef3c57f7-a44b-a351-4499-1e0e1dba5491 -11:35.748 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef3c57f7-a44b-a351-4499-1e0e1dba5491 -11:35.749 [INFO] TestAgent.server: New leader elected: payload=Node-ef3c57f7-a44b-a351-4499-1e0e1dba5491 -11:35.749 [INFO] TestAgent.server: federation state anti-entropy synced -11:35.840 [INFO] TestAgent: Synced node info -11:35.841 [INFO] TestAgent: Synced service: service=test -11:35.841 [DEBUG] TestAgent: Check in sync: check=service:test:1 -11:35.841 [DEBUG] TestAgent: Check in sync: check=service:test:2 -11:35.841 [DEBUG] TestAgent: Check in sync: check=service:test:3 -11:35.841 [INFO] TestAgent: Requesting shutdown -11:35.841 [INFO] TestAgent.server: shutting down server -11:35.841 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:35.841 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:35.841 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:35.841 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:35.841 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:35.841 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:35.841 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:35.841 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:35.841 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:35.841 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:35.841 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:35.841 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:35.842Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:35.842Z [INFO] agent.router.manager: shutting down -11:35.842 [INFO] TestAgent: consul server down -11:35.842 [INFO] TestAgent: shutdown complete -11:35.842 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16141 network=tcp -11:35.842 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16141 network=udp -11:35.842 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16142 network=tcp -11:36.344 [INFO] TestAgent: Waiting for endpoints to shut down -11:36.344 [INFO] TestAgent: Endpoints down +2022-12-04T05:47:53.215Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:53.266Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:53.269 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:662db82d-9213-2733-3649-5b8e6e335e0b Address:127.0.0.1:23293}]" +47:53.270 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-662db82d-9213-2733-3649-5b8e6e335e0b.dc1 127.0.0.1 +47:53.271 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-662db82d-9213-2733-3649-5b8e6e335e0b 127.0.0.1 +2022-12-04T05:47:53.271Z [INFO] agent.router: Initializing LAN area manager +47:53.271 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23288 network=udp +47:53.276 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23293 [Follower]" leader= +47:53.277 [INFO] TestAgent.server: Adding LAN server: server="Node-662db82d-9213-2733-3649-5b8e6e335e0b (Addr: tcp/127.0.0.1:23293) (DC: dc1)" +47:53.277 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-662db82d-9213-2733-3649-5b8e6e335e0b.dc1 area=wan +47:53.277 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23288 network=tcp +47:53.278 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23289 network=tcp +47:53.278 [INFO] TestAgent: started state syncer +47:53.280 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23294 network=tcp +47:53.336 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:53.336 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23293 [Candidate]" term=2 +47:53.336 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:53.336 [DEBUG] TestAgent.server.raft: vote granted: from=662db82d-9213-2733-3649-5b8e6e335e0b term=2 tally=1 +47:53.336 [INFO] TestAgent.server.raft: election won: tally=1 +47:53.336 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23293 [Leader]" +47:53.336 [INFO] TestAgent.server: cluster leadership acquired +47:53.336 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23293 +47:53.337 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:53.338 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:53.338 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:53.338 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:53.338 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:53.338 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:53.339 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-662db82d-9213-2733-3649-5b8e6e335e0b +47:53.339 [INFO] TestAgent.server: member joined, marking health alive: member=Node-662db82d-9213-2733-3649-5b8e6e335e0b +47:53.339 [INFO] TestAgent.server: New leader elected: payload=Node-662db82d-9213-2733-3649-5b8e6e335e0b +47:53.339 [INFO] TestAgent.server: federation state anti-entropy synced +47:53.434 [INFO] TestAgent: Synced node info +47:53.435 [INFO] TestAgent: Synced service: service=test +47:53.435 [DEBUG] TestAgent: Check in sync: check=service:test:1 +47:53.435 [DEBUG] TestAgent: Check in sync: check=service:test:2 +47:53.435 [DEBUG] TestAgent: Check in sync: check=service:test:3 +47:53.435 [INFO] TestAgent: Requesting shutdown +47:53.435 [INFO] TestAgent.server: shutting down server +47:53.435 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:53.435 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:53.435 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:53.435 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:53.435 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:53.435 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:53.435 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:53.435 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +47:53.435 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:53.435 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:53.435 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:53.435 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:53.435Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:53.435Z [INFO] agent.router.manager: shutting down +47:53.435 [INFO] TestAgent: consul server down +47:53.435 [INFO] TestAgent: shutdown complete +47:53.435 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23288 network=tcp +47:53.435 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23288 network=udp +47:53.435 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23289 network=tcp +47:53.936 [INFO] TestAgent: Waiting for endpoints to shut down +47:53.936 [INFO] TestAgent: Endpoints down +2022-12-04T05:47:53.226Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:53.299Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:53.301 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9daafe3a-414f-4854-d9d0-38c6e6e868f9 Address:127.0.0.1:23286}]" +47:53.302 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9daafe3a-414f-4854-d9d0-38c6e6e868f9.dc1 127.0.0.1 +47:53.302 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9daafe3a-414f-4854-d9d0-38c6e6e868f9 127.0.0.1 +2022-12-04T05:47:53.302Z [INFO] agent.router: Initializing LAN area manager +47:53.303 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23286 [Follower]" leader= +47:53.303 [INFO] TestAgent.server: Adding LAN server: server="Node-9daafe3a-414f-4854-d9d0-38c6e6e868f9 (Addr: tcp/127.0.0.1:23286) (DC: dc1)" +47:53.303 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9daafe3a-414f-4854-d9d0-38c6e6e868f9.dc1 area=wan +47:53.303 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23281 network=tcp +47:53.303 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23281 network=udp +47:53.304 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23282 network=tcp +47:53.304 [INFO] TestAgent: started state syncer +47:53.304 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23287 network=tcp +47:53.372 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:53.372 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23286 [Candidate]" term=2 +47:53.372 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:53.372 [DEBUG] TestAgent.server.raft: vote granted: from=9daafe3a-414f-4854-d9d0-38c6e6e868f9 term=2 tally=1 +47:53.372 [INFO] TestAgent.server.raft: election won: tally=1 +47:53.372 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23286 [Leader]" +47:53.372 [INFO] TestAgent.server: cluster leadership acquired +47:53.372 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23286 +47:53.373 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:53.375 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:53.375 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:53.375 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:53.375 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:53.375 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:53.375 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9daafe3a-414f-4854-d9d0-38c6e6e868f9 +47:53.375 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9daafe3a-414f-4854-d9d0-38c6e6e868f9 +47:53.375 [INFO] TestAgent.server: New leader elected: payload=Node-9daafe3a-414f-4854-d9d0-38c6e6e868f9 +47:53.375 [INFO] TestAgent.server: federation state anti-entropy synced +47:53.501 [INFO] TestAgent: Synced node info +47:53.502 [INFO] TestAgent: Synced service: service=test +47:53.502 [DEBUG] TestAgent: Check in sync: check=service:test:1 +47:53.502 [DEBUG] TestAgent: Check in sync: check=service:test:2 +47:53.502 [DEBUG] TestAgent: Check in sync: check=service:test:3 +47:53.502 [INFO] TestAgent: Requesting shutdown +47:53.502 [INFO] TestAgent.server: shutting down server +47:53.502 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:53.502 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:53.502 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:53.502 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:53.502 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:53.502 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:53.502 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:53.502 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +47:53.502 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:53.502 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:53.502 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:53.502 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:53.502Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:53.502Z [INFO] agent.router.manager: shutting down +47:53.502 [INFO] TestAgent: consul server down +47:53.502 [INFO] TestAgent: shutdown complete +47:53.502 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23281 network=tcp +47:53.503 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23281 network=udp +47:53.503 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23282 network=tcp +47:54.004 [INFO] TestAgent: Waiting for endpoints to shut down +47:54.004 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService (0.00s) - --- PASS: TestAgent_RegisterService/normal (0.68s) - --- PASS: TestAgent_RegisterService/service_manager (0.71s) + --- PASS: TestAgent_RegisterService/service_manager (0.75s) + --- PASS: TestAgent_RegisterService/normal (0.82s) === RUN TestAgent_RegisterService_ReRegister === RUN TestAgent_RegisterService_ReRegister/normal === PAUSE TestAgent_RegisterService_ReRegister/normal @@ -6052,137 +6096,137 @@ === PAUSE TestAgent_RegisterService_ReRegister/service_manager === CONT TestAgent_RegisterService_ReRegister/normal === CONT TestAgent_RegisterService_ReRegister/service_manager -2021-10-31T23:11:36.363Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:36.423Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:36.425 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9e1087c3-0256-45d7-e9b2-fa802c81628a Address:127.0.0.1:16153}]" -11:36.425 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16153 [Follower]" leader= -11:36.434 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9e1087c3-0256-45d7-e9b2-fa802c81628a.dc1 127.0.0.1 -11:36.435 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9e1087c3-0256-45d7-e9b2-fa802c81628a 127.0.0.1 -2021-10-31T23:11:36.435Z [INFO] agent.router: Initializing LAN area manager -11:36.435 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16148 network=udp -11:36.436 [INFO] TestAgent.server: Adding LAN server: server="Node-9e1087c3-0256-45d7-e9b2-fa802c81628a (Addr: tcp/127.0.0.1:16153) (DC: dc1)" -11:36.436 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9e1087c3-0256-45d7-e9b2-fa802c81628a.dc1 area=wan -11:36.436 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16148 network=tcp -11:36.436 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16149 network=tcp -11:36.436 [INFO] TestAgent: started state syncer -11:36.437 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16154 network=tcp -11:36.490 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:36.490 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16153 [Candidate]" term=2 -11:36.491 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:36.491 [DEBUG] TestAgent.server.raft: vote granted: from=9e1087c3-0256-45d7-e9b2-fa802c81628a term=2 tally=1 -11:36.491 [INFO] TestAgent.server.raft: election won: tally=1 -11:36.491 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16153 [Leader]" -11:36.491 [INFO] TestAgent.server: cluster leadership acquired -11:36.491 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16153 -11:36.492 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:36.493 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:36.493 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:36.493 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:36.493 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:36.493 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:36.493 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9e1087c3-0256-45d7-e9b2-fa802c81628a -11:36.493 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9e1087c3-0256-45d7-e9b2-fa802c81628a -11:36.493 [INFO] TestAgent.server: New leader elected: payload=Node-9e1087c3-0256-45d7-e9b2-fa802c81628a -11:36.494 [INFO] TestAgent.server: federation state anti-entropy synced -11:36.599 [INFO] TestAgent: Synced node info -11:36.600 [INFO] TestAgent: Synced service: service=test -11:36.600 [DEBUG] TestAgent: Check in sync: check=check_1 -11:36.600 [DEBUG] TestAgent: Check in sync: check=check_2 -11:36.600 [DEBUG] TestAgent: Node info in sync -11:36.601 [INFO] TestAgent: Synced service: service=test -11:36.601 [DEBUG] TestAgent: Check in sync: check=check_1 -11:36.601 [DEBUG] TestAgent: Check in sync: check=check_2 -11:36.601 [DEBUG] TestAgent: Check in sync: check=check_3 -11:36.601 [INFO] TestAgent: Requesting shutdown -11:36.601 [INFO] TestAgent.server: shutting down server -11:36.601 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:36.601 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:36.601 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:36.601 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:36.601 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:36.601 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:36.601 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:36.601 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:36.601 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:36.601 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:36.601 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:36.601 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:36.601Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:36.601Z [INFO] agent.router.manager: shutting down -11:36.601 [INFO] TestAgent: consul server down -11:36.601 [INFO] TestAgent: shutdown complete -11:36.601 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16148 network=tcp -11:36.601 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16148 network=udp -11:36.601 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16149 network=tcp -11:37.105 [INFO] TestAgent: Waiting for endpoints to shut down -11:37.105 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:36.364Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:36.408Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:36.414 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2fd676be-3712-8ee6-0ae2-e9b85cc0d390 Address:127.0.0.1:16160}]" -11:36.414 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2fd676be-3712-8ee6-0ae2-e9b85cc0d390.dc1 127.0.0.1 -11:36.415 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2fd676be-3712-8ee6-0ae2-e9b85cc0d390 127.0.0.1 -2021-10-31T23:11:36.415Z [INFO] agent.router: Initializing LAN area manager -11:36.415 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16155 network=udp -11:36.415 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16160 [Follower]" leader= -11:36.415 [INFO] TestAgent.server: Adding LAN server: server="Node-2fd676be-3712-8ee6-0ae2-e9b85cc0d390 (Addr: tcp/127.0.0.1:16160) (DC: dc1)" -11:36.416 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2fd676be-3712-8ee6-0ae2-e9b85cc0d390.dc1 area=wan -11:36.416 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16155 network=tcp -11:36.416 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16156 network=tcp -11:36.416 [INFO] TestAgent: started state syncer -11:36.416 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16161 network=tcp -11:36.465 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:36.465 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16160 [Candidate]" term=2 -11:36.465 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:36.465 [DEBUG] TestAgent.server.raft: vote granted: from=2fd676be-3712-8ee6-0ae2-e9b85cc0d390 term=2 tally=1 -11:36.465 [INFO] TestAgent.server.raft: election won: tally=1 -11:36.465 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16160 [Leader]" -11:36.465 [INFO] TestAgent.server: cluster leadership acquired -11:36.465 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16160 -11:36.465 [INFO] TestAgent.server: New leader elected: payload=Node-2fd676be-3712-8ee6-0ae2-e9b85cc0d390 -11:36.466 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:36.468 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:36.468 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:36.468 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:36.468 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:36.468 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:36.468 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2fd676be-3712-8ee6-0ae2-e9b85cc0d390 -11:36.468 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2fd676be-3712-8ee6-0ae2-e9b85cc0d390 -11:36.469 [INFO] TestAgent.server: federation state anti-entropy synced -11:36.695 [INFO] TestAgent: Synced node info -11:36.695 [INFO] TestAgent: Synced service: service=test -11:36.695 [DEBUG] TestAgent: Check in sync: check=check_1 -11:36.695 [DEBUG] TestAgent: Check in sync: check=check_2 -11:36.695 [DEBUG] TestAgent: Node info in sync -11:36.696 [INFO] TestAgent: Synced service: service=test -11:36.696 [DEBUG] TestAgent: Check in sync: check=check_3 -11:36.696 [DEBUG] TestAgent: Check in sync: check=check_1 -11:36.696 [DEBUG] TestAgent: Check in sync: check=check_2 -11:36.696 [INFO] TestAgent: Requesting shutdown -11:36.696 [INFO] TestAgent.server: shutting down server -11:36.696 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:36.696 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:36.696 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:36.696 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:36.696 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:36.696 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:36.696 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:36.696 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:36.696 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:36.696 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:36.696 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:36.696 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:36.696Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:36.696Z [INFO] agent.router.manager: shutting down -11:36.696 [INFO] TestAgent: consul server down -11:36.696 [INFO] TestAgent: shutdown complete -11:36.696 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16155 network=tcp -11:36.696 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16155 network=udp -11:36.696 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16156 network=tcp -11:37.197 [INFO] TestAgent: Waiting for endpoints to shut down -11:37.197 [INFO] TestAgent: Endpoints down +2022-12-04T05:47:54.065Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:54.080Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:54.082 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1ce940ab-1a29-9ca7-37f9-9b3072d6c8c1 Address:127.0.0.1:23307}]" +47:54.083 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1ce940ab-1a29-9ca7-37f9-9b3072d6c8c1.dc1 127.0.0.1 +47:54.083 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1ce940ab-1a29-9ca7-37f9-9b3072d6c8c1 127.0.0.1 +2022-12-04T05:47:54.083Z [INFO] agent.router: Initializing LAN area manager +47:54.083 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23302 network=udp +47:54.083 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23307 [Follower]" leader= +47:54.084 [INFO] TestAgent.server: Adding LAN server: server="Node-1ce940ab-1a29-9ca7-37f9-9b3072d6c8c1 (Addr: tcp/127.0.0.1:23307) (DC: dc1)" +47:54.084 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1ce940ab-1a29-9ca7-37f9-9b3072d6c8c1.dc1 area=wan +47:54.084 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23302 network=tcp +47:54.096 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23303 network=tcp +47:54.096 [INFO] TestAgent: started state syncer +47:54.100 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23308 network=tcp +47:54.124 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:54.124 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23307 [Candidate]" term=2 +47:54.124 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:54.124 [DEBUG] TestAgent.server.raft: vote granted: from=1ce940ab-1a29-9ca7-37f9-9b3072d6c8c1 term=2 tally=1 +47:54.124 [INFO] TestAgent.server.raft: election won: tally=1 +47:54.124 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23307 [Leader]" +47:54.124 [INFO] TestAgent.server: cluster leadership acquired +47:54.124 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23307 +47:54.125 [INFO] TestAgent.server: New leader elected: payload=Node-1ce940ab-1a29-9ca7-37f9-9b3072d6c8c1 +47:54.125 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:54.127 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:54.127 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:54.127 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:54.127 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:54.127 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:54.127 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1ce940ab-1a29-9ca7-37f9-9b3072d6c8c1 +47:54.127 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1ce940ab-1a29-9ca7-37f9-9b3072d6c8c1 +47:54.128 [INFO] TestAgent.server: federation state anti-entropy synced +47:54.190 [INFO] TestAgent: Synced node info +47:54.190 [INFO] TestAgent: Synced service: service=test +47:54.190 [DEBUG] TestAgent: Check in sync: check=check_1 +47:54.190 [DEBUG] TestAgent: Check in sync: check=check_2 +47:54.191 [DEBUG] TestAgent: Node info in sync +47:54.191 [INFO] TestAgent: Synced service: service=test +47:54.191 [DEBUG] TestAgent: Check in sync: check=check_1 +47:54.191 [DEBUG] TestAgent: Check in sync: check=check_2 +47:54.191 [DEBUG] TestAgent: Check in sync: check=check_3 +47:54.191 [INFO] TestAgent: Requesting shutdown +47:54.192 [INFO] TestAgent.server: shutting down server +47:54.192 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:54.192 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:54.192 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:54.192 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:54.192 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:54.192 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:54.192 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:54.192 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +47:54.192 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:54.192 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:54.192 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:54.192 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:54.192Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:54.192Z [INFO] agent.router.manager: shutting down +47:54.192 [INFO] TestAgent: consul server down +47:54.192 [INFO] TestAgent: shutdown complete +47:54.192 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23302 network=tcp +47:54.192 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23302 network=udp +47:54.192 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23303 network=tcp +47:54.695 [INFO] TestAgent: Waiting for endpoints to shut down +47:54.695 [INFO] TestAgent: Endpoints down +2022-12-04T05:47:54.059Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:54.101Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:54.113 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8ca7a8be-141c-0164-ef6b-d0a313cd4193 Address:127.0.0.1:23300}]" +47:54.114 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8ca7a8be-141c-0164-ef6b-d0a313cd4193.dc1 127.0.0.1 +47:54.114 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23300 [Follower]" leader= +47:54.115 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8ca7a8be-141c-0164-ef6b-d0a313cd4193 127.0.0.1 +2022-12-04T05:47:54.115Z [INFO] agent.router: Initializing LAN area manager +47:54.115 [INFO] TestAgent.server: Adding LAN server: server="Node-8ca7a8be-141c-0164-ef6b-d0a313cd4193 (Addr: tcp/127.0.0.1:23300) (DC: dc1)" +47:54.115 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8ca7a8be-141c-0164-ef6b-d0a313cd4193.dc1 area=wan +47:54.116 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23295 network=tcp +47:54.116 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23295 network=udp +47:54.116 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23296 network=tcp +47:54.116 [INFO] TestAgent: started state syncer +47:54.116 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23301 network=tcp +47:54.180 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:54.180 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23300 [Candidate]" term=2 +47:54.180 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:54.180 [DEBUG] TestAgent.server.raft: vote granted: from=8ca7a8be-141c-0164-ef6b-d0a313cd4193 term=2 tally=1 +47:54.180 [INFO] TestAgent.server.raft: election won: tally=1 +47:54.180 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23300 [Leader]" +47:54.180 [INFO] TestAgent.server: cluster leadership acquired +47:54.181 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23300 +47:54.181 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:54.183 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:54.183 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:54.183 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:54.183 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:54.183 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:54.183 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8ca7a8be-141c-0164-ef6b-d0a313cd4193 +47:54.183 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8ca7a8be-141c-0164-ef6b-d0a313cd4193 +47:54.183 [INFO] TestAgent.server: New leader elected: payload=Node-8ca7a8be-141c-0164-ef6b-d0a313cd4193 +47:54.184 [INFO] TestAgent.server: federation state anti-entropy synced +47:54.490 [INFO] TestAgent: Synced node info +47:54.491 [INFO] TestAgent: Synced service: service=test +47:54.491 [DEBUG] TestAgent: Check in sync: check=check_1 +47:54.491 [DEBUG] TestAgent: Check in sync: check=check_2 +47:54.491 [DEBUG] TestAgent: Node info in sync +47:54.492 [INFO] TestAgent: Synced service: service=test +47:54.492 [DEBUG] TestAgent: Check in sync: check=check_2 +47:54.492 [DEBUG] TestAgent: Check in sync: check=check_3 +47:54.492 [DEBUG] TestAgent: Check in sync: check=check_1 +47:54.492 [INFO] TestAgent: Requesting shutdown +47:54.492 [INFO] TestAgent.server: shutting down server +47:54.492 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:54.492 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:54.492 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:54.492 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:54.492 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:54.492 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:54.492 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:54.492 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +47:54.492 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:54.492 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:54.492 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:54.492 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:54.493Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:54.493Z [INFO] agent.router.manager: shutting down +47:54.493 [INFO] TestAgent: consul server down +47:54.493 [INFO] TestAgent: shutdown complete +47:54.493 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23295 network=tcp +47:54.493 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23295 network=udp +47:54.493 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23296 network=tcp +47:54.994 [INFO] TestAgent: Waiting for endpoints to shut down +47:54.994 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_ReRegister (0.00s) - --- PASS: TestAgent_RegisterService_ReRegister/normal (0.76s) - --- PASS: TestAgent_RegisterService_ReRegister/service_manager (0.85s) + --- PASS: TestAgent_RegisterService_ReRegister/service_manager (0.69s) + --- PASS: TestAgent_RegisterService_ReRegister/normal (0.99s) === RUN TestAgent_RegisterService_ReRegister_ReplaceExistingChecks === RUN TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/normal === PAUSE TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/normal @@ -6190,141 +6234,141 @@ === PAUSE TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/service_manager === CONT TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/normal === CONT TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/service_manager -2021-10-31T23:11:37.254Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:37.274Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:37.311 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:166db38c-6098-fb02-d14b-c5c88349c1f6 Address:127.0.0.1:16174}]" -11:37.311 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16174 [Follower]" leader= -11:37.311 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-166db38c-6098-fb02-d14b-c5c88349c1f6.dc1 127.0.0.1 -11:37.312 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-166db38c-6098-fb02-d14b-c5c88349c1f6 127.0.0.1 -2021-10-31T23:11:37.312Z [INFO] agent.router: Initializing LAN area manager -11:37.312 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16169 network=udp -11:37.312 [INFO] TestAgent.server: Adding LAN server: server="Node-166db38c-6098-fb02-d14b-c5c88349c1f6 (Addr: tcp/127.0.0.1:16174) (DC: dc1)" -11:37.312 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-166db38c-6098-fb02-d14b-c5c88349c1f6.dc1 area=wan -11:37.312 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16169 network=tcp -11:37.313 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16170 network=tcp -11:37.313 [INFO] TestAgent: started state syncer -11:37.313 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16175 network=tcp -11:37.369 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:37.370 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16174 [Candidate]" term=2 -11:37.370 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:37.370 [DEBUG] TestAgent.server.raft: vote granted: from=166db38c-6098-fb02-d14b-c5c88349c1f6 term=2 tally=1 -11:37.370 [INFO] TestAgent.server.raft: election won: tally=1 -11:37.370 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16174 [Leader]" -11:37.370 [INFO] TestAgent.server: cluster leadership acquired -11:37.370 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16174 -11:37.371 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:37.372 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:37.372 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:37.372 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:37.372 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:37.372 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:37.372 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-166db38c-6098-fb02-d14b-c5c88349c1f6 -11:37.372 [INFO] TestAgent.server: member joined, marking health alive: member=Node-166db38c-6098-fb02-d14b-c5c88349c1f6 -11:37.373 [INFO] TestAgent.server: New leader elected: payload=Node-166db38c-6098-fb02-d14b-c5c88349c1f6 -11:37.373 [INFO] TestAgent.server: federation state anti-entropy synced -11:37.405 [INFO] TestAgent: Synced node info -11:37.405 [INFO] TestAgent: Synced service: service=test -11:37.405 [DEBUG] TestAgent: Check in sync: check=service:test:1 -11:37.405 [DEBUG] TestAgent: Check in sync: check=check_2 -11:37.406 [DEBUG] TestAgent: removed check: check=check_2 -11:37.406 [DEBUG] TestAgent: Node info in sync -11:37.407 [INFO] TestAgent: Synced service: service=test -11:37.407 [DEBUG] TestAgent: Check in sync: check=service:test:1 -11:37.407 [INFO] TestAgent: Deregistered check: check=check_2 -11:37.407 [DEBUG] TestAgent: Check in sync: check=check_3 -11:37.407 [INFO] TestAgent: Requesting shutdown -11:37.407 [INFO] TestAgent.server: shutting down server -11:37.407 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:37.407 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:37.407 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:37.407 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:37.407 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:37.407 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:37.407 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:37.407 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:37.407 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:37.407 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:37.407 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:37.407 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:37.408Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:37.408Z [INFO] agent.router.manager: shutting down -11:37.408 [INFO] TestAgent: consul server down -11:37.408 [INFO] TestAgent: shutdown complete -11:37.408 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16169 network=tcp -11:37.408 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16169 network=udp -11:37.408 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16170 network=tcp -11:37.909 [INFO] TestAgent: Waiting for endpoints to shut down -11:37.909 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:37.236Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:37.273Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:37.275 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:70551bf4-812b-d9cc-1d18-9d8fb9a5e644 Address:127.0.0.1:16167}]" -11:37.294 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-70551bf4-812b-d9cc-1d18-9d8fb9a5e644.dc1 127.0.0.1 -11:37.295 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-70551bf4-812b-d9cc-1d18-9d8fb9a5e644 127.0.0.1 -2021-10-31T23:11:37.295Z [INFO] agent.router: Initializing LAN area manager -11:37.301 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16162 network=udp -11:37.309 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-70551bf4-812b-d9cc-1d18-9d8fb9a5e644.dc1 area=wan -11:37.310 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16162 network=tcp -11:37.310 [INFO] TestAgent.server: Adding LAN server: server="Node-70551bf4-812b-d9cc-1d18-9d8fb9a5e644 (Addr: tcp/127.0.0.1:16167) (DC: dc1)" -11:37.310 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16167 [Follower]" leader= -11:37.310 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16163 network=tcp -11:37.310 [INFO] TestAgent: started state syncer -11:37.310 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16168 network=tcp -11:37.356 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:37.356 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16167 [Candidate]" term=2 -11:37.356 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:37.356 [DEBUG] TestAgent.server.raft: vote granted: from=70551bf4-812b-d9cc-1d18-9d8fb9a5e644 term=2 tally=1 -11:37.356 [INFO] TestAgent.server.raft: election won: tally=1 -11:37.356 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16167 [Leader]" -11:37.356 [INFO] TestAgent.server: cluster leadership acquired -11:37.356 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16167 -11:37.356 [INFO] TestAgent.server: New leader elected: payload=Node-70551bf4-812b-d9cc-1d18-9d8fb9a5e644 -11:37.357 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:37.359 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:37.359 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:37.359 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:37.359 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:37.359 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:37.359 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-70551bf4-812b-d9cc-1d18-9d8fb9a5e644 -11:37.359 [INFO] TestAgent.server: member joined, marking health alive: member=Node-70551bf4-812b-d9cc-1d18-9d8fb9a5e644 -11:37.360 [INFO] TestAgent.server: federation state anti-entropy synced -11:37.403 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:37.404 [INFO] TestAgent: Synced node info -11:37.404 [DEBUG] TestAgent: Node info in sync -11:37.448 [DEBUG] TestAgent: Node info in sync -11:37.449 [INFO] TestAgent: Synced service: service=test -11:37.449 [DEBUG] TestAgent: Check in sync: check=service:test:1 -11:37.449 [DEBUG] TestAgent: Check in sync: check=check_2 -11:37.450 [DEBUG] TestAgent: removed check: check=check_2 -11:37.450 [DEBUG] TestAgent: Node info in sync -11:37.450 [INFO] TestAgent: Synced service: service=test -11:37.450 [DEBUG] TestAgent: Check in sync: check=service:test:1 -11:37.450 [INFO] TestAgent: Deregistered check: check=check_2 -11:37.450 [DEBUG] TestAgent: Check in sync: check=check_3 -11:37.451 [INFO] TestAgent: Requesting shutdown -11:37.451 [INFO] TestAgent.server: shutting down server -11:37.451 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:37.451 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:37.451 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:37.451 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:37.451 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:37.451 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:37.451 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:37.451 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:37.451 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:37.451 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:37.451 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:37.451Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:37.451Z [INFO] agent.router.manager: shutting down -11:37.451 [INFO] TestAgent: consul server down -11:37.451 [INFO] TestAgent: shutdown complete -11:37.451 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16162 network=tcp -11:37.451 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16162 network=udp -11:37.451 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16163 network=tcp -11:37.953 [INFO] TestAgent: Waiting for endpoints to shut down -11:37.953 [INFO] TestAgent: Endpoints down +2022-12-04T05:47:55.053Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:55.086Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:55.087 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6da121d0-47e5-7842-6fe2-ad3c4766244b Address:127.0.0.1:23321}]" +47:55.088 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23321 [Follower]" leader= +47:55.088 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6da121d0-47e5-7842-6fe2-ad3c4766244b.dc1 127.0.0.1 +47:55.089 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6da121d0-47e5-7842-6fe2-ad3c4766244b 127.0.0.1 +2022-12-04T05:47:55.089Z [INFO] agent.router: Initializing LAN area manager +47:55.100 [INFO] TestAgent.server: Adding LAN server: server="Node-6da121d0-47e5-7842-6fe2-ad3c4766244b (Addr: tcp/127.0.0.1:23321) (DC: dc1)" +47:55.100 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6da121d0-47e5-7842-6fe2-ad3c4766244b.dc1 area=wan +47:55.101 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23316 network=tcp +47:55.101 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23316 network=udp +47:55.103 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23317 network=tcp +47:55.103 [INFO] TestAgent: started state syncer +47:55.103 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23322 network=tcp +47:55.161 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:55.161 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23321 [Candidate]" term=2 +47:55.161 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:55.161 [DEBUG] TestAgent.server.raft: vote granted: from=6da121d0-47e5-7842-6fe2-ad3c4766244b term=2 tally=1 +47:55.161 [INFO] TestAgent.server.raft: election won: tally=1 +47:55.161 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23321 [Leader]" +47:55.161 [INFO] TestAgent.server: cluster leadership acquired +47:55.161 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23321 +47:55.162 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:55.163 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:55.163 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:55.163 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:55.163 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:55.163 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:55.163 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6da121d0-47e5-7842-6fe2-ad3c4766244b +47:55.163 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6da121d0-47e5-7842-6fe2-ad3c4766244b +47:55.164 [INFO] TestAgent.server: New leader elected: payload=Node-6da121d0-47e5-7842-6fe2-ad3c4766244b +47:55.165 [INFO] TestAgent.server: federation state anti-entropy synced +47:55.341 [INFO] TestAgent: Synced node info +47:55.342 [INFO] TestAgent: Synced service: service=test +47:55.342 [DEBUG] TestAgent: Check in sync: check=service:test:1 +47:55.342 [DEBUG] TestAgent: Check in sync: check=check_2 +47:55.343 [DEBUG] TestAgent: removed check: check=check_2 +47:55.343 [DEBUG] TestAgent: Node info in sync +47:55.343 [INFO] TestAgent: Synced service: service=test +47:55.343 [DEBUG] TestAgent: Check in sync: check=check_3 +47:55.343 [DEBUG] TestAgent: Check in sync: check=service:test:1 +47:55.343 [INFO] TestAgent: Deregistered check: check=check_2 +47:55.343 [INFO] TestAgent: Requesting shutdown +47:55.344 [INFO] TestAgent.server: shutting down server +47:55.344 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:55.344 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:55.344 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:55.344 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:55.344 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:55.344 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:55.344 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:55.344 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +47:55.344 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:55.344 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:55.344 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:55.344 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:55.344Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:55.344Z [INFO] agent.router.manager: shutting down +47:55.344 [INFO] TestAgent: consul server down +47:55.344 [INFO] TestAgent: shutdown complete +47:55.344 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23316 network=tcp +47:55.344 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23316 network=udp +47:55.344 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23317 network=tcp +47:55.848 [INFO] TestAgent: Waiting for endpoints to shut down +47:55.848 [INFO] TestAgent: Endpoints down +2022-12-04T05:47:55.026Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:55.065Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:55.066 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:746e8602-dd66-e0b2-3ad2-a36c451f1fbe Address:127.0.0.1:23314}]" +47:55.067 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-746e8602-dd66-e0b2-3ad2-a36c451f1fbe.dc1 127.0.0.1 +47:55.068 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-746e8602-dd66-e0b2-3ad2-a36c451f1fbe 127.0.0.1 +2022-12-04T05:47:55.068Z [INFO] agent.router: Initializing LAN area manager +47:55.068 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23314 [Follower]" leader= +47:55.068 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-746e8602-dd66-e0b2-3ad2-a36c451f1fbe.dc1 area=wan +47:55.068 [INFO] TestAgent.server: Adding LAN server: server="Node-746e8602-dd66-e0b2-3ad2-a36c451f1fbe (Addr: tcp/127.0.0.1:23314) (DC: dc1)" +47:55.069 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23309 network=udp +47:55.069 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23309 network=tcp +47:55.070 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23310 network=tcp +47:55.070 [INFO] TestAgent: started state syncer +47:55.070 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23315 network=tcp +47:55.111 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:55.111 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23314 [Candidate]" term=2 +47:55.111 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:55.111 [DEBUG] TestAgent.server.raft: vote granted: from=746e8602-dd66-e0b2-3ad2-a36c451f1fbe term=2 tally=1 +47:55.111 [INFO] TestAgent.server.raft: election won: tally=1 +47:55.111 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23314 [Leader]" +47:55.111 [INFO] TestAgent.server: cluster leadership acquired +47:55.111 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23314 +47:55.112 [INFO] TestAgent.server: New leader elected: payload=Node-746e8602-dd66-e0b2-3ad2-a36c451f1fbe +47:55.113 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:55.114 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:55.114 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:55.114 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:55.114 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:55.114 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:55.124 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-746e8602-dd66-e0b2-3ad2-a36c451f1fbe +47:55.124 [INFO] TestAgent.server: member joined, marking health alive: member=Node-746e8602-dd66-e0b2-3ad2-a36c451f1fbe +47:55.125 [INFO] TestAgent.server: federation state anti-entropy synced +47:55.152 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.153 [INFO] TestAgent: Synced node info +47:55.153 [DEBUG] TestAgent: Node info in sync +47:55.442 [DEBUG] TestAgent: Node info in sync +47:55.443 [INFO] TestAgent: Synced service: service=test +47:55.443 [DEBUG] TestAgent: Check in sync: check=check_2 +47:55.443 [DEBUG] TestAgent: Check in sync: check=service:test:1 +47:55.443 [DEBUG] TestAgent: removed check: check=check_2 +47:55.443 [DEBUG] TestAgent: Node info in sync +47:55.444 [INFO] TestAgent: Synced service: service=test +47:55.444 [DEBUG] TestAgent: Check in sync: check=service:test:1 +47:55.444 [INFO] TestAgent: Deregistered check: check=check_2 +47:55.444 [DEBUG] TestAgent: Check in sync: check=check_3 +47:55.444 [INFO] TestAgent: Requesting shutdown +47:55.444 [INFO] TestAgent.server: shutting down server +47:55.444 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:55.444 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:55.444 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:55.444 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:55.445 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:55.445 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:55.445 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:55.445 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:55.445 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:55.445 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:55.445 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:55.445Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:55.445Z [INFO] agent.router.manager: shutting down +47:55.445 [INFO] TestAgent: consul server down +47:55.445 [INFO] TestAgent: shutdown complete +47:55.445 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23309 network=tcp +47:55.445 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23309 network=udp +47:55.445 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23310 network=tcp +47:55.947 [INFO] TestAgent: Waiting for endpoints to shut down +47:55.947 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_ReRegister_ReplaceExistingChecks (0.00s) - --- PASS: TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/service_manager (0.70s) - --- PASS: TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/normal (0.76s) + --- PASS: TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/service_manager (0.85s) + --- PASS: TestAgent_RegisterService_ReRegister_ReplaceExistingChecks/normal (0.95s) === RUN TestAgent_RegisterService_TranslateKeys === RUN TestAgent_RegisterService_TranslateKeys/normal === PAUSE TestAgent_RegisterService_TranslateKeys/normal @@ -6332,178 +6376,174 @@ === PAUSE TestAgent_RegisterService_TranslateKeys/service_manager === CONT TestAgent_RegisterService_TranslateKeys/normal === CONT TestAgent_RegisterService_TranslateKeys/service_manager -=== RUN TestAgent_RegisterService_TranslateKeys/service_manager/no_token -=== RUN TestAgent_RegisterService_TranslateKeys/service_manager/root_token === RUN TestAgent_RegisterService_TranslateKeys/normal/no_token === RUN TestAgent_RegisterService_TranslateKeys/normal/root_token -2021-10-31T23:11:38.018Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:38.018Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:38.055Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:38.055Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:38.057 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4346a767-b438-7d3e-55de-cdd50ac8e8da Address:127.0.0.1:16188}]" -11:38.057 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16188 [Follower]" leader= -11:38.066 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4346a767-b438-7d3e-55de-cdd50ac8e8da.dc1 127.0.0.1 -11:38.074 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4346a767-b438-7d3e-55de-cdd50ac8e8da 127.0.0.1 -2021-10-31T23:11:38.074Z [INFO] agent.router: Initializing LAN area manager -11:38.082 [INFO] TestAgent.server: Adding LAN server: server="Node-4346a767-b438-7d3e-55de-cdd50ac8e8da (Addr: tcp/127.0.0.1:16188) (DC: dc1)" -11:38.082 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16183 network=tcp -11:38.082 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16183 network=udp -11:38.082 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4346a767-b438-7d3e-55de-cdd50ac8e8da.dc1 area=wan -11:38.083 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16184 network=tcp -11:38.083 [INFO] TestAgent: started state syncer -11:38.083 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16189 network=tcp -11:38.117 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:38.117 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16188 [Candidate]" term=2 -11:38.118 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:38.118 [DEBUG] TestAgent.server.raft: vote granted: from=4346a767-b438-7d3e-55de-cdd50ac8e8da term=2 tally=1 -11:38.118 [INFO] TestAgent.server.raft: election won: tally=1 -11:38.118 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16188 [Leader]" -11:38.118 [INFO] TestAgent.server: cluster leadership acquired -11:38.118 [INFO] TestAgent.server: initializing acls -11:38.118 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:38.118 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:38.119 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:38.119 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:38.119 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:38.119 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:38.119 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4346a767-b438-7d3e-55de-cdd50ac8e8da -11:38.119 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4346a767-b438-7d3e-55de-cdd50ac8e8da.dc1 -11:38.120 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:38.121 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:38.121 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:38.121 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:38.121 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:38.121 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:38.121 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4346a767-b438-7d3e-55de-cdd50ac8e8da -11:38.121 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4346a767-b438-7d3e-55de-cdd50ac8e8da -11:38.122 [INFO] TestAgent.server: New leader elected: payload=Node-4346a767-b438-7d3e-55de-cdd50ac8e8da -11:38.122 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4346a767-b438-7d3e-55de-cdd50ac8e8da.dc1 area=wan -11:38.122 [INFO] TestAgent.server: Updating LAN server: server="Node-4346a767-b438-7d3e-55de-cdd50ac8e8da (Addr: tcp/127.0.0.1:16188) (DC: dc1)" -11:38.122 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4346a767-b438-7d3e-55de-cdd50ac8e8da -11:38.122 [INFO] TestAgent.server: federation state anti-entropy synced -11:38.370 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4346a767-b438-7d3e-55de-cdd50ac8e8da -11:38.370 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4346a767-b438-7d3e-55de-cdd50ac8e8da -11:38.372 [INFO] TestAgent: Synced node info -11:38.373 [INFO] TestAgent: Synced service: service=test -11:38.373 [DEBUG] TestAgent: Check in sync: check=service:test:1 -11:38.373 [DEBUG] TestAgent: Check in sync: check=service:test:2 -11:38.373 [DEBUG] TestAgent: Check in sync: check=service:test:3 -11:38.373 [INFO] TestAgent: Requesting shutdown -11:38.373 [INFO] TestAgent.server: shutting down server -11:38.373 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:38.373 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:38.373 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:38.373 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:38.373 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:38.373 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:38.373 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:38.373 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:38.373 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:38.373 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:38.373 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:38.373 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:38.373 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:38.373 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:38.373 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:38.373 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:38.373Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:38.373Z [INFO] agent.router.manager: shutting down -11:38.373 [INFO] TestAgent: consul server down -11:38.381 [INFO] TestAgent: shutdown complete -11:38.381 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16183 network=tcp -11:38.382 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16183 network=udp -11:38.382 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16184 network=tcp -11:38.885 [INFO] TestAgent: Waiting for endpoints to shut down -11:38.885 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:38.012Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:38.012Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:38.029Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:38.029Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:38.039 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9f68fa29-9d06-ac3c-08ef-d4fb10493722 Address:127.0.0.1:16181}]" -11:38.039 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9f68fa29-9d06-ac3c-08ef-d4fb10493722.dc1 127.0.0.1 -11:38.040 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9f68fa29-9d06-ac3c-08ef-d4fb10493722 127.0.0.1 -2021-10-31T23:11:38.040Z [INFO] agent.router: Initializing LAN area manager -11:38.040 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16176 network=udp -11:38.040 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16181 [Follower]" leader= -11:38.040 [INFO] TestAgent.server: Adding LAN server: server="Node-9f68fa29-9d06-ac3c-08ef-d4fb10493722 (Addr: tcp/127.0.0.1:16181) (DC: dc1)" -11:38.040 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9f68fa29-9d06-ac3c-08ef-d4fb10493722.dc1 area=wan -11:38.040 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16176 network=tcp -11:38.041 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16177 network=tcp -11:38.041 [INFO] TestAgent: started state syncer -11:38.041 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16182 network=tcp -11:38.093 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -11:38.101 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:38.101 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16181 [Candidate]" term=2 -11:38.102 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:38.102 [DEBUG] TestAgent.server.raft: vote granted: from=9f68fa29-9d06-ac3c-08ef-d4fb10493722 term=2 tally=1 -11:38.102 [INFO] TestAgent.server.raft: election won: tally=1 -11:38.102 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16181 [Leader]" -11:38.102 [INFO] TestAgent.server: cluster leadership acquired -11:38.102 [INFO] TestAgent.server: initializing acls -11:38.102 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:38.102 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:38.103 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:38.103 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:38.103 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:38.103 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:38.103 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9f68fa29-9d06-ac3c-08ef-d4fb10493722 -11:38.103 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9f68fa29-9d06-ac3c-08ef-d4fb10493722.dc1 -11:38.104 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:38.105 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:38.105 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:38.105 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:38.105 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:38.105 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:38.105 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9f68fa29-9d06-ac3c-08ef-d4fb10493722 -11:38.105 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9f68fa29-9d06-ac3c-08ef-d4fb10493722 -11:38.105 [INFO] TestAgent.server: New leader elected: payload=Node-9f68fa29-9d06-ac3c-08ef-d4fb10493722 -11:38.105 [INFO] TestAgent.server: Updating LAN server: server="Node-9f68fa29-9d06-ac3c-08ef-d4fb10493722 (Addr: tcp/127.0.0.1:16181) (DC: dc1)" -11:38.106 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9f68fa29-9d06-ac3c-08ef-d4fb10493722.dc1 area=wan -11:38.110 [INFO] TestAgent.server: federation state anti-entropy synced -11:38.110 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9f68fa29-9d06-ac3c-08ef-d4fb10493722 -11:38.470 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:38.471 [INFO] TestAgent: Synced node info -11:38.471 [DEBUG] TestAgent: Node info in sync -11:38.485 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9f68fa29-9d06-ac3c-08ef-d4fb10493722 -11:38.485 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9f68fa29-9d06-ac3c-08ef-d4fb10493722 -11:38.486 [DEBUG] TestAgent: Node info in sync -11:38.487 [INFO] TestAgent: Synced service: service=test -11:38.487 [DEBUG] TestAgent: Check in sync: check=service:test:1 -11:38.487 [DEBUG] TestAgent: Check in sync: check=service:test:2 -11:38.487 [DEBUG] TestAgent: Check in sync: check=service:test:3 -11:38.487 [INFO] TestAgent: Requesting shutdown -11:38.487 [INFO] TestAgent.server: shutting down server -11:38.487 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:38.487 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:38.487 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:38.487 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:38.487 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:38.487 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:38.487 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:38.487 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:38.487 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:38.487 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:38.487 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:38.487 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:38.487 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:38.487 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:38.487 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:38.488Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:38.488Z [INFO] agent.router.manager: shutting down -11:38.488 [INFO] TestAgent: consul server down -11:38.488 [INFO] TestAgent: shutdown complete -11:38.488 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16176 network=tcp -11:38.488 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16176 network=udp -11:38.488 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16177 network=tcp -11:38.989 [INFO] TestAgent: Waiting for endpoints to shut down -11:38.989 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_RegisterService_TranslateKeys/service_manager/no_token +=== RUN TestAgent_RegisterService_TranslateKeys/service_manager/root_token +2022-12-04T05:47:55.980Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:47:55.980Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:56.021Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:47:56.021Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:56.023 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:04ae61a9-a69c-de0c-fc12-ff6c91470e10 Address:127.0.0.1:23328}]" +47:56.023 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-04ae61a9-a69c-de0c-fc12-ff6c91470e10.dc1 127.0.0.1 +47:56.024 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-04ae61a9-a69c-de0c-fc12-ff6c91470e10 127.0.0.1 +2022-12-04T05:47:56.024Z [INFO] agent.router: Initializing LAN area manager +47:56.024 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23328 [Follower]" leader= +47:56.024 [INFO] TestAgent.server: Adding LAN server: server="Node-04ae61a9-a69c-de0c-fc12-ff6c91470e10 (Addr: tcp/127.0.0.1:23328) (DC: dc1)" +47:56.024 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-04ae61a9-a69c-de0c-fc12-ff6c91470e10.dc1 area=wan +47:56.025 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23323 network=udp +47:56.025 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23323 network=tcp +47:56.025 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23324 network=tcp +47:56.025 [INFO] TestAgent: started state syncer +47:56.025 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23329 network=tcp +47:56.068 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:56.068 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23328 [Candidate]" term=2 +47:56.068 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:56.068 [DEBUG] TestAgent.server.raft: vote granted: from=04ae61a9-a69c-de0c-fc12-ff6c91470e10 term=2 tally=1 +47:56.068 [INFO] TestAgent.server.raft: election won: tally=1 +47:56.068 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23328 [Leader]" +47:56.068 [INFO] TestAgent.server: cluster leadership acquired +47:56.069 [INFO] TestAgent.server: initializing acls +47:56.069 [INFO] TestAgent.server: Created ACL 'global-management' policy +47:56.069 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +47:56.069 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +47:56.070 [INFO] TestAgent.server: Created ACL anonymous token from configuration +47:56.070 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +47:56.070 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +47:56.070 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-04ae61a9-a69c-de0c-fc12-ff6c91470e10 +47:56.070 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-04ae61a9-a69c-de0c-fc12-ff6c91470e10.dc1 +47:56.071 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:56.072 [INFO] TestAgent.server: New leader elected: payload=Node-04ae61a9-a69c-de0c-fc12-ff6c91470e10 +47:56.072 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-04ae61a9-a69c-de0c-fc12-ff6c91470e10.dc1 area=wan +47:56.072 [INFO] TestAgent.server: Updating LAN server: server="Node-04ae61a9-a69c-de0c-fc12-ff6c91470e10 (Addr: tcp/127.0.0.1:23328) (DC: dc1)" +47:56.072 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:56.072 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:56.072 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:56.072 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:56.072 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:56.072 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-04ae61a9-a69c-de0c-fc12-ff6c91470e10 +47:56.072 [INFO] TestAgent.server: member joined, marking health alive: member=Node-04ae61a9-a69c-de0c-fc12-ff6c91470e10 +47:56.073 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-04ae61a9-a69c-de0c-fc12-ff6c91470e10 +47:56.073 [INFO] TestAgent.server: federation state anti-entropy synced +47:56.103 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-04ae61a9-a69c-de0c-fc12-ff6c91470e10 +47:56.103 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-04ae61a9-a69c-de0c-fc12-ff6c91470e10 +47:56.120 [INFO] TestAgent: Synced node info +47:56.122 [INFO] TestAgent: Synced service: service=test +47:56.122 [DEBUG] TestAgent: Check in sync: check=service:test:1 +47:56.122 [DEBUG] TestAgent: Check in sync: check=service:test:2 +47:56.122 [DEBUG] TestAgent: Check in sync: check=service:test:3 +47:56.122 [INFO] TestAgent: Requesting shutdown +47:56.122 [INFO] TestAgent.server: shutting down server +47:56.122 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:56.122 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:56.122 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:56.122 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:56.122 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +47:56.122 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +47:56.122 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:56.122 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:56.122 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:56.122 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +47:56.122 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:56.122 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:56.122 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:56.122 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +47:56.122 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +47:56.122 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:56.122Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:56.122Z [INFO] agent.router.manager: shutting down +47:56.122 [INFO] TestAgent: consul server down +47:56.122 [INFO] TestAgent: shutdown complete +47:56.122 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23323 network=tcp +47:56.123 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23323 network=udp +47:56.123 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23324 network=tcp +47:56.623 [INFO] TestAgent: Waiting for endpoints to shut down +47:56.623 [INFO] TestAgent: Endpoints down +2022-12-04T05:47:55.975Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:47:55.975Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:56.029Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:47:56.029Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:56.030 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e17e30f1-d302-995f-b7e4-ae01562c096b Address:127.0.0.1:23335}]" +47:56.031 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e17e30f1-d302-995f-b7e4-ae01562c096b.dc1 127.0.0.1 +47:56.032 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e17e30f1-d302-995f-b7e4-ae01562c096b 127.0.0.1 +2022-12-04T05:47:56.032Z [INFO] agent.router: Initializing LAN area manager +47:56.032 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23335 [Follower]" leader= +47:56.032 [INFO] TestAgent.server: Adding LAN server: server="Node-e17e30f1-d302-995f-b7e4-ae01562c096b (Addr: tcp/127.0.0.1:23335) (DC: dc1)" +47:56.032 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23330 network=udp +47:56.032 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e17e30f1-d302-995f-b7e4-ae01562c096b.dc1 area=wan +47:56.032 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23330 network=tcp +47:56.033 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23331 network=tcp +47:56.033 [INFO] TestAgent: started state syncer +47:56.033 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23336 network=tcp +47:56.076 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:56.076 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23335 [Candidate]" term=2 +47:56.076 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:56.076 [DEBUG] TestAgent.server.raft: vote granted: from=e17e30f1-d302-995f-b7e4-ae01562c096b term=2 tally=1 +47:56.076 [INFO] TestAgent.server.raft: election won: tally=1 +47:56.076 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23335 [Leader]" +47:56.076 [INFO] TestAgent.server: cluster leadership acquired +47:56.076 [INFO] TestAgent.server: initializing acls +47:56.077 [INFO] TestAgent.server: Created ACL 'global-management' policy +47:56.077 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +47:56.077 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +47:56.077 [INFO] TestAgent.server: Created ACL anonymous token from configuration +47:56.077 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +47:56.077 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +47:56.078 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e17e30f1-d302-995f-b7e4-ae01562c096b +47:56.078 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e17e30f1-d302-995f-b7e4-ae01562c096b.dc1 +47:56.078 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:56.080 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:56.080 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:56.080 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:56.080 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:56.080 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:56.080 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e17e30f1-d302-995f-b7e4-ae01562c096b +47:56.080 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e17e30f1-d302-995f-b7e4-ae01562c096b +47:56.080 [INFO] TestAgent.server: New leader elected: payload=Node-e17e30f1-d302-995f-b7e4-ae01562c096b +47:56.080 [INFO] TestAgent.server: Updating LAN server: server="Node-e17e30f1-d302-995f-b7e4-ae01562c096b (Addr: tcp/127.0.0.1:23335) (DC: dc1)" +47:56.080 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e17e30f1-d302-995f-b7e4-ae01562c096b.dc1 area=wan +47:56.084 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e17e30f1-d302-995f-b7e4-ae01562c096b +47:56.085 [INFO] TestAgent.server: federation state anti-entropy synced +47:56.108 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e17e30f1-d302-995f-b7e4-ae01562c096b +47:56.123 [INFO] TestAgent: Synced node info +47:56.126 [INFO] TestAgent: Synced service: service=test +47:56.126 [DEBUG] TestAgent: Check in sync: check=service:test:1 +47:56.126 [DEBUG] TestAgent: Check in sync: check=service:test:2 +47:56.126 [DEBUG] TestAgent: Check in sync: check=service:test:3 +47:56.126 [INFO] TestAgent: Requesting shutdown +47:56.126 [INFO] TestAgent.server: shutting down server +47:56.126 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:56.126 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +47:56.127 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +47:56.127 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:56.127 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:56.127 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:56.127 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:56.127 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +47:56.127 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" +47:56.127 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +47:56.127 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:56.127 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:56.127 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +47:56.127 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:56.127 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:56.127 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:56.127Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:56.127Z [INFO] agent.router.manager: shutting down +47:56.127 [INFO] TestAgent: consul server down +47:56.127 [INFO] TestAgent: shutdown complete +47:56.127 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23330 network=tcp +47:56.127 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23330 network=udp +47:56.127 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23331 network=tcp +47:56.627 [INFO] TestAgent: Waiting for endpoints to shut down +47:56.627 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_TranslateKeys (0.00s) - --- PASS: TestAgent_RegisterService_TranslateKeys/service_manager (0.93s) - --- PASS: TestAgent_RegisterService_TranslateKeys/service_manager/no_token (0.00s) - --- PASS: TestAgent_RegisterService_TranslateKeys/service_manager/root_token (0.00s) - --- PASS: TestAgent_RegisterService_TranslateKeys/normal (1.04s) + --- PASS: TestAgent_RegisterService_TranslateKeys/normal (0.68s) --- PASS: TestAgent_RegisterService_TranslateKeys/normal/no_token (0.00s) - --- PASS: TestAgent_RegisterService_TranslateKeys/normal/root_token (0.00s) + --- PASS: TestAgent_RegisterService_TranslateKeys/normal/root_token (0.02s) + --- PASS: TestAgent_RegisterService_TranslateKeys/service_manager (0.68s) + --- PASS: TestAgent_RegisterService_TranslateKeys/service_manager/no_token (0.00s) + --- PASS: TestAgent_RegisterService_TranslateKeys/service_manager/root_token (0.02s) === RUN TestAgent_RegisterService_ACLDeny === RUN TestAgent_RegisterService_ACLDeny/normal === PAUSE TestAgent_RegisterService_ACLDeny/normal @@ -6511,178 +6551,180 @@ === PAUSE TestAgent_RegisterService_ACLDeny/service_manager === CONT TestAgent_RegisterService_ACLDeny/normal === CONT TestAgent_RegisterService_ACLDeny/service_manager -=== RUN TestAgent_RegisterService_ACLDeny/normal/no_token -=== RUN TestAgent_RegisterService_ACLDeny/normal/root_token === RUN TestAgent_RegisterService_ACLDeny/service_manager/no_token === RUN TestAgent_RegisterService_ACLDeny/service_manager/root_token -2021-10-31T23:11:39.034Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:39.034Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:39.060Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:39.060Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:39.062 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bcbf3e09-03b0-bd8b-3787-263ccac1ef4e Address:127.0.0.1:16195}]" -11:39.072 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bcbf3e09-03b0-bd8b-3787-263ccac1ef4e.dc1 127.0.0.1 -11:39.077 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16195 [Follower]" leader= -11:39.087 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bcbf3e09-03b0-bd8b-3787-263ccac1ef4e 127.0.0.1 -2021-10-31T23:11:39.087Z [INFO] agent.router: Initializing LAN area manager -11:39.087 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16190 network=udp -11:39.088 [INFO] TestAgent.server: Adding LAN server: server="Node-bcbf3e09-03b0-bd8b-3787-263ccac1ef4e (Addr: tcp/127.0.0.1:16195) (DC: dc1)" -11:39.088 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bcbf3e09-03b0-bd8b-3787-263ccac1ef4e.dc1 area=wan -11:39.088 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16190 network=tcp -11:39.089 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16191 network=tcp -11:39.089 [INFO] TestAgent: started state syncer -11:39.089 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16196 network=tcp -11:39.142 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -11:39.145 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:39.145 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16195 [Candidate]" term=2 -11:39.146 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:39.146 [DEBUG] TestAgent.server.raft: vote granted: from=bcbf3e09-03b0-bd8b-3787-263ccac1ef4e term=2 tally=1 -11:39.146 [INFO] TestAgent.server.raft: election won: tally=1 -11:39.146 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16195 [Leader]" -11:39.146 [INFO] TestAgent.server: cluster leadership acquired -11:39.146 [INFO] TestAgent.server: initializing acls -11:39.146 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:39.146 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:39.147 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:39.147 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:39.147 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:39.147 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:39.147 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-bcbf3e09-03b0-bd8b-3787-263ccac1ef4e -11:39.147 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-bcbf3e09-03b0-bd8b-3787-263ccac1ef4e.dc1 -11:39.148 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:39.149 [INFO] TestAgent.server: New leader elected: payload=Node-bcbf3e09-03b0-bd8b-3787-263ccac1ef4e -11:39.149 [INFO] TestAgent.server: Updating LAN server: server="Node-bcbf3e09-03b0-bd8b-3787-263ccac1ef4e (Addr: tcp/127.0.0.1:16195) (DC: dc1)" -11:39.149 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-bcbf3e09-03b0-bd8b-3787-263ccac1ef4e.dc1 area=wan -11:39.150 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:39.150 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:39.150 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:39.150 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:39.150 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:39.150 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bcbf3e09-03b0-bd8b-3787-263ccac1ef4e -11:39.150 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bcbf3e09-03b0-bd8b-3787-263ccac1ef4e -11:39.151 [INFO] TestAgent.server: federation state anti-entropy synced -11:39.151 [INFO] TestAgent: Synced node info -11:39.151 [DEBUG] TestAgent: Node info in sync -11:39.151 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bcbf3e09-03b0-bd8b-3787-263ccac1ef4e -11:39.322 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bcbf3e09-03b0-bd8b-3787-263ccac1ef4e -11:39.322 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bcbf3e09-03b0-bd8b-3787-263ccac1ef4e -11:39.323 [DEBUG] TestAgent: Node info in sync -11:39.324 [INFO] TestAgent: Synced service: service=test -11:39.324 [DEBUG] TestAgent: Check in sync: check=service:test:1 -11:39.324 [DEBUG] TestAgent: Check in sync: check=service:test:2 -11:39.324 [DEBUG] TestAgent: Check in sync: check=service:test:3 -11:39.324 [INFO] TestAgent: Requesting shutdown -11:39.324 [INFO] TestAgent.server: shutting down server -11:39.324 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:39.324 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:39.324 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:39.324 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:39.324 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:39.324 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:39.324 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:39.324 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:39.324 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:39.324 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:39.324 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:39.324 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:39.324 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:39.324 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:39.324 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -11:39.324 [INFO] TestAgent: consul server down -11:39.324 [INFO] TestAgent: shutdown complete -11:39.324 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16190 network=tcp -2021-10-31T23:11:39.324Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:39.324Z [INFO] agent.router.manager: shutting down -11:39.324 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16190 network=udp -11:39.324 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16191 network=tcp -11:39.825 [INFO] TestAgent: Waiting for endpoints to shut down -11:39.825 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:39.039Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:39.039Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:39.060Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:39.060Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:39.079 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4b1947ed-11bb-5c76-d48a-97fcc2582cae Address:127.0.0.1:16202}]" -11:39.079 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16202 [Follower]" leader= -11:39.080 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4b1947ed-11bb-5c76-d48a-97fcc2582cae.dc1 127.0.0.1 -11:39.087 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4b1947ed-11bb-5c76-d48a-97fcc2582cae 127.0.0.1 -2021-10-31T23:11:39.088Z [INFO] agent.router: Initializing LAN area manager -11:39.088 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16197 network=udp -11:39.089 [INFO] TestAgent.server: Adding LAN server: server="Node-4b1947ed-11bb-5c76-d48a-97fcc2582cae (Addr: tcp/127.0.0.1:16202) (DC: dc1)" -11:39.089 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4b1947ed-11bb-5c76-d48a-97fcc2582cae.dc1 area=wan -11:39.089 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16197 network=tcp -11:39.093 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16198 network=tcp -11:39.093 [INFO] TestAgent: started state syncer -11:39.093 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16203 network=tcp -11:39.141 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -11:39.149 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:39.149 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16202 [Candidate]" term=2 -11:39.149 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:39.149 [DEBUG] TestAgent.server.raft: vote granted: from=4b1947ed-11bb-5c76-d48a-97fcc2582cae term=2 tally=1 -11:39.149 [INFO] TestAgent.server.raft: election won: tally=1 -11:39.149 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16202 [Leader]" -11:39.149 [INFO] TestAgent.server: cluster leadership acquired -11:39.150 [INFO] TestAgent.server: initializing acls -11:39.150 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:39.150 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:39.150 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:39.151 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:39.151 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:39.151 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:39.151 [INFO] TestAgent.server: New leader elected: payload=Node-4b1947ed-11bb-5c76-d48a-97fcc2582cae -11:39.151 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4b1947ed-11bb-5c76-d48a-97fcc2582cae -11:39.151 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4b1947ed-11bb-5c76-d48a-97fcc2582cae.dc1 -11:39.151 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4b1947ed-11bb-5c76-d48a-97fcc2582cae.dc1 area=wan -11:39.151 [INFO] TestAgent.server: Updating LAN server: server="Node-4b1947ed-11bb-5c76-d48a-97fcc2582cae (Addr: tcp/127.0.0.1:16202) (DC: dc1)" -11:39.152 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:39.152 [INFO] TestAgent: Synced node info -11:39.153 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:39.153 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:39.153 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:39.153 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:39.153 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:39.153 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4b1947ed-11bb-5c76-d48a-97fcc2582cae -11:39.153 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4b1947ed-11bb-5c76-d48a-97fcc2582cae -11:39.154 [INFO] TestAgent.server: federation state anti-entropy synced -11:39.154 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4b1947ed-11bb-5c76-d48a-97fcc2582cae -11:39.569 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4b1947ed-11bb-5c76-d48a-97fcc2582cae -11:39.570 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4b1947ed-11bb-5c76-d48a-97fcc2582cae -11:39.571 [DEBUG] TestAgent: Node info in sync -11:39.571 [INFO] TestAgent: Synced service: service=test -11:39.571 [DEBUG] TestAgent: Check in sync: check=service:test:1 -11:39.571 [DEBUG] TestAgent: Check in sync: check=service:test:2 -11:39.571 [DEBUG] TestAgent: Check in sync: check=service:test:3 -11:39.571 [INFO] TestAgent: Requesting shutdown -11:39.571 [INFO] TestAgent.server: shutting down server -11:39.571 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:39.571 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:39.571 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:39.571 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:39.571 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:39.571 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:39.571 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:39.572 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:39.572 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:39.572 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:39.572 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:39.572 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:39.572 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:39.572 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:39.572 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:39.572Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:39.572Z [INFO] agent.router.manager: shutting down -11:39.572 [INFO] TestAgent: consul server down -11:39.572 [INFO] TestAgent: shutdown complete -11:39.572 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16197 network=tcp -11:39.572 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16197 network=udp -11:39.572 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16198 network=tcp -11:40.073 [INFO] TestAgent: Waiting for endpoints to shut down -11:40.073 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_RegisterService_ACLDeny/normal/no_token +=== RUN TestAgent_RegisterService_ACLDeny/normal/root_token +2022-12-04T05:47:56.643Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:47:56.643Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:56.667Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:47:56.667Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:56.669 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bfcbb5c1-3c58-5339-649c-a8f6f82ca638 Address:127.0.0.1:23349}]" +47:56.669 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23349 [Follower]" leader= +47:56.669 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bfcbb5c1-3c58-5339-649c-a8f6f82ca638.dc1 127.0.0.1 +47:56.670 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bfcbb5c1-3c58-5339-649c-a8f6f82ca638 127.0.0.1 +2022-12-04T05:47:56.670Z [INFO] agent.router: Initializing LAN area manager +47:56.670 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23344 network=udp +47:56.670 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bfcbb5c1-3c58-5339-649c-a8f6f82ca638.dc1 area=wan +47:56.670 [INFO] TestAgent.server: Adding LAN server: server="Node-bfcbb5c1-3c58-5339-649c-a8f6f82ca638 (Addr: tcp/127.0.0.1:23349) (DC: dc1)" +47:56.670 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23344 network=tcp +47:56.671 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23345 network=tcp +47:56.671 [INFO] TestAgent: started state syncer +47:56.671 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23350 network=tcp +47:56.720 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +47:56.721 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:56.721 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23349 [Candidate]" term=2 +47:56.721 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:56.721 [DEBUG] TestAgent.server.raft: vote granted: from=bfcbb5c1-3c58-5339-649c-a8f6f82ca638 term=2 tally=1 +47:56.721 [INFO] TestAgent.server.raft: election won: tally=1 +47:56.721 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23349 [Leader]" +47:56.721 [INFO] TestAgent.server: cluster leadership acquired +47:56.721 [INFO] TestAgent.server: initializing acls +47:56.721 [INFO] TestAgent.server: Created ACL 'global-management' policy +47:56.721 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +47:56.722 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +47:56.722 [INFO] TestAgent.server: Created ACL anonymous token from configuration +47:56.722 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +47:56.722 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +47:56.722 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-bfcbb5c1-3c58-5339-649c-a8f6f82ca638 +47:56.722 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-bfcbb5c1-3c58-5339-649c-a8f6f82ca638.dc1 +47:56.723 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:56.724 [INFO] TestAgent.server: New leader elected: payload=Node-bfcbb5c1-3c58-5339-649c-a8f6f82ca638 +47:56.724 [INFO] TestAgent.server: Updating LAN server: server="Node-bfcbb5c1-3c58-5339-649c-a8f6f82ca638 (Addr: tcp/127.0.0.1:23349) (DC: dc1)" +47:56.724 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-bfcbb5c1-3c58-5339-649c-a8f6f82ca638.dc1 area=wan +47:56.728 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:56.728 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:56.728 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:56.728 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:56.728 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:56.728 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bfcbb5c1-3c58-5339-649c-a8f6f82ca638 +47:56.728 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bfcbb5c1-3c58-5339-649c-a8f6f82ca638 +47:56.729 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bfcbb5c1-3c58-5339-649c-a8f6f82ca638 +47:56.729 [INFO] TestAgent.server: federation state anti-entropy synced +47:56.745 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:56.746 [INFO] TestAgent: Synced node info +47:56.746 [DEBUG] TestAgent: Node info in sync +47:56.960 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bfcbb5c1-3c58-5339-649c-a8f6f82ca638 +47:56.960 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bfcbb5c1-3c58-5339-649c-a8f6f82ca638 +47:56.961 [DEBUG] TestAgent: Node info in sync +47:56.962 [INFO] TestAgent: Synced service: service=test +47:56.962 [DEBUG] TestAgent: Check in sync: check=service:test:3 +47:56.962 [DEBUG] TestAgent: Check in sync: check=service:test:1 +47:56.962 [DEBUG] TestAgent: Check in sync: check=service:test:2 +47:56.962 [INFO] TestAgent: Requesting shutdown +47:56.962 [INFO] TestAgent.server: shutting down server +47:56.962 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +47:56.962 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +47:56.962 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:56.962 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:56.962 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:56.962 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:56.962 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:56.962 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +47:56.962 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:56.962 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:56.962 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:56.962 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:56.962 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:56.962 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +47:56.962 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +2022-12-04T05:47:56.962Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:56.962Z [INFO] agent.router.manager: shutting down +47:56.963 [INFO] TestAgent: consul server down +47:56.963 [INFO] TestAgent: shutdown complete +47:56.963 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23344 network=tcp +47:56.963 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23344 network=udp +47:56.963 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23345 network=tcp +47:57.466 [INFO] TestAgent: Waiting for endpoints to shut down +47:57.466 [INFO] TestAgent: Endpoints down +2022-12-04T05:47:56.643Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:47:56.643Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:56.662Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:47:56.662Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:56.664 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3ef3c879-0490-e5dc-23c8-3a31ee489530 Address:127.0.0.1:23342}]" +47:56.664 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23342 [Follower]" leader= +47:56.664 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3ef3c879-0490-e5dc-23c8-3a31ee489530.dc1 127.0.0.1 +47:56.665 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3ef3c879-0490-e5dc-23c8-3a31ee489530 127.0.0.1 +2022-12-04T05:47:56.665Z [INFO] agent.router: Initializing LAN area manager +47:56.665 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23337 network=udp +47:56.665 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3ef3c879-0490-e5dc-23c8-3a31ee489530.dc1 area=wan +47:56.665 [INFO] TestAgent.server: Adding LAN server: server="Node-3ef3c879-0490-e5dc-23c8-3a31ee489530 (Addr: tcp/127.0.0.1:23342) (DC: dc1)" +47:56.665 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23337 network=tcp +47:56.666 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23338 network=tcp +47:56.666 [INFO] TestAgent: started state syncer +47:56.666 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23343 network=tcp +47:56.716 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +47:56.724 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:56.724 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23342 [Candidate]" term=2 +47:56.724 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:56.724 [DEBUG] TestAgent.server.raft: vote granted: from=3ef3c879-0490-e5dc-23c8-3a31ee489530 term=2 tally=1 +47:56.724 [INFO] TestAgent.server.raft: election won: tally=1 +47:56.724 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23342 [Leader]" +47:56.724 [INFO] TestAgent.server: cluster leadership acquired +47:56.724 [INFO] TestAgent.server: initializing acls +47:56.725 [INFO] TestAgent.server: Created ACL 'global-management' policy +47:56.725 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +47:56.725 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +47:56.725 [INFO] TestAgent.server: Created ACL anonymous token from configuration +47:56.725 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +47:56.725 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +47:56.725 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-3ef3c879-0490-e5dc-23c8-3a31ee489530 +47:56.725 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-3ef3c879-0490-e5dc-23c8-3a31ee489530.dc1 +47:56.726 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:56.727 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:56.727 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:56.727 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:56.727 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:56.727 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:56.727 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3ef3c879-0490-e5dc-23c8-3a31ee489530 +47:56.727 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3ef3c879-0490-e5dc-23c8-3a31ee489530 +47:56.728 [INFO] TestAgent.server: New leader elected: payload=Node-3ef3c879-0490-e5dc-23c8-3a31ee489530 +47:56.728 [INFO] TestAgent.server: Updating LAN server: server="Node-3ef3c879-0490-e5dc-23c8-3a31ee489530 (Addr: tcp/127.0.0.1:23342) (DC: dc1)" +47:56.728 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-3ef3c879-0490-e5dc-23c8-3a31ee489530.dc1 area=wan +47:56.728 [INFO] TestAgent.server: federation state anti-entropy synced +47:56.736 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3ef3c879-0490-e5dc-23c8-3a31ee489530 +47:56.925 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:56.925 [INFO] TestAgent: Synced node info +47:57.060 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-3ef3c879-0490-e5dc-23c8-3a31ee489530 +47:57.060 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-3ef3c879-0490-e5dc-23c8-3a31ee489530 +47:57.061 [DEBUG] TestAgent: Node info in sync +47:57.076 [INFO] TestAgent: Synced service: service=test +47:57.076 [DEBUG] TestAgent: Check in sync: check=service:test:2 +47:57.076 [DEBUG] TestAgent: Check in sync: check=service:test:3 +47:57.076 [DEBUG] TestAgent: Check in sync: check=service:test:1 +47:57.076 [INFO] TestAgent: Requesting shutdown +47:57.076 [INFO] TestAgent.server: shutting down server +47:57.076 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:57.076 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:57.076 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:57.076 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +47:57.076 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +47:57.076 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:57.076 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:57.076 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:57.076 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:57.076 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:57.076 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:57.076 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:57.076 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +47:57.076 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +47:57.076 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:57.076Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:57.076Z [INFO] agent.router.manager: shutting down +47:57.076 [INFO] TestAgent: consul server down +47:57.076 [INFO] TestAgent: shutdown complete +47:57.076 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23337 network=tcp +47:57.077 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23337 network=udp +47:57.077 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23338 network=tcp +47:57.577 [INFO] TestAgent: Waiting for endpoints to shut down +47:57.577 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_ACLDeny (0.00s) - --- PASS: TestAgent_RegisterService_ACLDeny/normal (0.84s) - --- PASS: TestAgent_RegisterService_ACLDeny/normal/no_token (0.00s) - --- PASS: TestAgent_RegisterService_ACLDeny/normal/root_token (0.00s) - --- PASS: TestAgent_RegisterService_ACLDeny/service_manager (1.08s) + --- PASS: TestAgent_RegisterService_ACLDeny/service_manager (0.84s) --- PASS: TestAgent_RegisterService_ACLDeny/service_manager/no_token (0.00s) --- PASS: TestAgent_RegisterService_ACLDeny/service_manager/root_token (0.00s) + --- PASS: TestAgent_RegisterService_ACLDeny/normal (0.95s) + --- PASS: TestAgent_RegisterService_ACLDeny/normal/no_token (0.00s) + --- PASS: TestAgent_RegisterService_ACLDeny/normal/root_token (0.02s) === RUN TestAgent_RegisterService_InvalidAddress === RUN TestAgent_RegisterService_InvalidAddress/normal === PAUSE TestAgent_RegisterService_InvalidAddress/normal @@ -6693,124 +6735,124 @@ === RUN TestAgent_RegisterService_InvalidAddress/service_manager/addr_0.0.0.0 === RUN TestAgent_RegisterService_InvalidAddress/service_manager/addr_:: === RUN TestAgent_RegisterService_InvalidAddress/service_manager/addr_[::] -2021-10-31T23:11:40.112Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:40.136Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:40.138 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0a6c29ca-5011-2329-b2e1-04ac982ffcfa Address:127.0.0.1:16209}]" -11:40.138 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0a6c29ca-5011-2329-b2e1-04ac982ffcfa.dc1 127.0.0.1 -11:40.139 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0a6c29ca-5011-2329-b2e1-04ac982ffcfa 127.0.0.1 -2021-10-31T23:11:40.139Z [INFO] agent.router: Initializing LAN area manager -11:40.139 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16204 network=udp -11:40.139 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16209 [Follower]" leader= -11:40.139 [INFO] TestAgent.server: Adding LAN server: server="Node-0a6c29ca-5011-2329-b2e1-04ac982ffcfa (Addr: tcp/127.0.0.1:16209) (DC: dc1)" -11:40.139 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0a6c29ca-5011-2329-b2e1-04ac982ffcfa.dc1 area=wan -11:40.139 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16204 network=tcp -11:40.140 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16205 network=tcp -11:40.140 [INFO] TestAgent: started state syncer -11:40.140 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16210 network=tcp -11:40.185 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:40.185 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16209 [Candidate]" term=2 -11:40.185 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:40.185 [DEBUG] TestAgent.server.raft: vote granted: from=0a6c29ca-5011-2329-b2e1-04ac982ffcfa term=2 tally=1 -11:40.185 [INFO] TestAgent.server.raft: election won: tally=1 -11:40.185 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16209 [Leader]" -11:40.185 [INFO] TestAgent.server: cluster leadership acquired -11:40.185 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16209 -11:40.186 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:40.187 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:40.187 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:40.187 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:40.187 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:40.187 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:40.187 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a6c29ca-5011-2329-b2e1-04ac982ffcfa -11:40.187 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0a6c29ca-5011-2329-b2e1-04ac982ffcfa -11:40.188 [INFO] TestAgent.server: New leader elected: payload=Node-0a6c29ca-5011-2329-b2e1-04ac982ffcfa -11:40.188 [INFO] TestAgent.server: federation state anti-entropy synced -11:40.218 [INFO] TestAgent: Synced node info -11:40.219 [INFO] TestAgent: Synced service: service=connect-proxy -11:40.219 [INFO] TestAgent: Requesting shutdown -11:40.219 [INFO] TestAgent.server: shutting down server -11:40.219 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:40.219 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:40.219 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:40.219 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:40.219 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:40.219 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:40.219 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:40.219 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:40.219 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:40.219 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:40.219 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:40.219 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -11:40.220 [INFO] TestAgent: consul server down -11:40.220 [INFO] TestAgent: shutdown complete -11:40.220 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16204 network=tcp -11:40.220 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16204 network=udp -11:40.220 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16205 network=tcp -2021-10-31T23:11:40.220Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:40.220Z [INFO] agent.router.manager: shutting down -11:40.721 [INFO] TestAgent: Waiting for endpoints to shut down -11:40.721 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:40.143Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:40.185Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:40.202 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:48c6b3ad-52f3-c3af-c025-24d12f9e0997 Address:127.0.0.1:16216}]" -11:40.203 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16216 [Follower]" leader= -11:40.203 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-48c6b3ad-52f3-c3af-c025-24d12f9e0997.dc1 127.0.0.1 -11:40.204 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-48c6b3ad-52f3-c3af-c025-24d12f9e0997 127.0.0.1 -2021-10-31T23:11:40.204Z [INFO] agent.router: Initializing LAN area manager -11:40.204 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16211 network=udp -11:40.207 [INFO] TestAgent.server: Adding LAN server: server="Node-48c6b3ad-52f3-c3af-c025-24d12f9e0997 (Addr: tcp/127.0.0.1:16216) (DC: dc1)" -11:40.216 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-48c6b3ad-52f3-c3af-c025-24d12f9e0997.dc1 area=wan -11:40.216 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16211 network=tcp -11:40.220 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16212 network=tcp -11:40.220 [INFO] TestAgent: started state syncer -11:40.221 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16217 network=tcp -11:40.246 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:40.246 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16216 [Candidate]" term=2 -11:40.246 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:40.246 [DEBUG] TestAgent.server.raft: vote granted: from=48c6b3ad-52f3-c3af-c025-24d12f9e0997 term=2 tally=1 -11:40.246 [INFO] TestAgent.server.raft: election won: tally=1 -11:40.246 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16216 [Leader]" -11:40.246 [INFO] TestAgent.server: cluster leadership acquired -11:40.246 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16216 -11:40.247 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:40.248 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:40.248 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:40.248 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:40.248 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:40.248 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:40.248 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-48c6b3ad-52f3-c3af-c025-24d12f9e0997 -11:40.248 [INFO] TestAgent.server: member joined, marking health alive: member=Node-48c6b3ad-52f3-c3af-c025-24d12f9e0997 -11:40.249 [INFO] TestAgent.server: New leader elected: payload=Node-48c6b3ad-52f3-c3af-c025-24d12f9e0997 -11:40.249 [INFO] TestAgent.server: federation state anti-entropy synced -11:40.258 [INFO] TestAgent: Requesting shutdown -11:40.258 [INFO] TestAgent.server: shutting down server -11:40.258 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:40.258 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:40.258 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:40.258 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:40.258 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:40.259 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:40.259 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:40.259 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:40.259 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:40.259 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:40.259 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:40.259 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:40.259Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:40.259Z [INFO] agent.router.manager: shutting down -11:40.259 [INFO] TestAgent: consul server down -11:40.259 [INFO] TestAgent: shutdown complete -11:40.259 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16211 network=tcp -11:40.259 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16211 network=udp -11:40.259 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16212 network=tcp -11:40.761 [INFO] TestAgent: Waiting for endpoints to shut down -11:40.761 [INFO] TestAgent: Endpoints down +2022-12-04T05:47:57.602Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:57.643Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:57.645 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:757c1b31-63ee-a936-c305-da0d9d465d72 Address:127.0.0.1:23363}]" +47:57.645 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-757c1b31-63ee-a936-c305-da0d9d465d72.dc1 127.0.0.1 +47:57.646 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-757c1b31-63ee-a936-c305-da0d9d465d72 127.0.0.1 +2022-12-04T05:47:57.646Z [INFO] agent.router: Initializing LAN area manager +47:57.646 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23358 network=udp +47:57.646 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23363 [Follower]" leader= +47:57.646 [INFO] TestAgent.server: Adding LAN server: server="Node-757c1b31-63ee-a936-c305-da0d9d465d72 (Addr: tcp/127.0.0.1:23363) (DC: dc1)" +47:57.646 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-757c1b31-63ee-a936-c305-da0d9d465d72.dc1 area=wan +47:57.646 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23358 network=tcp +47:57.647 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23359 network=tcp +47:57.647 [INFO] TestAgent: started state syncer +47:57.647 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23364 network=tcp +47:57.712 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:57.712 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23363 [Candidate]" term=2 +47:57.712 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:57.712 [DEBUG] TestAgent.server.raft: vote granted: from=757c1b31-63ee-a936-c305-da0d9d465d72 term=2 tally=1 +47:57.712 [INFO] TestAgent.server.raft: election won: tally=1 +47:57.712 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23363 [Leader]" +47:57.712 [INFO] TestAgent.server: cluster leadership acquired +47:57.713 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23363 +47:57.713 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:57.714 [INFO] TestAgent.server: New leader elected: payload=Node-757c1b31-63ee-a936-c305-da0d9d465d72 +47:57.715 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:57.715 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:57.715 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:57.715 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:57.715 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:57.715 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-757c1b31-63ee-a936-c305-da0d9d465d72 +47:57.715 [INFO] TestAgent.server: member joined, marking health alive: member=Node-757c1b31-63ee-a936-c305-da0d9d465d72 +47:57.716 [INFO] TestAgent.server: federation state anti-entropy synced +47:57.754 [INFO] TestAgent: Requesting shutdown +47:57.754 [INFO] TestAgent.server: shutting down server +47:57.754 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:57.754 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:57.754 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:57.754 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:57.754 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:57.754 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:57.754 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:57.754 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +47:57.754 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:57.754 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:57.754 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:57.754 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:57.754Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:57.754Z [INFO] agent.router.manager: shutting down +47:57.755 [INFO] TestAgent: consul server down +47:57.755 [INFO] TestAgent: shutdown complete +47:57.755 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23358 network=tcp +47:57.755 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23358 network=udp +47:57.755 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23359 network=tcp +47:58.256 [INFO] TestAgent: Waiting for endpoints to shut down +47:58.256 [INFO] TestAgent: Endpoints down +2022-12-04T05:47:57.593Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:57.612Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:57.636 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7b97d4d7-d6ec-0141-3eee-9576e26a7bea Address:127.0.0.1:23356}]" +47:57.637 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7b97d4d7-d6ec-0141-3eee-9576e26a7bea.dc1 127.0.0.1 +47:57.638 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7b97d4d7-d6ec-0141-3eee-9576e26a7bea 127.0.0.1 +2022-12-04T05:47:57.638Z [INFO] agent.router: Initializing LAN area manager +47:57.638 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23351 network=udp +47:57.638 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23356 [Follower]" leader= +47:57.638 [INFO] TestAgent.server: Adding LAN server: server="Node-7b97d4d7-d6ec-0141-3eee-9576e26a7bea (Addr: tcp/127.0.0.1:23356) (DC: dc1)" +47:57.639 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7b97d4d7-d6ec-0141-3eee-9576e26a7bea.dc1 area=wan +47:57.639 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23351 network=tcp +47:57.639 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23352 network=tcp +47:57.639 [INFO] TestAgent: started state syncer +47:57.639 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23357 network=tcp +47:57.708 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:57.708 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23356 [Candidate]" term=2 +47:57.708 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:57.708 [DEBUG] TestAgent.server.raft: vote granted: from=7b97d4d7-d6ec-0141-3eee-9576e26a7bea term=2 tally=1 +47:57.708 [INFO] TestAgent.server.raft: election won: tally=1 +47:57.708 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23356 [Leader]" +47:57.708 [INFO] TestAgent.server: cluster leadership acquired +47:57.708 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23356 +47:57.709 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:57.711 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:57.711 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:57.711 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:57.711 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:57.711 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:57.711 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7b97d4d7-d6ec-0141-3eee-9576e26a7bea +47:57.711 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7b97d4d7-d6ec-0141-3eee-9576e26a7bea +47:57.712 [INFO] TestAgent.server: New leader elected: payload=Node-7b97d4d7-d6ec-0141-3eee-9576e26a7bea +47:57.714 [INFO] TestAgent.server: federation state anti-entropy synced +47:57.989 [INFO] TestAgent: Synced node info +47:57.990 [INFO] TestAgent: Synced service: service=connect-proxy +47:57.990 [INFO] TestAgent: Requesting shutdown +47:57.990 [INFO] TestAgent.server: shutting down server +47:57.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:57.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:57.990 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:57.990 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:57.990 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:57.990 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:57.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:57.990 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +47:57.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:57.990 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:57.990 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:57.990 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:57.990Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:57.990Z [INFO] agent.router.manager: shutting down +47:57.991 [INFO] TestAgent: consul server down +47:57.991 [INFO] TestAgent: shutdown complete +47:57.991 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23351 network=tcp +47:57.991 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23351 network=udp +47:57.991 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23352 network=tcp +47:58.492 [INFO] TestAgent: Waiting for endpoints to shut down +47:58.492 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_InvalidAddress (0.00s) - --- PASS: TestAgent_RegisterService_InvalidAddress/normal (0.65s) --- PASS: TestAgent_RegisterService_InvalidAddress/service_manager (0.68s) --- PASS: TestAgent_RegisterService_InvalidAddress/service_manager/addr_0.0.0.0 (0.00s) --- PASS: TestAgent_RegisterService_InvalidAddress/service_manager/addr_:: (0.00s) --- PASS: TestAgent_RegisterService_InvalidAddress/service_manager/addr_[::] (0.00s) + --- PASS: TestAgent_RegisterService_InvalidAddress/normal (0.91s) === RUN TestAgent_RegisterService_UnmanagedConnectProxy === RUN TestAgent_RegisterService_UnmanagedConnectProxy/normal === PAUSE TestAgent_RegisterService_UnmanagedConnectProxy/normal @@ -6818,122 +6860,125 @@ === PAUSE TestAgent_RegisterService_UnmanagedConnectProxy/service_manager === CONT TestAgent_RegisterService_UnmanagedConnectProxy/normal === CONT TestAgent_RegisterService_UnmanagedConnectProxy/service_manager -2021-10-31T23:11:40.786Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:40.843Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:40.845 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8238b220-3934-563a-83c1-bfdbc70a7b89 Address:127.0.0.1:16223}]" -11:40.845 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16223 [Follower]" leader= -11:40.847 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8238b220-3934-563a-83c1-bfdbc70a7b89.dc1 127.0.0.1 -11:40.848 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8238b220-3934-563a-83c1-bfdbc70a7b89 127.0.0.1 -2021-10-31T23:11:40.848Z [INFO] agent.router: Initializing LAN area manager -11:40.848 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16218 network=udp -11:40.848 [INFO] TestAgent.server: Adding LAN server: server="Node-8238b220-3934-563a-83c1-bfdbc70a7b89 (Addr: tcp/127.0.0.1:16223) (DC: dc1)" -11:40.848 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8238b220-3934-563a-83c1-bfdbc70a7b89.dc1 area=wan -11:40.848 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16218 network=tcp -11:40.849 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16219 network=tcp -11:40.849 [INFO] TestAgent: started state syncer -11:40.849 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16224 network=tcp -11:40.917 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:40.917 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16223 [Candidate]" term=2 -11:40.918 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:40.918 [DEBUG] TestAgent.server.raft: vote granted: from=8238b220-3934-563a-83c1-bfdbc70a7b89 term=2 tally=1 -11:40.918 [INFO] TestAgent.server.raft: election won: tally=1 -11:40.918 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16223 [Leader]" -11:40.918 [INFO] TestAgent.server: cluster leadership acquired -11:40.918 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16223 -11:40.918 [INFO] TestAgent.server: New leader elected: payload=Node-8238b220-3934-563a-83c1-bfdbc70a7b89 -11:40.922 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:40.923 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:40.923 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:40.923 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:40.923 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:40.923 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:40.923 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8238b220-3934-563a-83c1-bfdbc70a7b89 -11:40.923 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8238b220-3934-563a-83c1-bfdbc70a7b89 -11:40.924 [INFO] TestAgent.server: federation state anti-entropy synced -11:41.040 [INFO] TestAgent: Synced node info -11:41.040 [INFO] TestAgent: Synced service: service=connect-proxy -11:41.040 [INFO] TestAgent: Requesting shutdown -11:41.040 [INFO] TestAgent.server: shutting down server -11:41.040 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:41.040 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:41.040 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:41.040 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:41.040 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:41.040 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:41.040 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:41.040 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:41.040 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:41.040 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:41.040 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:41.040 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:41.041Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:41.041Z [INFO] agent.router.manager: shutting down -11:41.041 [INFO] TestAgent: consul server down -11:41.041 [INFO] TestAgent: shutdown complete -11:41.041 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16218 network=tcp -11:41.041 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16218 network=udp -11:41.041 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16219 network=tcp -11:41.541 [INFO] TestAgent: Waiting for endpoints to shut down -11:41.541 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:40.789Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:40.848Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:40.858 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6559f1ea-5413-bbdb-ba7e-b095ffffb844 Address:127.0.0.1:16230}]" -11:40.859 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6559f1ea-5413-bbdb-ba7e-b095ffffb844.dc1 127.0.0.1 -11:40.859 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6559f1ea-5413-bbdb-ba7e-b095ffffb844 127.0.0.1 -2021-10-31T23:11:40.859Z [INFO] agent.router: Initializing LAN area manager -11:40.860 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16225 network=udp -11:40.860 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16230 [Follower]" leader= -11:40.860 [INFO] TestAgent.server: Adding LAN server: server="Node-6559f1ea-5413-bbdb-ba7e-b095ffffb844 (Addr: tcp/127.0.0.1:16230) (DC: dc1)" -11:40.860 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6559f1ea-5413-bbdb-ba7e-b095ffffb844.dc1 area=wan -11:40.860 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16225 network=tcp -11:40.861 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16226 network=tcp -11:40.861 [INFO] TestAgent: started state syncer -11:40.861 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16231 network=tcp -11:40.917 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:40.917 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16230 [Candidate]" term=2 -11:40.918 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:40.918 [DEBUG] TestAgent.server.raft: vote granted: from=6559f1ea-5413-bbdb-ba7e-b095ffffb844 term=2 tally=1 -11:40.918 [INFO] TestAgent.server.raft: election won: tally=1 -11:40.918 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16230 [Leader]" -11:40.918 [INFO] TestAgent.server: cluster leadership acquired -11:40.918 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16230 -11:40.918 [INFO] TestAgent.server: New leader elected: payload=Node-6559f1ea-5413-bbdb-ba7e-b095ffffb844 -11:40.919 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:40.920 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:40.920 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:40.920 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:40.920 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:40.920 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:40.920 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6559f1ea-5413-bbdb-ba7e-b095ffffb844 -11:40.920 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6559f1ea-5413-bbdb-ba7e-b095ffffb844 -11:40.921 [INFO] TestAgent.server: federation state anti-entropy synced -11:41.239 [INFO] TestAgent: Synced node info -11:41.239 [INFO] TestAgent: Synced service: service=connect-proxy -11:41.239 [INFO] TestAgent: Requesting shutdown -11:41.239 [INFO] TestAgent.server: shutting down server -11:41.239 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:41.239 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:41.239 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:41.239 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:41.239 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:41.239 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:41.239 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:41.239 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:41.239 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:41.239 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:41.239 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:41.240 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:41.240Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:41.240Z [INFO] agent.router.manager: shutting down -11:41.240 [INFO] TestAgent: consul server down -11:41.240 [INFO] TestAgent: shutdown complete -11:41.240 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16225 network=tcp -11:41.240 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16225 network=udp -11:41.240 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16226 network=tcp -11:41.740 [INFO] TestAgent: Waiting for endpoints to shut down -11:41.740 [INFO] TestAgent: Endpoints down +2022-12-04T05:47:58.534Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:58.544Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:58.546 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b9568321-390a-6aa3-15bb-6445a443bc99 Address:127.0.0.1:23370}]" +47:58.546 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b9568321-390a-6aa3-15bb-6445a443bc99.dc1 127.0.0.1 +47:58.546 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23370 [Follower]" leader= +47:58.547 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b9568321-390a-6aa3-15bb-6445a443bc99 127.0.0.1 +2022-12-04T05:47:58.547Z [INFO] agent.router: Initializing LAN area manager +47:58.547 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23365 network=udp +47:58.547 [INFO] TestAgent.server: Adding LAN server: server="Node-b9568321-390a-6aa3-15bb-6445a443bc99 (Addr: tcp/127.0.0.1:23370) (DC: dc1)" +47:58.547 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b9568321-390a-6aa3-15bb-6445a443bc99.dc1 area=wan +47:58.547 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23365 network=tcp +47:58.548 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23366 network=tcp +47:58.548 [INFO] TestAgent: started state syncer +47:58.548 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23371 network=tcp +47:58.588 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:58.588 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23370 [Candidate]" term=2 +47:58.589 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:58.589 [DEBUG] TestAgent.server.raft: vote granted: from=b9568321-390a-6aa3-15bb-6445a443bc99 term=2 tally=1 +47:58.589 [INFO] TestAgent.server.raft: election won: tally=1 +47:58.589 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23370 [Leader]" +47:58.589 [INFO] TestAgent.server: cluster leadership acquired +47:58.589 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23370 +47:58.589 [INFO] TestAgent.server: New leader elected: payload=Node-b9568321-390a-6aa3-15bb-6445a443bc99 +47:58.590 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:58.591 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:58.591 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:58.591 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:58.591 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:58.591 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:58.592 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b9568321-390a-6aa3-15bb-6445a443bc99 +47:58.592 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b9568321-390a-6aa3-15bb-6445a443bc99 +47:58.592 [INFO] TestAgent.server: federation state anti-entropy synced +47:58.832 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:58.833 [INFO] TestAgent: Synced node info +47:58.905 [DEBUG] TestAgent: Node info in sync +47:58.905 [INFO] TestAgent: Synced service: service=connect-proxy +47:58.905 [INFO] TestAgent: Requesting shutdown +47:58.905 [INFO] TestAgent.server: shutting down server +47:58.905 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:58.905 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:58.905 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:58.905 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:58.905 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:58.906 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:58.906 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:58.906 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:58.906 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:58.906 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:58.906 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:58.906Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:58.906Z [INFO] agent.router.manager: shutting down +47:58.906 [INFO] TestAgent: consul server down +47:58.906 [INFO] TestAgent: shutdown complete +47:58.906 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23365 network=tcp +47:58.906 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23365 network=udp +47:58.906 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23366 network=tcp +47:59.412 [INFO] TestAgent: Waiting for endpoints to shut down +47:59.412 [INFO] TestAgent: Endpoints down +2022-12-04T05:47:58.542Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:58.560Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:58.563 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:355a32b9-c5f6-8098-b1f3-827fe8c9ca7c Address:127.0.0.1:23377}]" +47:58.563 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23377 [Follower]" leader= +47:58.564 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-355a32b9-c5f6-8098-b1f3-827fe8c9ca7c.dc1 127.0.0.1 +47:58.565 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-355a32b9-c5f6-8098-b1f3-827fe8c9ca7c 127.0.0.1 +2022-12-04T05:47:58.565Z [INFO] agent.router: Initializing LAN area manager +47:58.565 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23372 network=udp +47:58.565 [INFO] TestAgent.server: Adding LAN server: server="Node-355a32b9-c5f6-8098-b1f3-827fe8c9ca7c (Addr: tcp/127.0.0.1:23377) (DC: dc1)" +47:58.565 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-355a32b9-c5f6-8098-b1f3-827fe8c9ca7c.dc1 area=wan +47:58.566 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23372 network=tcp +47:58.566 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23373 network=tcp +47:58.566 [INFO] TestAgent: started state syncer +47:58.566 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23378 network=tcp +47:58.631 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:58.631 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23377 [Candidate]" term=2 +47:58.632 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:58.632 [DEBUG] TestAgent.server.raft: vote granted: from=355a32b9-c5f6-8098-b1f3-827fe8c9ca7c term=2 tally=1 +47:58.632 [INFO] TestAgent.server.raft: election won: tally=1 +47:58.632 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23377 [Leader]" +47:58.632 [INFO] TestAgent.server: cluster leadership acquired +47:58.632 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23377 +47:58.632 [INFO] TestAgent.server: New leader elected: payload=Node-355a32b9-c5f6-8098-b1f3-827fe8c9ca7c +47:58.633 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:58.634 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:58.634 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:58.634 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:58.634 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:58.634 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:58.634 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-355a32b9-c5f6-8098-b1f3-827fe8c9ca7c +47:58.634 [INFO] TestAgent.server: member joined, marking health alive: member=Node-355a32b9-c5f6-8098-b1f3-827fe8c9ca7c +47:58.635 [INFO] TestAgent.server: federation state anti-entropy synced +47:58.920 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:58.920 [INFO] TestAgent: Synced node info +47:58.921 [DEBUG] TestAgent: Node info in sync +47:58.973 [DEBUG] TestAgent: Node info in sync +47:58.973 [INFO] TestAgent: Synced service: service=connect-proxy +47:58.973 [INFO] TestAgent: Requesting shutdown +47:58.973 [INFO] TestAgent.server: shutting down server +47:58.973 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:58.973 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:58.973 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:58.973 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:58.973 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:58.974 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:58.974 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:58.974 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:58.974 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:58.974 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:58.974 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:58.974Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:58.974Z [INFO] agent.router.manager: shutting down +47:58.974 [INFO] TestAgent: consul server down +47:58.974 [INFO] TestAgent: shutdown complete +47:58.974 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23372 network=tcp +47:58.974 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23372 network=udp +47:58.974 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23373 network=tcp +47:59.474 [INFO] TestAgent: Waiting for endpoints to shut down +47:59.474 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_UnmanagedConnectProxy (0.00s) - --- PASS: TestAgent_RegisterService_UnmanagedConnectProxy/normal (0.78s) + --- PASS: TestAgent_RegisterService_UnmanagedConnectProxy/normal (0.92s) --- PASS: TestAgent_RegisterService_UnmanagedConnectProxy/service_manager (0.98s) === RUN TestAgent_RegisterServiceDeregisterService_Sidecar === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal @@ -6944,2135 +6989,2124 @@ === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/sanity_check_no_sidecar_case === CONT TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/sanity_check_no_sidecar_case -2021-10-31T23:11:41.792Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:41.819Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:41.821 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fe8010d1-f99e-8779-4afd-76dc963c7176 Address:127.0.0.1:16237}]" -11:41.821 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16237 [Follower]" leader= -11:41.827 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fe8010d1-f99e-8779-4afd-76dc963c7176.dc1 127.0.0.1 -11:41.828 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fe8010d1-f99e-8779-4afd-76dc963c7176 127.0.0.1 -2021-10-31T23:11:41.828Z [INFO] agent.router: Initializing LAN area manager -11:41.828 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16232 network=udp -11:41.829 [INFO] TestAgent.server: Adding LAN server: server="Node-fe8010d1-f99e-8779-4afd-76dc963c7176 (Addr: tcp/127.0.0.1:16237) (DC: dc1)" -11:41.829 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fe8010d1-f99e-8779-4afd-76dc963c7176.dc1 area=wan -11:41.829 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16232 network=tcp -11:41.829 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16233 network=tcp -11:41.830 [INFO] TestAgent: started state syncer -11:41.830 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16238 network=tcp -11:41.877 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:41.877 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16237 [Candidate]" term=2 -11:41.878 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:41.878 [DEBUG] TestAgent.server.raft: vote granted: from=fe8010d1-f99e-8779-4afd-76dc963c7176 term=2 tally=1 -11:41.878 [INFO] TestAgent.server.raft: election won: tally=1 -11:41.878 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16237 [Leader]" -11:41.878 [INFO] TestAgent.server: cluster leadership acquired -11:41.878 [INFO] TestAgent.server: New leader elected: payload=Node-fe8010d1-f99e-8779-4afd-76dc963c7176 -11:41.878 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16237 -11:41.879 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:41.880 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:41.880 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:41.880 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:41.880 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:41.880 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:41.880 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe8010d1-f99e-8779-4afd-76dc963c7176 -11:41.880 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fe8010d1-f99e-8779-4afd-76dc963c7176 -11:41.881 [INFO] TestAgent.server: federation state anti-entropy synced -11:41.976 [INFO] TestAgent: Synced node info -11:41.976 [INFO] TestAgent: Synced service: service=web -11:41.976 [INFO] TestAgent: Requesting shutdown -11:41.976 [INFO] TestAgent.server: shutting down server -11:41.976 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:41.976 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:41.976 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:41.976 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:41.976 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:41.976 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:41.976 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:41.976 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:41.976 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:41.976 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:41.976 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:41.976 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:41.977Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:41.977Z [INFO] agent.router.manager: shutting down -11:41.977 [INFO] TestAgent: consul server down -11:41.977 [INFO] TestAgent: shutdown complete -11:41.977 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16232 network=tcp -11:41.977 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16232 network=udp -11:41.977 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16233 network=tcp -11:42.477 [INFO] TestAgent: Waiting for endpoints to shut down -11:42.477 [INFO] TestAgent: Endpoints down +2022-12-04T05:47:59.529Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:59.561Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:59.563 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2edb8879-9b31-b1c0-2f3e-289fa7b4cb8b Address:127.0.0.1:23384}]" +47:59.563 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23384 [Follower]" leader= +47:59.564 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2edb8879-9b31-b1c0-2f3e-289fa7b4cb8b.dc1 127.0.0.1 +47:59.575 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2edb8879-9b31-b1c0-2f3e-289fa7b4cb8b 127.0.0.1 +2022-12-04T05:47:59.575Z [INFO] agent.router: Initializing LAN area manager +47:59.576 [INFO] TestAgent.server: Adding LAN server: server="Node-2edb8879-9b31-b1c0-2f3e-289fa7b4cb8b (Addr: tcp/127.0.0.1:23384) (DC: dc1)" +47:59.576 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2edb8879-9b31-b1c0-2f3e-289fa7b4cb8b.dc1 area=wan +47:59.576 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23379 network=udp +47:59.576 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23379 network=tcp +47:59.577 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23380 network=tcp +47:59.577 [INFO] TestAgent: started state syncer +47:59.577 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23385 network=tcp +47:59.614 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:59.614 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23384 [Candidate]" term=2 +47:59.615 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:59.615 [DEBUG] TestAgent.server.raft: vote granted: from=2edb8879-9b31-b1c0-2f3e-289fa7b4cb8b term=2 tally=1 +47:59.615 [INFO] TestAgent.server.raft: election won: tally=1 +47:59.615 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23384 [Leader]" +47:59.615 [INFO] TestAgent.server: cluster leadership acquired +47:59.615 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23384 +47:59.616 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:59.617 [INFO] TestAgent.server: New leader elected: payload=Node-2edb8879-9b31-b1c0-2f3e-289fa7b4cb8b +47:59.617 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:59.617 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:59.617 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:59.617 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:59.617 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:59.617 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2edb8879-9b31-b1c0-2f3e-289fa7b4cb8b +47:59.617 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2edb8879-9b31-b1c0-2f3e-289fa7b4cb8b +47:59.618 [INFO] TestAgent.server: federation state anti-entropy synced +47:59.659 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:59.659 [INFO] TestAgent: Synced node info +47:59.659 [DEBUG] TestAgent: Node info in sync +47:59.745 [DEBUG] TestAgent: Node info in sync +47:59.745 [INFO] TestAgent: Synced service: service=web +47:59.745 [INFO] TestAgent: Requesting shutdown +47:59.746 [INFO] TestAgent.server: shutting down server +47:59.746 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:59.746 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:59.746 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:59.746 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:59.746 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:59.746 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:59.746 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:59.746 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:59.746 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:59.746 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:59.746 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:59.746Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:59.746Z [INFO] agent.router.manager: shutting down +47:59.746 [INFO] TestAgent: consul server down +47:59.746 [INFO] TestAgent: shutdown complete +47:59.746 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23379 network=tcp +47:59.746 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23379 network=udp +47:59.746 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23380 network=tcp +48:00.248 [INFO] TestAgent: Waiting for endpoints to shut down +48:00.248 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/default_sidecar -2021-10-31T23:11:41.798Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:41.825Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:41.832 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:66757a14-7a72-7d42-6b2d-6666bc9771be Address:127.0.0.1:16244}]" -11:41.832 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-66757a14-7a72-7d42-6b2d-6666bc9771be.dc1 127.0.0.1 -11:41.832 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16244 [Follower]" leader= -11:41.847 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-66757a14-7a72-7d42-6b2d-6666bc9771be 127.0.0.1 -2021-10-31T23:11:41.847Z [INFO] agent.router: Initializing LAN area manager -11:41.847 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16239 network=udp -11:41.847 [INFO] TestAgent.server: Adding LAN server: server="Node-66757a14-7a72-7d42-6b2d-6666bc9771be (Addr: tcp/127.0.0.1:16244) (DC: dc1)" -11:41.847 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-66757a14-7a72-7d42-6b2d-6666bc9771be.dc1 area=wan -11:41.848 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16239 network=tcp -11:41.849 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16240 network=tcp -11:41.849 [INFO] TestAgent: started state syncer -11:41.849 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16245 network=tcp -11:41.889 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:41.889 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16244 [Candidate]" term=2 -11:41.890 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:41.890 [DEBUG] TestAgent.server.raft: vote granted: from=66757a14-7a72-7d42-6b2d-6666bc9771be term=2 tally=1 -11:41.890 [INFO] TestAgent.server.raft: election won: tally=1 -11:41.890 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16244 [Leader]" -11:41.890 [INFO] TestAgent.server: cluster leadership acquired -11:41.890 [INFO] TestAgent.server: New leader elected: payload=Node-66757a14-7a72-7d42-6b2d-6666bc9771be -11:41.890 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16244 -11:41.891 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:41.892 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:41.892 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:41.892 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:41.892 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:41.892 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:41.892 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-66757a14-7a72-7d42-6b2d-6666bc9771be -11:41.892 [INFO] TestAgent.server: member joined, marking health alive: member=Node-66757a14-7a72-7d42-6b2d-6666bc9771be -11:41.893 [INFO] TestAgent.server: federation state anti-entropy synced -11:42.179 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:42.180 [INFO] TestAgent: Synced node info -11:42.233 [DEBUG] TestAgent: Node info in sync -11:42.233 [INFO] TestAgent: Synced service: service=web -11:42.233 [INFO] TestAgent: Requesting shutdown -11:42.233 [INFO] TestAgent.server: shutting down server -11:42.233 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:42.233 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:42.233 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:42.233 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:42.233 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:42.233 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:42.233 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:42.233 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:42.233 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:42.233 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:42.234 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:42.234Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:42.234Z [INFO] agent.router.manager: shutting down -11:42.234 [INFO] TestAgent: consul server down -11:42.234 [INFO] TestAgent: shutdown complete -11:42.234 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16239 network=tcp -11:42.234 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16239 network=udp -11:42.234 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16240 network=tcp -11:42.735 [INFO] TestAgent: Waiting for endpoints to shut down -11:42.735 [INFO] TestAgent: Endpoints down +2022-12-04T05:47:59.498Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:59.549Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:59.550 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dad0627e-978b-2762-0533-9df69b1ea84b Address:127.0.0.1:23391}]" +47:59.551 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dad0627e-978b-2762-0533-9df69b1ea84b.dc1 127.0.0.1 +47:59.551 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dad0627e-978b-2762-0533-9df69b1ea84b 127.0.0.1 +2022-12-04T05:47:59.552Z [INFO] agent.router: Initializing LAN area manager +47:59.552 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23386 network=udp +47:59.552 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23391 [Follower]" leader= +47:59.552 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23386 network=tcp +47:59.552 [INFO] TestAgent.server: Adding LAN server: server="Node-dad0627e-978b-2762-0533-9df69b1ea84b (Addr: tcp/127.0.0.1:23391) (DC: dc1)" +47:59.552 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dad0627e-978b-2762-0533-9df69b1ea84b.dc1 area=wan +47:59.553 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23387 network=tcp +47:59.553 [INFO] TestAgent: started state syncer +47:59.553 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23392 network=tcp +47:59.589 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:59.589 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23391 [Candidate]" term=2 +47:59.589 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:59.589 [DEBUG] TestAgent.server.raft: vote granted: from=dad0627e-978b-2762-0533-9df69b1ea84b term=2 tally=1 +47:59.589 [INFO] TestAgent.server.raft: election won: tally=1 +47:59.589 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23391 [Leader]" +47:59.589 [INFO] TestAgent.server: cluster leadership acquired +47:59.589 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23391 +47:59.590 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:59.591 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:59.591 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:59.591 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:59.591 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:59.591 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:59.591 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dad0627e-978b-2762-0533-9df69b1ea84b +47:59.592 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dad0627e-978b-2762-0533-9df69b1ea84b +47:59.592 [INFO] TestAgent.server: New leader elected: payload=Node-dad0627e-978b-2762-0533-9df69b1ea84b +47:59.592 [INFO] TestAgent.server: federation state anti-entropy synced +47:59.756 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:59.756 [INFO] TestAgent: Synced node info +47:59.756 [DEBUG] TestAgent: Node info in sync +47:59.883 [DEBUG] TestAgent: Node info in sync +47:59.883 [INFO] TestAgent: Synced service: service=web +47:59.883 [INFO] TestAgent: Requesting shutdown +47:59.883 [INFO] TestAgent.server: shutting down server +47:59.883 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:59.883 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:59.883 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:59.883 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:59.883 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:59.883 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:59.883 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:59.883 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:59.883 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:59.883 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:59.884 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:59.884Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:59.884Z [INFO] agent.router.manager: shutting down +47:59.884 [INFO] TestAgent: consul server down +47:59.884 [INFO] TestAgent: shutdown complete +47:59.884 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23386 network=tcp +47:59.884 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23386 network=udp +47:59.884 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23387 network=tcp +48:00.384 [INFO] TestAgent: Waiting for endpoints to shut down +48:00.384 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/default_sidecar -2021-10-31T23:11:42.507Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:42.536Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:42.552 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a4054324-697e-eb34-7ca1-e60ac71883ea Address:127.0.0.1:16251}]" -11:42.552 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16251 [Follower]" leader= -11:42.553 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a4054324-697e-eb34-7ca1-e60ac71883ea.dc1 127.0.0.1 -11:42.553 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a4054324-697e-eb34-7ca1-e60ac71883ea 127.0.0.1 -2021-10-31T23:11:42.553Z [INFO] agent.router: Initializing LAN area manager -11:42.553 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a4054324-697e-eb34-7ca1-e60ac71883ea.dc1 area=wan -11:42.554 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16246 network=tcp -11:42.554 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16246 network=udp -11:42.554 [INFO] TestAgent.server: Adding LAN server: server="Node-a4054324-697e-eb34-7ca1-e60ac71883ea (Addr: tcp/127.0.0.1:16251) (DC: dc1)" -11:42.554 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16247 network=tcp -11:42.554 [INFO] TestAgent: started state syncer -11:42.554 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16252 network=tcp -11:42.613 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:42.614 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16251 [Candidate]" term=2 -11:42.614 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:42.614 [DEBUG] TestAgent.server.raft: vote granted: from=a4054324-697e-eb34-7ca1-e60ac71883ea term=2 tally=1 -11:42.614 [INFO] TestAgent.server.raft: election won: tally=1 -11:42.614 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16251 [Leader]" -11:42.614 [INFO] TestAgent.server: cluster leadership acquired -11:42.614 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16251 -11:42.615 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:42.617 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:42.617 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:42.617 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:42.617 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:42.617 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:42.617 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a4054324-697e-eb34-7ca1-e60ac71883ea -11:42.617 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a4054324-697e-eb34-7ca1-e60ac71883ea -11:42.617 [INFO] TestAgent.server: New leader elected: payload=Node-a4054324-697e-eb34-7ca1-e60ac71883ea -11:42.617 [INFO] TestAgent.server: federation state anti-entropy synced -11:42.691 [INFO] TestAgent: Synced node info -11:42.691 [INFO] TestAgent: Synced service: service=web -11:42.713 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -11:42.713 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 -11:42.713 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 -11:42.714 [DEBUG] TestAgent: removed service: service=web -11:42.714 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -11:42.714 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -11:42.714 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -11:42.714 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -11:42.714 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 -11:42.715 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 -11:42.715 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -11:42.715 [DEBUG] TestAgent: Node info in sync -11:42.715 [INFO] TestAgent: Deregistered service: service=web -11:42.716 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -11:42.716 [INFO] TestAgent: Requesting shutdown -11:42.716 [INFO] TestAgent.server: shutting down server -11:42.716 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:42.716 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:42.716 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:42.716 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:42.716 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:42.716 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:42.716 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:42.717 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:42.717 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:42.717 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:42.717 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:42.717 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:42.717Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:42.717Z [INFO] agent.router.manager: shutting down -11:42.717 [INFO] TestAgent: consul server down -11:42.717 [INFO] TestAgent: shutdown complete -11:42.717 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16246 network=tcp -11:42.717 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16246 network=udp -11:42.717 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16247 network=tcp -11:43.220 [INFO] TestAgent: Waiting for endpoints to shut down -11:43.220 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:00.302Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:00.327Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:00.337 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b1faa5e6-0885-0282-fb87-152d8e87c5cc Address:127.0.0.1:23398}]" +48:00.338 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b1faa5e6-0885-0282-fb87-152d8e87c5cc.dc1 127.0.0.1 +48:00.338 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b1faa5e6-0885-0282-fb87-152d8e87c5cc 127.0.0.1 +2022-12-04T05:48:00.338Z [INFO] agent.router: Initializing LAN area manager +48:00.339 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23393 network=udp +48:00.339 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23398 [Follower]" leader= +48:00.339 [INFO] TestAgent.server: Adding LAN server: server="Node-b1faa5e6-0885-0282-fb87-152d8e87c5cc (Addr: tcp/127.0.0.1:23398) (DC: dc1)" +48:00.339 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b1faa5e6-0885-0282-fb87-152d8e87c5cc.dc1 area=wan +48:00.339 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23393 network=tcp +48:00.340 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23394 network=tcp +48:00.344 [INFO] TestAgent: started state syncer +48:00.344 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23399 network=tcp +48:00.388 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:00.388 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23398 [Candidate]" term=2 +48:00.388 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:00.388 [DEBUG] TestAgent.server.raft: vote granted: from=b1faa5e6-0885-0282-fb87-152d8e87c5cc term=2 tally=1 +48:00.388 [INFO] TestAgent.server.raft: election won: tally=1 +48:00.388 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23398 [Leader]" +48:00.388 [INFO] TestAgent.server: cluster leadership acquired +48:00.388 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23398 +48:00.389 [INFO] TestAgent.server: New leader elected: payload=Node-b1faa5e6-0885-0282-fb87-152d8e87c5cc +48:00.389 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:00.390 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:00.390 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:00.390 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:00.390 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:00.390 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:00.391 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b1faa5e6-0885-0282-fb87-152d8e87c5cc +48:00.391 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b1faa5e6-0885-0282-fb87-152d8e87c5cc +48:00.392 [INFO] TestAgent.server: federation state anti-entropy synced +48:00.420 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:00.420 [INFO] TestAgent: Synced node info +48:00.589 [DEBUG] TestAgent: Node info in sync +48:00.589 [INFO] TestAgent: Synced service: service=web +48:00.590 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +48:00.590 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +48:00.590 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +48:00.590 [DEBUG] TestAgent: removed service: service=web +48:00.590 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 +48:00.590 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 +48:00.590 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +48:00.590 [DEBUG] TestAgent: Node info in sync +48:00.591 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +48:00.591 [INFO] TestAgent: Deregistered service: service=web +48:00.591 [INFO] TestAgent: Requesting shutdown +48:00.591 [INFO] TestAgent.server: shutting down server +48:00.591 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:00.591 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:00.591 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:00.591 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:00.591 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:00.591 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:00.591 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:00.591 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:00.591 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:00.591 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:00.591 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:00.592Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:00.592Z [INFO] agent.router.manager: shutting down +48:00.592 [INFO] TestAgent: consul server down +48:00.592 [INFO] TestAgent: shutdown complete +48:00.592 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23393 network=tcp +48:00.592 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23393 network=udp +48:00.592 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23394 network=tcp +48:01.092 [INFO] TestAgent: Waiting for endpoints to shut down +48:01.092 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_defaults -2021-10-31T23:11:42.807Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:42.883Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:42.885 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b5a7a218-3222-512e-34d0-d92e6c213281 Address:127.0.0.1:16258}]" -11:42.885 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16258 [Follower]" leader= -11:42.892 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b5a7a218-3222-512e-34d0-d92e6c213281.dc1 127.0.0.1 -11:42.893 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b5a7a218-3222-512e-34d0-d92e6c213281 127.0.0.1 -2021-10-31T23:11:42.893Z [INFO] agent.router: Initializing LAN area manager -11:42.893 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16253 network=udp -11:42.893 [INFO] TestAgent.server: Adding LAN server: server="Node-b5a7a218-3222-512e-34d0-d92e6c213281 (Addr: tcp/127.0.0.1:16258) (DC: dc1)" -11:42.893 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b5a7a218-3222-512e-34d0-d92e6c213281.dc1 area=wan -11:42.893 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16253 network=tcp -11:42.894 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16254 network=tcp -11:42.894 [INFO] TestAgent: started state syncer -11:42.894 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16259 network=tcp -11:42.947 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:42.947 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16258 [Candidate]" term=2 -11:42.947 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:42.947 [DEBUG] TestAgent.server.raft: vote granted: from=b5a7a218-3222-512e-34d0-d92e6c213281 term=2 tally=1 -11:42.947 [INFO] TestAgent.server.raft: election won: tally=1 -11:42.947 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16258 [Leader]" -11:42.947 [INFO] TestAgent.server: cluster leadership acquired -11:42.947 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16258 -11:42.948 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:42.949 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:42.949 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:42.949 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:42.949 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:42.949 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:42.950 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b5a7a218-3222-512e-34d0-d92e6c213281 -11:42.950 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b5a7a218-3222-512e-34d0-d92e6c213281 -11:42.950 [INFO] TestAgent.server: New leader elected: payload=Node-b5a7a218-3222-512e-34d0-d92e6c213281 -11:42.950 [INFO] TestAgent.server: federation state anti-entropy synced -11:43.034 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:43.034 [INFO] TestAgent: Synced node info -11:43.034 [DEBUG] TestAgent: Node info in sync -11:43.252 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy -11:43.252 [DEBUG] TestAgent: Node info in sync -11:43.252 [INFO] TestAgent: Synced service: service=web -11:43.253 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -11:43.253 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 -11:43.253 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 -11:43.253 [DEBUG] TestAgent: removed service: service=web -11:43.253 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 -11:43.253 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 -11:43.253 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -11:43.253 [DEBUG] TestAgent: Node info in sync -11:43.270 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -11:43.271 [INFO] TestAgent: Deregistered service: service=web -11:43.271 [INFO] TestAgent: Requesting shutdown -11:43.271 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -11:43.271 [INFO] TestAgent.server: shutting down server -11:43.271 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:43.271 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:43.271 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:43.271 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:43.271 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:43.271 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:43.271 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:43.271 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:43.271 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:43.271 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:43.272 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:43.272Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:43.272Z [INFO] agent.router.manager: shutting down -11:43.272 [INFO] TestAgent: consul server down -11:43.272 [INFO] TestAgent: shutdown complete -11:43.272 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16253 network=tcp -11:43.272 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16253 network=udp -11:43.273 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16254 network=tcp -11:43.773 [INFO] TestAgent: Waiting for endpoints to shut down -11:43.773 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:00.415Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:00.461Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:00.464 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bef5fa0f-4704-d642-9dd6-c6551db64b81 Address:127.0.0.1:23405}]" +48:00.465 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23405 [Follower]" leader= +48:00.466 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bef5fa0f-4704-d642-9dd6-c6551db64b81.dc1 127.0.0.1 +48:00.467 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bef5fa0f-4704-d642-9dd6-c6551db64b81 127.0.0.1 +2022-12-04T05:48:00.467Z [INFO] agent.router: Initializing LAN area manager +48:00.467 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23400 network=udp +48:00.467 [INFO] TestAgent.server: Adding LAN server: server="Node-bef5fa0f-4704-d642-9dd6-c6551db64b81 (Addr: tcp/127.0.0.1:23405) (DC: dc1)" +48:00.467 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bef5fa0f-4704-d642-9dd6-c6551db64b81.dc1 area=wan +48:00.467 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23400 network=tcp +48:00.468 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23401 network=tcp +48:00.469 [INFO] TestAgent: started state syncer +48:00.469 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23406 network=tcp +48:00.532 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:00.532 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23405 [Candidate]" term=2 +48:00.532 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:00.532 [DEBUG] TestAgent.server.raft: vote granted: from=bef5fa0f-4704-d642-9dd6-c6551db64b81 term=2 tally=1 +48:00.532 [INFO] TestAgent.server.raft: election won: tally=1 +48:00.532 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23405 [Leader]" +48:00.532 [INFO] TestAgent.server: cluster leadership acquired +48:00.532 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23405 +48:00.533 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:00.534 [INFO] TestAgent.server: New leader elected: payload=Node-bef5fa0f-4704-d642-9dd6-c6551db64b81 +48:00.534 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:00.534 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:00.534 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:00.534 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:00.534 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:00.535 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bef5fa0f-4704-d642-9dd6-c6551db64b81 +48:00.535 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bef5fa0f-4704-d642-9dd6-c6551db64b81 +48:00.535 [INFO] TestAgent.server: federation state anti-entropy synced +48:00.632 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:00.632 [INFO] TestAgent: Synced node info +48:00.734 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy +48:00.734 [DEBUG] TestAgent: Node info in sync +48:00.734 [INFO] TestAgent: Synced service: service=web +48:00.735 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +48:00.735 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +48:00.735 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +48:00.735 [DEBUG] TestAgent: removed service: service=web +48:00.735 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 +48:00.735 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 +48:00.735 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +48:00.735 [DEBUG] TestAgent: Node info in sync +48:00.736 [INFO] TestAgent: Deregistered service: service=web +48:00.746 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +48:00.746 [INFO] TestAgent: Requesting shutdown +48:00.746 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +48:00.746 [INFO] TestAgent.server: shutting down server +48:00.746 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:00.746 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:00.746 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:00.746 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:00.746 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:00.746 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:00.746 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:00.746 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:00.746 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:00.746 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:00.746 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:00.746Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:00.746Z [INFO] agent.router.manager: shutting down +48:00.746 [INFO] TestAgent: consul server down +48:00.746 [INFO] TestAgent: shutdown complete +48:00.746 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23400 network=tcp +48:00.746 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23400 network=udp +48:00.746 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23401 network=tcp +48:01.248 [INFO] TestAgent: Waiting for endpoints to shut down +48:01.248 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_defaults -2021-10-31T23:11:43.248Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:43.248Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:43.308Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:43.308Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:43.314 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:428b5123-e301-41bb-8af5-4058b83db3bd Address:127.0.0.1:16265}]" -11:43.315 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-428b5123-e301-41bb-8af5-4058b83db3bd.dc1 127.0.0.1 -11:43.315 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-428b5123-e301-41bb-8af5-4058b83db3bd 127.0.0.1 -2021-10-31T23:11:43.315Z [INFO] agent.router: Initializing LAN area manager -11:43.316 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16260 network=udp -11:43.316 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16265 [Follower]" leader= -11:43.316 [INFO] TestAgent.server: Adding LAN server: server="Node-428b5123-e301-41bb-8af5-4058b83db3bd (Addr: tcp/127.0.0.1:16265) (DC: dc1)" -11:43.316 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-428b5123-e301-41bb-8af5-4058b83db3bd.dc1 area=wan -11:43.316 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16260 network=tcp -11:43.316 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16261 network=tcp -11:43.316 [INFO] TestAgent: started state syncer -11:43.317 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16266 network=tcp -11:43.355 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:43.355 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16265 [Candidate]" term=2 -11:43.355 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:43.355 [DEBUG] TestAgent.server.raft: vote granted: from=428b5123-e301-41bb-8af5-4058b83db3bd term=2 tally=1 -11:43.355 [INFO] TestAgent.server.raft: election won: tally=1 -11:43.355 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16265 [Leader]" -11:43.355 [INFO] TestAgent.server: cluster leadership acquired -11:43.355 [INFO] TestAgent.server: initializing acls -11:43.355 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:43.356 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:43.356 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:43.356 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:43.356 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:43.356 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:43.356 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-428b5123-e301-41bb-8af5-4058b83db3bd -11:43.356 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-428b5123-e301-41bb-8af5-4058b83db3bd.dc1 -11:43.357 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:43.358 [INFO] TestAgent.server: New leader elected: payload=Node-428b5123-e301-41bb-8af5-4058b83db3bd -11:43.358 [INFO] TestAgent.server: Updating LAN server: server="Node-428b5123-e301-41bb-8af5-4058b83db3bd (Addr: tcp/127.0.0.1:16265) (DC: dc1)" -11:43.358 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-428b5123-e301-41bb-8af5-4058b83db3bd.dc1 area=wan -11:43.359 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:43.359 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:43.359 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:43.359 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:43.359 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:43.359 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-428b5123-e301-41bb-8af5-4058b83db3bd -11:43.359 [INFO] TestAgent.server: member joined, marking health alive: member=Node-428b5123-e301-41bb-8af5-4058b83db3bd -11:43.359 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-428b5123-e301-41bb-8af5-4058b83db3bd -11:43.359 [INFO] TestAgent.server: federation state anti-entropy synced -11:43.389 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-428b5123-e301-41bb-8af5-4058b83db3bd -11:43.390 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-428b5123-e301-41bb-8af5-4058b83db3bd -11:43.392 [INFO] TestAgent: Synced node info -11:43.393 [INFO] TestAgent: Synced service: service=web -11:43.393 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -11:43.393 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 -11:43.393 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 -11:43.394 [DEBUG] TestAgent: removed service: service=web -11:43.394 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -11:43.394 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -11:43.394 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -11:43.394 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -11:43.394 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 -11:43.394 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 -11:43.394 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -11:43.394 [DEBUG] TestAgent: Node info in sync -11:43.395 [INFO] TestAgent: Deregistered service: service=web -11:43.395 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -11:43.395 [INFO] TestAgent: Requesting shutdown -11:43.395 [INFO] TestAgent.server: shutting down server -11:43.395 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:43.395 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:43.395 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:43.395 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:43.395 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:43.395 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:43.395 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:43.396 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:43.396 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:43.396 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:43.396 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:43.396 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:43.396 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:43.396 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:43.396 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:43.396 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:43.396Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:43.396Z [INFO] agent.router.manager: shutting down -11:43.396 [INFO] TestAgent: consul server down -11:43.396 [INFO] TestAgent: shutdown complete -11:43.396 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16260 network=tcp -11:43.396 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16260 network=udp -11:43.396 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16261 network=tcp -11:43.897 [INFO] TestAgent: Waiting for endpoints to shut down -11:43.897 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_denied -2021-10-31T23:11:43.806Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:43.806Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:43.851Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:43.851Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:43.859 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7dc87db3-2667-0b7e-afb8-9657ceb8c8ee Address:127.0.0.1:16272}]" -11:43.860 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7dc87db3-2667-0b7e-afb8-9657ceb8c8ee.dc1 127.0.0.1 -11:43.860 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16272 [Follower]" leader= -11:43.861 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7dc87db3-2667-0b7e-afb8-9657ceb8c8ee 127.0.0.1 -2021-10-31T23:11:43.861Z [INFO] agent.router: Initializing LAN area manager -11:43.861 [INFO] TestAgent.server: Adding LAN server: server="Node-7dc87db3-2667-0b7e-afb8-9657ceb8c8ee (Addr: tcp/127.0.0.1:16272) (DC: dc1)" -11:43.861 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7dc87db3-2667-0b7e-afb8-9657ceb8c8ee.dc1 area=wan -11:43.861 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16267 network=tcp -11:43.866 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16267 network=udp -11:43.866 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16268 network=tcp -11:43.866 [INFO] TestAgent: started state syncer -11:43.867 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16273 network=tcp -11:43.909 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:43.910 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16272 [Candidate]" term=2 -11:43.910 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:43.910 [DEBUG] TestAgent.server.raft: vote granted: from=7dc87db3-2667-0b7e-afb8-9657ceb8c8ee term=2 tally=1 -11:43.910 [INFO] TestAgent.server.raft: election won: tally=1 -11:43.910 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16272 [Leader]" -11:43.910 [INFO] TestAgent.server: cluster leadership acquired -11:43.910 [INFO] TestAgent.server: initializing acls -11:43.910 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:43.910 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:43.911 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:43.911 [INFO] TestAgent.server: New leader elected: payload=Node-7dc87db3-2667-0b7e-afb8-9657ceb8c8ee -11:43.911 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:43.911 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:43.911 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:43.911 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7dc87db3-2667-0b7e-afb8-9657ceb8c8ee -11:43.911 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7dc87db3-2667-0b7e-afb8-9657ceb8c8ee.dc1 -11:43.912 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:43.912 [INFO] TestAgent.server: Updating LAN server: server="Node-7dc87db3-2667-0b7e-afb8-9657ceb8c8ee (Addr: tcp/127.0.0.1:16272) (DC: dc1)" -11:43.912 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7dc87db3-2667-0b7e-afb8-9657ceb8c8ee.dc1 area=wan -11:43.913 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:43.913 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:43.913 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:43.913 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:43.913 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:43.913 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7dc87db3-2667-0b7e-afb8-9657ceb8c8ee -11:43.913 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7dc87db3-2667-0b7e-afb8-9657ceb8c8ee -11:43.914 [INFO] TestAgent.server: federation state anti-entropy synced -11:43.914 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7dc87db3-2667-0b7e-afb8-9657ceb8c8ee -11:43.982 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7dc87db3-2667-0b7e-afb8-9657ceb8c8ee -11:43.982 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7dc87db3-2667-0b7e-afb8-9657ceb8c8ee -11:43.985 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy -11:43.985 [INFO] TestAgent: Synced node info -11:43.987 [INFO] TestAgent: Synced service: service=web -11:43.988 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -11:43.988 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 -11:43.988 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 -11:43.990 [DEBUG] TestAgent: removed service: service=web -11:43.990 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 -11:43.990 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 -11:43.990 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -11:43.990 [DEBUG] TestAgent: Node info in sync -11:43.990 [INFO] TestAgent: Deregistered service: service=web -11:43.991 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -11:43.991 [INFO] TestAgent: Requesting shutdown -11:43.991 [INFO] TestAgent.server: shutting down server -11:43.991 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:43.991 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:43.991 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:43.991 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:43.991 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:43.991 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:43.991 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:43.991 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:43.991 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:43.991 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -11:43.991 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:43.991 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:43.991 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:43.991 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:43.991 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:43.991 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2021-10-31T23:11:43.991Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:43.991Z [INFO] agent.router.manager: shutting down -11:43.992 [INFO] TestAgent: consul server down -11:43.992 [INFO] TestAgent: shutdown complete -11:43.992 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16267 network=tcp -11:43.992 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16267 network=udp -11:43.992 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16268 network=tcp -11:44.493 [INFO] TestAgent: Waiting for endpoints to shut down -11:44.493 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:01.290Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:01.290Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:01.326Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:01.326Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:01.336 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:37b5c533-b8a4-0886-3d64-8e101f165340 Address:127.0.0.1:23419}]" +48:01.336 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23419 [Follower]" leader= +48:01.336 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-37b5c533-b8a4-0886-3d64-8e101f165340.dc1 127.0.0.1 +48:01.337 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-37b5c533-b8a4-0886-3d64-8e101f165340 127.0.0.1 +2022-12-04T05:48:01.337Z [INFO] agent.router: Initializing LAN area manager +48:01.337 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23414 network=udp +48:01.337 [INFO] TestAgent.server: Adding LAN server: server="Node-37b5c533-b8a4-0886-3d64-8e101f165340 (Addr: tcp/127.0.0.1:23419) (DC: dc1)" +48:01.337 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-37b5c533-b8a4-0886-3d64-8e101f165340.dc1 area=wan +48:01.337 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23414 network=tcp +48:01.338 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23415 network=tcp +48:01.338 [INFO] TestAgent: started state syncer +48:01.338 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23420 network=tcp +48:01.389 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:01.395 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:01.395 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23419 [Candidate]" term=2 +48:01.395 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:01.395 [DEBUG] TestAgent.server.raft: vote granted: from=37b5c533-b8a4-0886-3d64-8e101f165340 term=2 tally=1 +48:01.395 [INFO] TestAgent.server.raft: election won: tally=1 +48:01.395 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23419 [Leader]" +48:01.395 [INFO] TestAgent.server: cluster leadership acquired +48:01.395 [INFO] TestAgent.server: initializing acls +48:01.396 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:01.396 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:01.396 [INFO] TestAgent.server: New leader elected: payload=Node-37b5c533-b8a4-0886-3d64-8e101f165340 +48:01.396 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:01.396 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:01.396 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:01.396 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:01.397 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-37b5c533-b8a4-0886-3d64-8e101f165340 +48:01.397 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-37b5c533-b8a4-0886-3d64-8e101f165340.dc1 +48:01.397 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:01.398 [INFO] TestAgent.server: Updating LAN server: server="Node-37b5c533-b8a4-0886-3d64-8e101f165340 (Addr: tcp/127.0.0.1:23419) (DC: dc1)" +48:01.398 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-37b5c533-b8a4-0886-3d64-8e101f165340.dc1 area=wan +48:01.399 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:01.399 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:01.399 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:01.399 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:01.399 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:01.399 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-37b5c533-b8a4-0886-3d64-8e101f165340 +48:01.399 [INFO] TestAgent.server: member joined, marking health alive: member=Node-37b5c533-b8a4-0886-3d64-8e101f165340 +48:01.399 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-37b5c533-b8a4-0886-3d64-8e101f165340 +48:01.399 [INFO] TestAgent.server: federation state anti-entropy synced +48:01.480 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-37b5c533-b8a4-0886-3d64-8e101f165340 +48:01.480 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-37b5c533-b8a4-0886-3d64-8e101f165340 +48:01.483 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy +48:01.484 [INFO] TestAgent: Synced node info +48:01.488 [INFO] TestAgent: Synced service: service=web +48:01.489 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +48:01.489 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +48:01.489 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +48:01.491 [DEBUG] TestAgent: removed service: service=web +48:01.519 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +48:01.519 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +48:01.519 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +48:01.519 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +48:01.520 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 +48:01.520 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 +48:01.520 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +48:01.520 [DEBUG] TestAgent: Node info in sync +48:01.525 [INFO] TestAgent: Deregistered service: service=web +48:01.526 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +48:01.526 [INFO] TestAgent: Requesting shutdown +48:01.526 [INFO] TestAgent.server: shutting down server +48:01.526 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:01.526 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:01.526 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:01.526 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:01.526 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:01.526 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:01.526 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:01.527 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:01.527 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:01.527 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:01.527 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:01.527 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:01.527 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:01.527 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:01.527 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:01.527 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:01.527Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:01.527Z [INFO] agent.router.manager: shutting down +48:01.527 [INFO] TestAgent: consul server down +48:01.527 [INFO] TestAgent: shutdown complete +48:01.527 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23414 network=tcp +48:01.527 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23414 network=udp +48:01.527 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23415 network=tcp +48:02.028 [INFO] TestAgent: Waiting for endpoints to shut down +48:02.028 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_denied -2021-10-31T23:11:43.924Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:43.924Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:43.951Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:43.951Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:43.953 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9f5122b5-a173-83e1-8141-86734b29f468 Address:127.0.0.1:16279}]" -11:43.953 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16279 [Follower]" leader= -11:43.958 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9f5122b5-a173-83e1-8141-86734b29f468.dc1 127.0.0.1 -11:43.959 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9f5122b5-a173-83e1-8141-86734b29f468 127.0.0.1 -2021-10-31T23:11:43.959Z [INFO] agent.router: Initializing LAN area manager -11:43.959 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16274 network=udp -11:43.959 [INFO] TestAgent.server: Adding LAN server: server="Node-9f5122b5-a173-83e1-8141-86734b29f468 (Addr: tcp/127.0.0.1:16279) (DC: dc1)" -11:43.959 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9f5122b5-a173-83e1-8141-86734b29f468.dc1 area=wan -11:43.959 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16274 network=tcp -11:43.960 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16275 network=tcp -11:43.960 [INFO] TestAgent: started state syncer -11:43.960 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16280 network=tcp -11:44.010 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -11:44.011 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:44.011 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16279 [Candidate]" term=2 -11:44.012 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:44.012 [DEBUG] TestAgent.server.raft: vote granted: from=9f5122b5-a173-83e1-8141-86734b29f468 term=2 tally=1 -11:44.012 [INFO] TestAgent.server.raft: election won: tally=1 -11:44.012 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16279 [Leader]" -11:44.012 [INFO] TestAgent.server: cluster leadership acquired -11:44.012 [INFO] TestAgent.server: initializing acls -11:44.012 [INFO] TestAgent.server: New leader elected: payload=Node-9f5122b5-a173-83e1-8141-86734b29f468 -11:44.012 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:44.012 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:44.013 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:44.013 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:44.013 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:44.013 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:44.013 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9f5122b5-a173-83e1-8141-86734b29f468 -11:44.013 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9f5122b5-a173-83e1-8141-86734b29f468.dc1 -11:44.013 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9f5122b5-a173-83e1-8141-86734b29f468.dc1 area=wan -11:44.014 [INFO] TestAgent.server: Updating LAN server: server="Node-9f5122b5-a173-83e1-8141-86734b29f468 (Addr: tcp/127.0.0.1:16279) (DC: dc1)" -11:44.014 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:44.015 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:44.015 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:44.015 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:44.015 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:44.015 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:44.015 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9f5122b5-a173-83e1-8141-86734b29f468 -11:44.015 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9f5122b5-a173-83e1-8141-86734b29f468 -11:44.016 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9f5122b5-a173-83e1-8141-86734b29f468 -11:44.016 [INFO] TestAgent.server: federation state anti-entropy synced -11:44.205 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:44.206 [INFO] TestAgent: Synced node info -11:44.370 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9f5122b5-a173-83e1-8141-86734b29f468 -11:44.370 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9f5122b5-a173-83e1-8141-86734b29f468 -11:44.370 [INFO] TestAgent: Requesting shutdown -11:44.370 [INFO] TestAgent.server: shutting down server -11:44.370 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:44.370 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:44.370 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:44.370 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:44.370 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:44.370 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:44.370 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:44.370 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:44.370 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:44.370 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:44.370 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:44.370 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:44.370 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:44.370 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:44.370 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:44.370Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:44.370Z [INFO] agent.router.manager: shutting down -11:44.370 [INFO] TestAgent: consul server down -11:44.370 [INFO] TestAgent: shutdown complete -11:44.370 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16274 network=tcp -11:44.370 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16274 network=udp -11:44.370 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16275 network=tcp -11:44.871 [INFO] TestAgent: Waiting for endpoints to shut down -11:44.871 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:01.122Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:01.122Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:01.157Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:01.157Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:01.158 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e5acf5b2-de7c-ae41-d22d-4cbfb9879048 Address:127.0.0.1:23412}]" +48:01.159 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23412 [Follower]" leader= +48:01.159 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e5acf5b2-de7c-ae41-d22d-4cbfb9879048.dc1 127.0.0.1 +48:01.159 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e5acf5b2-de7c-ae41-d22d-4cbfb9879048 127.0.0.1 +2022-12-04T05:48:01.159Z [INFO] agent.router: Initializing LAN area manager +48:01.160 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23407 network=udp +48:01.160 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e5acf5b2-de7c-ae41-d22d-4cbfb9879048.dc1 area=wan +48:01.160 [INFO] TestAgent.server: Adding LAN server: server="Node-e5acf5b2-de7c-ae41-d22d-4cbfb9879048 (Addr: tcp/127.0.0.1:23412) (DC: dc1)" +48:01.160 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23407 network=tcp +48:01.178 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23408 network=tcp +48:01.178 [INFO] TestAgent: started state syncer +48:01.178 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23413 network=tcp +48:01.200 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:01.200 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23412 [Candidate]" term=2 +48:01.200 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:01.200 [DEBUG] TestAgent.server.raft: vote granted: from=e5acf5b2-de7c-ae41-d22d-4cbfb9879048 term=2 tally=1 +48:01.200 [INFO] TestAgent.server.raft: election won: tally=1 +48:01.200 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23412 [Leader]" +48:01.200 [INFO] TestAgent.server: cluster leadership acquired +48:01.200 [INFO] TestAgent.server: initializing acls +48:01.201 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:01.201 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:01.201 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:01.202 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:01.202 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:01.202 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:01.202 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e5acf5b2-de7c-ae41-d22d-4cbfb9879048 +48:01.202 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e5acf5b2-de7c-ae41-d22d-4cbfb9879048.dc1 +48:01.202 [INFO] TestAgent.server: New leader elected: payload=Node-e5acf5b2-de7c-ae41-d22d-4cbfb9879048 +48:01.202 [INFO] TestAgent.server: Updating LAN server: server="Node-e5acf5b2-de7c-ae41-d22d-4cbfb9879048 (Addr: tcp/127.0.0.1:23412) (DC: dc1)" +48:01.202 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e5acf5b2-de7c-ae41-d22d-4cbfb9879048.dc1 area=wan +48:01.202 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:01.204 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:01.204 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:01.204 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:01.204 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:01.204 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:01.205 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e5acf5b2-de7c-ae41-d22d-4cbfb9879048 +48:01.205 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e5acf5b2-de7c-ae41-d22d-4cbfb9879048 +48:01.205 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e5acf5b2-de7c-ae41-d22d-4cbfb9879048 +48:01.205 [INFO] TestAgent.server: federation state anti-entropy synced +48:01.269 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:01.269 [INFO] TestAgent: Synced node info +48:01.269 [DEBUG] TestAgent: Node info in sync +48:01.551 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e5acf5b2-de7c-ae41-d22d-4cbfb9879048 +48:01.551 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e5acf5b2-de7c-ae41-d22d-4cbfb9879048 +48:01.554 [DEBUG] TestAgent: Node info in sync +48:01.555 [INFO] TestAgent: Synced service: service=web +48:01.555 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +48:01.555 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +48:01.555 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +48:01.556 [DEBUG] TestAgent: removed service: service=web +48:01.556 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 +48:01.556 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 +48:01.556 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +48:01.556 [DEBUG] TestAgent: Node info in sync +48:01.556 [INFO] TestAgent: Deregistered service: service=web +48:01.557 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +48:01.557 [INFO] TestAgent: Requesting shutdown +48:01.557 [INFO] TestAgent.server: shutting down server +48:01.557 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:01.557 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:01.557 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:01.557 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:01.557 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:01.557 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:01.557 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:01.557 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:01.557 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:01.557 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:01.557 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:01.557 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:01.557 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:01.557 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:01.557 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:01.557Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:01.557Z [INFO] agent.router.manager: shutting down +48:01.557 [INFO] TestAgent: consul server down +48:01.557 [INFO] TestAgent: shutdown complete +48:01.557 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23407 network=tcp +48:01.557 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23407 network=udp +48:01.557 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23408 network=tcp +48:02.060 [INFO] TestAgent: Waiting for endpoints to shut down +48:02.060 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_denied +2022-12-04T05:48:02.099Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:02.099Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:02.169Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:02.169Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:02.170 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fea4c8b8-11a8-fb03-dcf4-adae28ccf001 Address:127.0.0.1:23433}]" +48:02.171 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fea4c8b8-11a8-fb03-dcf4-adae28ccf001.dc1 127.0.0.1 +48:02.172 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fea4c8b8-11a8-fb03-dcf4-adae28ccf001 127.0.0.1 +2022-12-04T05:48:02.172Z [INFO] agent.router: Initializing LAN area manager +48:02.172 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23433 [Follower]" leader= +48:02.172 [INFO] TestAgent.server: Adding LAN server: server="Node-fea4c8b8-11a8-fb03-dcf4-adae28ccf001 (Addr: tcp/127.0.0.1:23433) (DC: dc1)" +48:02.172 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fea4c8b8-11a8-fb03-dcf4-adae28ccf001.dc1 area=wan +48:02.176 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23428 network=udp +48:02.176 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23428 network=tcp +48:02.177 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23429 network=tcp +48:02.177 [INFO] TestAgent: started state syncer +48:02.177 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23434 network=tcp +48:02.224 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:02.228 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:02.228 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23433 [Candidate]" term=2 +48:02.228 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:02.228 [DEBUG] TestAgent.server.raft: vote granted: from=fea4c8b8-11a8-fb03-dcf4-adae28ccf001 term=2 tally=1 +48:02.228 [INFO] TestAgent.server.raft: election won: tally=1 +48:02.228 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23433 [Leader]" +48:02.228 [INFO] TestAgent.server: cluster leadership acquired +48:02.228 [INFO] TestAgent.server: initializing acls +48:02.229 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:02.229 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:02.229 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:02.229 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:02.229 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:02.229 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:02.229 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-fea4c8b8-11a8-fb03-dcf4-adae28ccf001 +48:02.230 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-fea4c8b8-11a8-fb03-dcf4-adae28ccf001.dc1 +48:02.230 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:02.231 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:02.231 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:02.231 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:02.231 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:02.231 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:02.231 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fea4c8b8-11a8-fb03-dcf4-adae28ccf001 +48:02.231 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fea4c8b8-11a8-fb03-dcf4-adae28ccf001 +48:02.232 [INFO] TestAgent.server: New leader elected: payload=Node-fea4c8b8-11a8-fb03-dcf4-adae28ccf001 +48:02.232 [INFO] TestAgent.server: Updating LAN server: server="Node-fea4c8b8-11a8-fb03-dcf4-adae28ccf001 (Addr: tcp/127.0.0.1:23433) (DC: dc1)" +48:02.232 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fea4c8b8-11a8-fb03-dcf4-adae28ccf001 +48:02.232 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-fea4c8b8-11a8-fb03-dcf4-adae28ccf001.dc1 area=wan +48:02.232 [INFO] TestAgent.server: federation state anti-entropy synced +48:02.412 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fea4c8b8-11a8-fb03-dcf4-adae28ccf001 +48:02.412 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fea4c8b8-11a8-fb03-dcf4-adae28ccf001 +48:02.412 [INFO] TestAgent: Requesting shutdown +48:02.412 [INFO] TestAgent.server: shutting down server +48:02.412 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:02.412 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:02.412 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:02.412 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:02.412 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:02.412 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:02.412 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:02.413 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:02.413 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:02.413 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:02.413 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:02.413 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:02.413 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:02.413 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:02.413 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:02.413 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:02.413Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:02.413Z [INFO] agent.router.manager: shutting down +48:02.413 [INFO] TestAgent: consul server down +48:02.413 [INFO] TestAgent: shutdown complete +48:02.413 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23428 network=tcp +48:02.413 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23428 network=udp +48:02.413 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23429 network=tcp +48:02.913 [INFO] TestAgent: Waiting for endpoints to shut down +48:02.913 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_not_for_sidecar -2021-10-31T23:11:44.559Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:44.559Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:44.596Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:44.596Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:44.602 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1520910d-59b2-fbaf-7465-cb709288fb1c Address:127.0.0.1:16286}]" -11:44.603 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1520910d-59b2-fbaf-7465-cb709288fb1c.dc1 127.0.0.1 -11:44.603 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1520910d-59b2-fbaf-7465-cb709288fb1c 127.0.0.1 -2021-10-31T23:11:44.604Z [INFO] agent.router: Initializing LAN area manager -11:44.604 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16281 network=udp -11:44.604 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16286 [Follower]" leader= -11:44.604 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1520910d-59b2-fbaf-7465-cb709288fb1c.dc1 area=wan -11:44.604 [INFO] TestAgent.server: Adding LAN server: server="Node-1520910d-59b2-fbaf-7465-cb709288fb1c (Addr: tcp/127.0.0.1:16286) (DC: dc1)" -11:44.604 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16281 network=tcp -11:44.605 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16282 network=tcp -11:44.605 [INFO] TestAgent: started state syncer -11:44.605 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16287 network=tcp -11:44.657 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -11:44.673 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:44.673 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16286 [Candidate]" term=2 -11:44.674 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:44.674 [DEBUG] TestAgent.server.raft: vote granted: from=1520910d-59b2-fbaf-7465-cb709288fb1c term=2 tally=1 -11:44.674 [INFO] TestAgent.server.raft: election won: tally=1 -11:44.674 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16286 [Leader]" -11:44.674 [INFO] TestAgent.server: cluster leadership acquired -11:44.674 [INFO] TestAgent.server: New leader elected: payload=Node-1520910d-59b2-fbaf-7465-cb709288fb1c -11:44.674 [INFO] TestAgent.server: initializing acls -11:44.674 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:44.674 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:44.675 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:44.675 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:44.675 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:44.675 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:44.675 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1520910d-59b2-fbaf-7465-cb709288fb1c -11:44.675 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1520910d-59b2-fbaf-7465-cb709288fb1c.dc1 -11:44.676 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:44.677 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:44.677 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:44.677 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:44.677 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:44.677 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:44.677 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1520910d-59b2-fbaf-7465-cb709288fb1c -11:44.677 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1520910d-59b2-fbaf-7465-cb709288fb1c -11:44.677 [INFO] TestAgent.server: Updating LAN server: server="Node-1520910d-59b2-fbaf-7465-cb709288fb1c (Addr: tcp/127.0.0.1:16286) (DC: dc1)" -11:44.678 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1520910d-59b2-fbaf-7465-cb709288fb1c.dc1 area=wan -11:44.682 [INFO] TestAgent.server: federation state anti-entropy synced -11:44.682 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1520910d-59b2-fbaf-7465-cb709288fb1c -11:44.864 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:44.864 [INFO] TestAgent: Synced node info -11:44.993 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-1520910d-59b2-fbaf-7465-cb709288fb1c -11:44.993 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-1520910d-59b2-fbaf-7465-cb709288fb1c -11:44.993 [INFO] TestAgent: Requesting shutdown -11:44.993 [INFO] TestAgent.server: shutting down server -11:44.993 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:44.993 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:44.993 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:44.993 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:44.993 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:44.993 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:44.993 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:44.993 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:44.994 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:44.994 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:44.994 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:44.994 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:44.994 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:44.994 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:44.994 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:44.994Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:44.994Z [INFO] agent.router.manager: shutting down -11:44.994 [INFO] TestAgent: consul server down -11:44.994 [INFO] TestAgent: shutdown complete -11:44.994 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16281 network=tcp -11:44.994 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16281 network=udp -11:44.994 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16282 network=tcp -11:45.498 [INFO] TestAgent: Waiting for endpoints to shut down -11:45.498 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:02.055Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:02.055Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:02.089Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:02.089Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:02.091 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d2902252-65c6-b1d3-c27e-1d1a752a2102 Address:127.0.0.1:23426}]" +48:02.091 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d2902252-65c6-b1d3-c27e-1d1a752a2102.dc1 127.0.0.1 +48:02.092 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23426 [Follower]" leader= +48:02.100 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d2902252-65c6-b1d3-c27e-1d1a752a2102 127.0.0.1 +2022-12-04T05:48:02.100Z [INFO] agent.router: Initializing LAN area manager +48:02.100 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23421 network=udp +48:02.100 [INFO] TestAgent.server: Adding LAN server: server="Node-d2902252-65c6-b1d3-c27e-1d1a752a2102 (Addr: tcp/127.0.0.1:23426) (DC: dc1)" +48:02.101 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d2902252-65c6-b1d3-c27e-1d1a752a2102.dc1 area=wan +48:02.101 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23421 network=tcp +48:02.101 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23422 network=tcp +48:02.101 [INFO] TestAgent: started state syncer +48:02.101 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23427 network=tcp +48:02.144 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:02.144 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23426 [Candidate]" term=2 +48:02.144 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:02.144 [DEBUG] TestAgent.server.raft: vote granted: from=d2902252-65c6-b1d3-c27e-1d1a752a2102 term=2 tally=1 +48:02.144 [INFO] TestAgent.server.raft: election won: tally=1 +48:02.144 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23426 [Leader]" +48:02.144 [INFO] TestAgent.server: cluster leadership acquired +48:02.145 [INFO] TestAgent.server: initializing acls +48:02.145 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:02.145 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:02.145 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:02.146 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:02.146 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:02.146 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:02.146 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d2902252-65c6-b1d3-c27e-1d1a752a2102 +48:02.146 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d2902252-65c6-b1d3-c27e-1d1a752a2102.dc1 +48:02.147 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:02.148 [INFO] TestAgent.server: New leader elected: payload=Node-d2902252-65c6-b1d3-c27e-1d1a752a2102 +48:02.148 [INFO] TestAgent.server: Updating LAN server: server="Node-d2902252-65c6-b1d3-c27e-1d1a752a2102 (Addr: tcp/127.0.0.1:23426) (DC: dc1)" +48:02.148 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d2902252-65c6-b1d3-c27e-1d1a752a2102.dc1 area=wan +48:02.153 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:02.153 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:02.153 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:02.153 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:02.153 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:02.153 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d2902252-65c6-b1d3-c27e-1d1a752a2102 +48:02.154 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d2902252-65c6-b1d3-c27e-1d1a752a2102 +48:02.154 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d2902252-65c6-b1d3-c27e-1d1a752a2102 +48:02.155 [INFO] TestAgent.server: federation state anti-entropy synced +48:02.280 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:02.281 [INFO] TestAgent: Synced node info +48:02.281 [DEBUG] TestAgent: Node info in sync +48:02.428 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d2902252-65c6-b1d3-c27e-1d1a752a2102 +48:02.428 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d2902252-65c6-b1d3-c27e-1d1a752a2102 +48:02.428 [INFO] TestAgent: Requesting shutdown +48:02.428 [INFO] TestAgent.server: shutting down server +48:02.428 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:02.428 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:02.428 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:02.428 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:02.428 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:02.428 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:02.428 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:02.429 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:02.429 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:02.429 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:02.429 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:02.429 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:02.429 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:02.429 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:02.429 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:02.429Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:02.429Z [INFO] agent.router.manager: shutting down +48:02.429 [INFO] TestAgent: consul server down +48:02.429 [INFO] TestAgent: shutdown complete +48:02.429 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23421 network=tcp +48:02.429 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23421 network=udp +48:02.429 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23422 network=tcp +48:02.932 [INFO] TestAgent: Waiting for endpoints to shut down +48:02.932 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_not_for_sidecar -2021-10-31T23:11:44.917Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:44.917Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:44.996Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:44.996Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:45.007 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:624cb864-1bb9-de31-2dc5-f4fb44bb3d02 Address:127.0.0.1:16293}]" -11:45.008 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-624cb864-1bb9-de31-2dc5-f4fb44bb3d02.dc1 127.0.0.1 -11:45.009 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-624cb864-1bb9-de31-2dc5-f4fb44bb3d02 127.0.0.1 -2021-10-31T23:11:45.009Z [INFO] agent.router: Initializing LAN area manager -11:45.009 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16288 network=udp -11:45.009 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16293 [Follower]" leader= -11:45.010 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-624cb864-1bb9-de31-2dc5-f4fb44bb3d02.dc1 area=wan -11:45.010 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16288 network=tcp -11:45.011 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16289 network=tcp -11:45.011 [INFO] TestAgent: started state syncer -11:45.011 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16294 network=tcp -11:45.017 [INFO] TestAgent.server: Adding LAN server: server="Node-624cb864-1bb9-de31-2dc5-f4fb44bb3d02 (Addr: tcp/127.0.0.1:16293) (DC: dc1)" -11:45.061 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -11:45.073 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:45.073 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16293 [Candidate]" term=2 -11:45.074 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:45.074 [DEBUG] TestAgent.server.raft: vote granted: from=624cb864-1bb9-de31-2dc5-f4fb44bb3d02 term=2 tally=1 -11:45.074 [INFO] TestAgent.server.raft: election won: tally=1 -11:45.074 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16293 [Leader]" -11:45.074 [INFO] TestAgent.server: cluster leadership acquired -11:45.074 [INFO] TestAgent.server: initializing acls -11:45.074 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:45.074 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:45.075 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:45.075 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:45.075 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:45.075 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:45.075 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-624cb864-1bb9-de31-2dc5-f4fb44bb3d02 -11:45.075 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-624cb864-1bb9-de31-2dc5-f4fb44bb3d02.dc1 -11:45.076 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:45.077 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:45.077 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:45.077 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:45.077 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:45.077 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:45.077 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-624cb864-1bb9-de31-2dc5-f4fb44bb3d02 -11:45.077 [INFO] TestAgent.server: member joined, marking health alive: member=Node-624cb864-1bb9-de31-2dc5-f4fb44bb3d02 -11:45.077 [INFO] TestAgent.server: New leader elected: payload=Node-624cb864-1bb9-de31-2dc5-f4fb44bb3d02 -11:45.078 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-624cb864-1bb9-de31-2dc5-f4fb44bb3d02.dc1 area=wan -11:45.078 [INFO] TestAgent.server: Updating LAN server: server="Node-624cb864-1bb9-de31-2dc5-f4fb44bb3d02 (Addr: tcp/127.0.0.1:16293) (DC: dc1)" -11:45.082 [INFO] TestAgent.server: federation state anti-entropy synced -11:45.082 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-624cb864-1bb9-de31-2dc5-f4fb44bb3d02 -11:45.121 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:45.121 [INFO] TestAgent: Synced node info -11:45.121 [DEBUG] TestAgent: Node info in sync -11:45.121 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-624cb864-1bb9-de31-2dc5-f4fb44bb3d02 -11:45.122 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-624cb864-1bb9-de31-2dc5-f4fb44bb3d02 -11:45.123 [INFO] TestAgent: Requesting shutdown -11:45.123 [INFO] TestAgent.server: shutting down server -11:45.123 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:45.123 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:45.123 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:45.123 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:45.123 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:45.123 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:45.123 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:45.123 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:45.123 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:45.123 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:45.123 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:45.123 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:45.123 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:45.123 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:45.123 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:45.124Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:45.124Z [INFO] agent.router.manager: shutting down -11:45.124 [INFO] TestAgent: consul server down -11:45.124 [INFO] TestAgent: shutdown complete -11:45.124 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16288 network=tcp -11:45.124 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16288 network=udp -11:45.124 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16289 network=tcp -11:45.624 [INFO] TestAgent: Waiting for endpoints to shut down -11:45.624 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_and_sidecar_but_not_sidecar's_overridden_destination -2021-10-31T23:11:45.521Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:45.521Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:45.552Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:45.552Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:45.555 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:71b53c53-8d7a-467d-f023-7aad9e8040a0 Address:127.0.0.1:16300}]" -11:45.555 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-71b53c53-8d7a-467d-f023-7aad9e8040a0.dc1 127.0.0.1 -11:45.556 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-71b53c53-8d7a-467d-f023-7aad9e8040a0 127.0.0.1 -2021-10-31T23:11:45.556Z [INFO] agent.router: Initializing LAN area manager -11:45.556 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16295 network=udp -11:45.556 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16300 [Follower]" leader= -11:45.556 [INFO] TestAgent.server: Adding LAN server: server="Node-71b53c53-8d7a-467d-f023-7aad9e8040a0 (Addr: tcp/127.0.0.1:16300) (DC: dc1)" -11:45.556 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-71b53c53-8d7a-467d-f023-7aad9e8040a0.dc1 area=wan -11:45.556 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16295 network=tcp -11:45.557 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16296 network=tcp -11:45.557 [INFO] TestAgent: started state syncer -11:45.557 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16301 network=tcp -11:45.605 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:45.605 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16300 [Candidate]" term=2 -11:45.606 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:45.606 [DEBUG] TestAgent.server.raft: vote granted: from=71b53c53-8d7a-467d-f023-7aad9e8040a0 term=2 tally=1 -11:45.606 [INFO] TestAgent.server.raft: election won: tally=1 -11:45.606 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16300 [Leader]" -11:45.606 [INFO] TestAgent.server: cluster leadership acquired -11:45.606 [INFO] TestAgent.server: initializing acls -11:45.606 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:45.606 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:45.607 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:45.607 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:45.607 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:45.607 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:45.607 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-71b53c53-8d7a-467d-f023-7aad9e8040a0 -11:45.607 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-71b53c53-8d7a-467d-f023-7aad9e8040a0.dc1 -11:45.608 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:45.609 [INFO] TestAgent.server: New leader elected: payload=Node-71b53c53-8d7a-467d-f023-7aad9e8040a0 -11:45.609 [INFO] TestAgent.server: Updating LAN server: server="Node-71b53c53-8d7a-467d-f023-7aad9e8040a0 (Addr: tcp/127.0.0.1:16300) (DC: dc1)" -11:45.609 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-71b53c53-8d7a-467d-f023-7aad9e8040a0.dc1 area=wan -11:45.609 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:45.609 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:45.609 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:45.609 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:45.609 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:45.609 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-71b53c53-8d7a-467d-f023-7aad9e8040a0 -11:45.609 [INFO] TestAgent.server: member joined, marking health alive: member=Node-71b53c53-8d7a-467d-f023-7aad9e8040a0 -11:45.610 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-71b53c53-8d7a-467d-f023-7aad9e8040a0 -11:45.610 [INFO] TestAgent.server: federation state anti-entropy synced -11:45.893 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-71b53c53-8d7a-467d-f023-7aad9e8040a0 -11:45.894 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-71b53c53-8d7a-467d-f023-7aad9e8040a0 -11:45.895 [INFO] TestAgent: Requesting shutdown -11:45.896 [INFO] TestAgent.server: shutting down server -11:45.896 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:45.896 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:45.896 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:45.896 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:45.896 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:45.896 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:45.896 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:45.896 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:45.896 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:45.896 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:45.896 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:45.896 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:45.896 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:45.896 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:45.896 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:45.896 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:45.896Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:45.896Z [INFO] agent.router.manager: shutting down -11:45.896 [INFO] TestAgent: consul server down -11:45.896 [INFO] TestAgent: shutdown complete -11:45.896 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16295 network=tcp -11:45.896 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16295 network=udp -11:45.896 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16296 network=tcp -11:46.397 [INFO] TestAgent: Waiting for endpoints to shut down -11:46.397 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:02.964Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:02.964Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:03.007Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:03.007Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:03.009 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b42c4ac5-958b-5415-e9e5-2203f63328ab Address:127.0.0.1:23447}]" +48:03.009 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23447 [Follower]" leader= +48:03.009 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b42c4ac5-958b-5415-e9e5-2203f63328ab.dc1 127.0.0.1 +48:03.010 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b42c4ac5-958b-5415-e9e5-2203f63328ab 127.0.0.1 +2022-12-04T05:48:03.010Z [INFO] agent.router: Initializing LAN area manager +48:03.010 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23442 network=udp +48:03.010 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b42c4ac5-958b-5415-e9e5-2203f63328ab.dc1 area=wan +48:03.010 [INFO] TestAgent.server: Adding LAN server: server="Node-b42c4ac5-958b-5415-e9e5-2203f63328ab (Addr: tcp/127.0.0.1:23447) (DC: dc1)" +48:03.010 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23442 network=tcp +48:03.011 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23443 network=tcp +48:03.011 [INFO] TestAgent: started state syncer +48:03.011 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23448 network=tcp +48:03.052 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:03.052 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23447 [Candidate]" term=2 +48:03.053 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:03.053 [DEBUG] TestAgent.server.raft: vote granted: from=b42c4ac5-958b-5415-e9e5-2203f63328ab term=2 tally=1 +48:03.053 [INFO] TestAgent.server.raft: election won: tally=1 +48:03.053 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23447 [Leader]" +48:03.053 [INFO] TestAgent.server: cluster leadership acquired +48:03.053 [INFO] TestAgent.server: initializing acls +48:03.053 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:03.053 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:03.054 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:03.054 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:03.054 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:03.054 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:03.054 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b42c4ac5-958b-5415-e9e5-2203f63328ab +48:03.054 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b42c4ac5-958b-5415-e9e5-2203f63328ab.dc1 +48:03.055 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:03.056 [INFO] TestAgent.server: New leader elected: payload=Node-b42c4ac5-958b-5415-e9e5-2203f63328ab +48:03.056 [INFO] TestAgent.server: Updating LAN server: server="Node-b42c4ac5-958b-5415-e9e5-2203f63328ab (Addr: tcp/127.0.0.1:23447) (DC: dc1)" +48:03.056 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b42c4ac5-958b-5415-e9e5-2203f63328ab.dc1 area=wan +48:03.057 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:03.057 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:03.057 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:03.057 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:03.057 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:03.057 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b42c4ac5-958b-5415-e9e5-2203f63328ab +48:03.057 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b42c4ac5-958b-5415-e9e5-2203f63328ab +48:03.057 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b42c4ac5-958b-5415-e9e5-2203f63328ab +48:03.058 [INFO] TestAgent.server: federation state anti-entropy synced +48:03.128 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b42c4ac5-958b-5415-e9e5-2203f63328ab +48:03.128 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b42c4ac5-958b-5415-e9e5-2203f63328ab +48:03.130 [INFO] TestAgent: Requesting shutdown +48:03.130 [INFO] TestAgent.server: shutting down server +48:03.130 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:03.130 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:03.130 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:03.130 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:03.130 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:03.130 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:03.130 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:03.130 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:03.130 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:03.130 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:03.130 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:03.130 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:03.130 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:03.130 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:03.130 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:03.130 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:03.130Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:03.130Z [INFO] agent.router.manager: shutting down +48:03.130 [INFO] TestAgent: consul server down +48:03.130 [INFO] TestAgent: shutdown complete +48:03.130 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23442 network=tcp +48:03.130 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23442 network=udp +48:03.130 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23443 network=tcp +48:03.630 [INFO] TestAgent: Waiting for endpoints to shut down +48:03.630 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_and_sidecar_but_not_sidecar's_overridden_destination -2021-10-31T23:11:45.660Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:45.660Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:45.701Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:45.701Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:45.711 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:da9014d6-2fac-ca1d-d6d4-6622d5cb29b1 Address:127.0.0.1:16307}]" -11:45.711 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-da9014d6-2fac-ca1d-d6d4-6622d5cb29b1.dc1 127.0.0.1 -11:45.712 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-da9014d6-2fac-ca1d-d6d4-6622d5cb29b1 127.0.0.1 -2021-10-31T23:11:45.712Z [INFO] agent.router: Initializing LAN area manager -11:45.712 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16302 network=udp -11:45.712 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16307 [Follower]" leader= -11:45.712 [INFO] TestAgent.server: Adding LAN server: server="Node-da9014d6-2fac-ca1d-d6d4-6622d5cb29b1 (Addr: tcp/127.0.0.1:16307) (DC: dc1)" -11:45.713 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-da9014d6-2fac-ca1d-d6d4-6622d5cb29b1.dc1 area=wan -11:45.713 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16302 network=tcp -11:45.713 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16303 network=tcp -11:45.713 [INFO] TestAgent: started state syncer -11:45.713 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16308 network=tcp -11:45.766 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -11:45.769 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:45.769 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16307 [Candidate]" term=2 -11:45.770 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:45.770 [DEBUG] TestAgent.server.raft: vote granted: from=da9014d6-2fac-ca1d-d6d4-6622d5cb29b1 term=2 tally=1 -11:45.770 [INFO] TestAgent.server.raft: election won: tally=1 -11:45.770 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16307 [Leader]" -11:45.770 [INFO] TestAgent.server: cluster leadership acquired -11:45.770 [INFO] TestAgent.server: initializing acls -11:45.770 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:45.770 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:45.771 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:45.771 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:45.771 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:45.771 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:45.771 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-da9014d6-2fac-ca1d-d6d4-6622d5cb29b1 -11:45.771 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-da9014d6-2fac-ca1d-d6d4-6622d5cb29b1.dc1 -11:45.772 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:45.773 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:45.773 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:45.773 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:45.773 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:45.773 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:45.773 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-da9014d6-2fac-ca1d-d6d4-6622d5cb29b1 -11:45.773 [INFO] TestAgent.server: member joined, marking health alive: member=Node-da9014d6-2fac-ca1d-d6d4-6622d5cb29b1 -11:45.773 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-da9014d6-2fac-ca1d-d6d4-6622d5cb29b1.dc1 area=wan -11:45.773 [INFO] TestAgent.server: New leader elected: payload=Node-da9014d6-2fac-ca1d-d6d4-6622d5cb29b1 -11:45.774 [INFO] TestAgent.server: Updating LAN server: server="Node-da9014d6-2fac-ca1d-d6d4-6622d5cb29b1 (Addr: tcp/127.0.0.1:16307) (DC: dc1)" -11:45.774 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-da9014d6-2fac-ca1d-d6d4-6622d5cb29b1 -11:45.774 [INFO] TestAgent.server: federation state anti-entropy synced -11:46.014 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:46.014 [INFO] TestAgent: Synced node info -11:46.014 [DEBUG] TestAgent: Node info in sync -11:46.184 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-da9014d6-2fac-ca1d-d6d4-6622d5cb29b1 -11:46.184 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-da9014d6-2fac-ca1d-d6d4-6622d5cb29b1 -11:46.186 [INFO] TestAgent: Requesting shutdown -11:46.186 [INFO] TestAgent.server: shutting down server -11:46.186 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:46.186 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:46.186 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:46.186 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:46.186 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:46.186 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:46.186 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:46.186 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:46.186 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:46.186 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:46.186 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:46.186 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:46.186 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:46.186 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:46.186 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:46.186Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:46.186Z [INFO] agent.router.manager: shutting down -11:46.186 [INFO] TestAgent: consul server down -11:46.186 [INFO] TestAgent: shutdown complete -11:46.186 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16302 network=tcp -11:46.186 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16302 network=udp -11:46.186 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16303 network=tcp -11:46.690 [INFO] TestAgent: Waiting for endpoints to shut down -11:46.690 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_not_for_overridden_sidecar -2021-10-31T23:11:46.420Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:46.420Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:46.447Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:46.447Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:46.448 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2ead6fdd-5452-a828-cb08-93edc665e3ee Address:127.0.0.1:16314}]" -11:46.449 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2ead6fdd-5452-a828-cb08-93edc665e3ee.dc1 127.0.0.1 -11:46.449 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2ead6fdd-5452-a828-cb08-93edc665e3ee 127.0.0.1 -2021-10-31T23:11:46.449Z [INFO] agent.router: Initializing LAN area manager -11:46.449 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16314 [Follower]" leader= -11:46.450 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16309 network=udp -11:46.450 [INFO] TestAgent.server: Adding LAN server: server="Node-2ead6fdd-5452-a828-cb08-93edc665e3ee (Addr: tcp/127.0.0.1:16314) (DC: dc1)" -11:46.450 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2ead6fdd-5452-a828-cb08-93edc665e3ee.dc1 area=wan -11:46.450 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16309 network=tcp -11:46.450 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16310 network=tcp -11:46.450 [INFO] TestAgent: started state syncer -11:46.450 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16315 network=tcp -11:46.500 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -11:46.509 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:46.509 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16314 [Candidate]" term=2 -11:46.510 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:46.510 [DEBUG] TestAgent.server.raft: vote granted: from=2ead6fdd-5452-a828-cb08-93edc665e3ee term=2 tally=1 -11:46.510 [INFO] TestAgent.server.raft: election won: tally=1 -11:46.510 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16314 [Leader]" -11:46.510 [INFO] TestAgent.server: cluster leadership acquired -11:46.510 [INFO] TestAgent.server: initializing acls -11:46.510 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:46.510 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:46.511 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:46.511 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:46.511 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:46.511 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:46.511 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-2ead6fdd-5452-a828-cb08-93edc665e3ee -11:46.511 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-2ead6fdd-5452-a828-cb08-93edc665e3ee.dc1 -11:46.512 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:46.514 [INFO] TestAgent.server: New leader elected: payload=Node-2ead6fdd-5452-a828-cb08-93edc665e3ee -11:46.514 [INFO] TestAgent.server: Updating LAN server: server="Node-2ead6fdd-5452-a828-cb08-93edc665e3ee (Addr: tcp/127.0.0.1:16314) (DC: dc1)" -11:46.514 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-2ead6fdd-5452-a828-cb08-93edc665e3ee.dc1 area=wan -11:46.516 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:46.516 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:46.516 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:46.516 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:46.516 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:46.516 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2ead6fdd-5452-a828-cb08-93edc665e3ee -11:46.516 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2ead6fdd-5452-a828-cb08-93edc665e3ee -11:46.516 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2ead6fdd-5452-a828-cb08-93edc665e3ee -11:46.517 [INFO] TestAgent.server: federation state anti-entropy synced -11:46.569 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-2ead6fdd-5452-a828-cb08-93edc665e3ee -11:46.570 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-2ead6fdd-5452-a828-cb08-93edc665e3ee -11:46.571 [INFO] TestAgent: Requesting shutdown -11:46.571 [INFO] TestAgent.server: shutting down server -11:46.571 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:46.571 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:46.571 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:46.571 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:46.571 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:46.571 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:46.571 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:46.571 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:46.572 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:46.572 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:46.572 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:46.572 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:46.572 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:46.572 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:46.572 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:46.572 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:46.572Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:46.572Z [INFO] agent.router.manager: shutting down -11:46.572 [INFO] TestAgent: consul server down -11:46.572 [INFO] TestAgent: shutdown complete -11:46.572 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16309 network=tcp -11:46.572 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16309 network=udp -11:46.572 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16310 network=tcp -11:47.072 [INFO] TestAgent: Waiting for endpoints to shut down -11:47.072 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:02.931Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:02.931Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:02.959Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:02.959Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:02.960 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:12b87779-995e-2f39-f8f6-b3a3e7aa7764 Address:127.0.0.1:23440}]" +48:02.961 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23440 [Follower]" leader= +48:02.961 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-12b87779-995e-2f39-f8f6-b3a3e7aa7764.dc1 127.0.0.1 +48:02.961 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-12b87779-995e-2f39-f8f6-b3a3e7aa7764 127.0.0.1 +2022-12-04T05:48:02.961Z [INFO] agent.router: Initializing LAN area manager +48:02.962 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23435 network=udp +48:02.962 [INFO] TestAgent.server: Adding LAN server: server="Node-12b87779-995e-2f39-f8f6-b3a3e7aa7764 (Addr: tcp/127.0.0.1:23440) (DC: dc1)" +48:02.962 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-12b87779-995e-2f39-f8f6-b3a3e7aa7764.dc1 area=wan +48:02.962 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23435 network=tcp +48:02.963 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23436 network=tcp +48:02.963 [INFO] TestAgent: started state syncer +48:02.963 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23441 network=tcp +48:03.012 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:03.028 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:03.028 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23440 [Candidate]" term=2 +48:03.028 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:03.028 [DEBUG] TestAgent.server.raft: vote granted: from=12b87779-995e-2f39-f8f6-b3a3e7aa7764 term=2 tally=1 +48:03.028 [INFO] TestAgent.server.raft: election won: tally=1 +48:03.028 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23440 [Leader]" +48:03.028 [INFO] TestAgent.server: cluster leadership acquired +48:03.028 [INFO] TestAgent.server: initializing acls +48:03.029 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:03.029 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:03.028 [INFO] TestAgent.server: New leader elected: payload=Node-12b87779-995e-2f39-f8f6-b3a3e7aa7764 +48:03.029 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:03.030 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:03.030 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:03.030 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:03.030 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-12b87779-995e-2f39-f8f6-b3a3e7aa7764 +48:03.030 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-12b87779-995e-2f39-f8f6-b3a3e7aa7764.dc1 +48:03.031 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:03.032 [INFO] TestAgent.server: Updating LAN server: server="Node-12b87779-995e-2f39-f8f6-b3a3e7aa7764 (Addr: tcp/127.0.0.1:23440) (DC: dc1)" +48:03.032 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:03.032 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:03.032 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-12b87779-995e-2f39-f8f6-b3a3e7aa7764.dc1 area=wan +48:03.032 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:03.032 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:03.032 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:03.032 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-12b87779-995e-2f39-f8f6-b3a3e7aa7764 +48:03.032 [INFO] TestAgent.server: member joined, marking health alive: member=Node-12b87779-995e-2f39-f8f6-b3a3e7aa7764 +48:03.033 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-12b87779-995e-2f39-f8f6-b3a3e7aa7764 +48:03.033 [INFO] TestAgent.server: federation state anti-entropy synced +48:03.251 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-12b87779-995e-2f39-f8f6-b3a3e7aa7764 +48:03.251 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-12b87779-995e-2f39-f8f6-b3a3e7aa7764 +48:03.253 [INFO] TestAgent: Requesting shutdown +48:03.253 [INFO] TestAgent.server: shutting down server +48:03.253 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:03.253 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:03.253 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:03.253 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:03.253 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:03.253 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:03.253 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:03.253 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:03.253 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:03.253 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:03.253 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:03.253 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:03.253 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:03.253 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:03.253 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:03.253 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:03.253Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:03.253Z [INFO] agent.router.manager: shutting down +48:03.253 [INFO] TestAgent: consul server down +48:03.253 [INFO] TestAgent: shutdown complete +48:03.253 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23435 network=tcp +48:03.254 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23435 network=udp +48:03.254 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23436 network=tcp +48:03.754 [INFO] TestAgent: Waiting for endpoints to shut down +48:03.754 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_and_sidecar_but_not_sidecar's_overridden_destination +2022-12-04T05:48:03.645Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:03.645Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:03.658Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:03.658Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:03.660 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a29b34a1-f8e6-a687-77c7-462c9e643ec5 Address:127.0.0.1:23454}]" +48:03.660 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23454 [Follower]" leader= +48:03.660 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a29b34a1-f8e6-a687-77c7-462c9e643ec5.dc1 127.0.0.1 +48:03.661 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a29b34a1-f8e6-a687-77c7-462c9e643ec5 127.0.0.1 +2022-12-04T05:48:03.661Z [INFO] agent.router: Initializing LAN area manager +48:03.661 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23449 network=udp +48:03.661 [INFO] TestAgent.server: Adding LAN server: server="Node-a29b34a1-f8e6-a687-77c7-462c9e643ec5 (Addr: tcp/127.0.0.1:23454) (DC: dc1)" +48:03.662 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a29b34a1-f8e6-a687-77c7-462c9e643ec5.dc1 area=wan +48:03.662 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23449 network=tcp +48:03.662 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23450 network=tcp +48:03.662 [INFO] TestAgent: started state syncer +48:03.662 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23455 network=tcp +48:03.710 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:03.710 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23454 [Candidate]" term=2 +48:03.710 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:03.710 [DEBUG] TestAgent.server.raft: vote granted: from=a29b34a1-f8e6-a687-77c7-462c9e643ec5 term=2 tally=1 +48:03.710 [INFO] TestAgent.server.raft: election won: tally=1 +48:03.710 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23454 [Leader]" +48:03.711 [INFO] TestAgent.server: cluster leadership acquired +48:03.711 [INFO] TestAgent.server: initializing acls +48:03.711 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:03.711 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:03.711 [INFO] TestAgent.server: New leader elected: payload=Node-a29b34a1-f8e6-a687-77c7-462c9e643ec5 +48:03.711 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:03.712 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:03.712 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:03.712 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:03.712 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a29b34a1-f8e6-a687-77c7-462c9e643ec5 +48:03.712 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a29b34a1-f8e6-a687-77c7-462c9e643ec5.dc1 +48:03.712 [INFO] TestAgent.server: Updating LAN server: server="Node-a29b34a1-f8e6-a687-77c7-462c9e643ec5 (Addr: tcp/127.0.0.1:23454) (DC: dc1)" +48:03.712 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a29b34a1-f8e6-a687-77c7-462c9e643ec5.dc1 area=wan +48:03.712 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:03.714 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:03.714 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:03.714 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:03.714 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:03.714 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:03.714 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a29b34a1-f8e6-a687-77c7-462c9e643ec5 +48:03.714 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a29b34a1-f8e6-a687-77c7-462c9e643ec5 +48:03.714 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a29b34a1-f8e6-a687-77c7-462c9e643ec5 +48:03.714 [INFO] TestAgent.server: federation state anti-entropy synced +48:03.937 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:03.938 [INFO] TestAgent: Synced node info +48:03.938 [DEBUG] TestAgent: Node info in sync +48:04.025 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a29b34a1-f8e6-a687-77c7-462c9e643ec5 +48:04.025 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a29b34a1-f8e6-a687-77c7-462c9e643ec5 +48:04.027 [INFO] TestAgent: Requesting shutdown +48:04.027 [INFO] TestAgent.server: shutting down server +48:04.027 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:04.027 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:04.027 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:04.027 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:04.027 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:04.027 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:04.027 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:04.027 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:04.027 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:04.027 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:04.027 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:04.027 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:04.027 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:04.027 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:04.027 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:04.027Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:04.027Z [INFO] agent.router.manager: shutting down +48:04.027 [INFO] TestAgent: consul server down +48:04.027 [INFO] TestAgent: shutdown complete +48:04.027 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23449 network=tcp +48:04.027 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23449 network=udp +48:04.027 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23450 network=tcp +48:04.528 [INFO] TestAgent: Waiting for endpoints to shut down +48:04.528 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_not_for_overridden_sidecar -2021-10-31T23:11:46.722Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:46.723Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:46.756Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:46.756Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:46.757 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e9017108-7f73-854b-1059-4016318389bb Address:127.0.0.1:16321}]" -11:46.757 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16321 [Follower]" leader= -11:46.758 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e9017108-7f73-854b-1059-4016318389bb.dc1 127.0.0.1 -11:46.758 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e9017108-7f73-854b-1059-4016318389bb 127.0.0.1 -2021-10-31T23:11:46.758Z [INFO] agent.router: Initializing LAN area manager -11:46.758 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16316 network=udp -11:46.759 [INFO] TestAgent.server: Adding LAN server: server="Node-e9017108-7f73-854b-1059-4016318389bb (Addr: tcp/127.0.0.1:16321) (DC: dc1)" -11:46.759 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e9017108-7f73-854b-1059-4016318389bb.dc1 area=wan -11:46.759 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16316 network=tcp -11:46.759 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16317 network=tcp -11:46.759 [INFO] TestAgent: started state syncer -11:46.759 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16322 network=tcp -11:46.809 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -11:46.826 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:46.826 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16321 [Candidate]" term=2 -11:46.826 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:46.826 [DEBUG] TestAgent.server.raft: vote granted: from=e9017108-7f73-854b-1059-4016318389bb term=2 tally=1 -11:46.826 [INFO] TestAgent.server.raft: election won: tally=1 -11:46.826 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16321 [Leader]" -11:46.826 [INFO] TestAgent.server: cluster leadership acquired -11:46.826 [INFO] TestAgent.server: initializing acls -11:46.826 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:46.826 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:46.827 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:46.827 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:46.827 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:46.827 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:46.827 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e9017108-7f73-854b-1059-4016318389bb -11:46.827 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e9017108-7f73-854b-1059-4016318389bb.dc1 -11:46.828 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:46.829 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:46.829 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:46.829 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:46.829 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:46.829 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:46.829 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e9017108-7f73-854b-1059-4016318389bb -11:46.829 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e9017108-7f73-854b-1059-4016318389bb -11:46.829 [INFO] TestAgent.server: New leader elected: payload=Node-e9017108-7f73-854b-1059-4016318389bb -11:46.829 [INFO] TestAgent.server: Updating LAN server: server="Node-e9017108-7f73-854b-1059-4016318389bb (Addr: tcp/127.0.0.1:16321) (DC: dc1)" -11:46.829 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e9017108-7f73-854b-1059-4016318389bb.dc1 area=wan -11:46.830 [INFO] TestAgent.server: federation state anti-entropy synced -11:46.830 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e9017108-7f73-854b-1059-4016318389bb -11:46.985 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e9017108-7f73-854b-1059-4016318389bb -11:46.986 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e9017108-7f73-854b-1059-4016318389bb -11:46.987 [INFO] TestAgent: Requesting shutdown -11:46.988 [INFO] TestAgent.server: shutting down server -11:46.988 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:46.988 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:46.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:46.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:46.988 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:46.988 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:46.988 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:46.988 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:46.988 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:46.988 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:46.988 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:46.988 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:46.988 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:46.988 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:46.988 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:46.988 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:46.988Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:46.988Z [INFO] agent.router.manager: shutting down -11:46.988 [INFO] TestAgent: consul server down -11:46.988 [INFO] TestAgent: shutdown complete -11:46.988 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16316 network=tcp -11:46.988 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16316 network=udp -11:46.988 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16317 network=tcp -11:47.489 [INFO] TestAgent: Waiting for endpoints to shut down -11:47.489 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_and_overridden_for_sidecar -2021-10-31T23:11:47.098Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:47.098Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:47.136Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:47.136Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:47.138 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:95afa584-2a2b-839f-61bd-267e7122fb00 Address:127.0.0.1:16328}]" -11:47.139 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-95afa584-2a2b-839f-61bd-267e7122fb00.dc1 127.0.0.1 -11:47.139 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-95afa584-2a2b-839f-61bd-267e7122fb00 127.0.0.1 -2021-10-31T23:11:47.139Z [INFO] agent.router: Initializing LAN area manager -11:47.139 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16323 network=udp -11:47.139 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16328 [Follower]" leader= -11:47.140 [INFO] TestAgent.server: Adding LAN server: server="Node-95afa584-2a2b-839f-61bd-267e7122fb00 (Addr: tcp/127.0.0.1:16328) (DC: dc1)" -11:47.140 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-95afa584-2a2b-839f-61bd-267e7122fb00.dc1 area=wan -11:47.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16323 network=tcp -11:47.140 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16324 network=tcp -11:47.141 [INFO] TestAgent: started state syncer -11:47.141 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16329 network=tcp -11:47.192 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -11:47.209 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:47.209 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16328 [Candidate]" term=2 -11:47.210 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:47.210 [DEBUG] TestAgent.server.raft: vote granted: from=95afa584-2a2b-839f-61bd-267e7122fb00 term=2 tally=1 -11:47.210 [INFO] TestAgent.server.raft: election won: tally=1 -11:47.210 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16328 [Leader]" -11:47.210 [INFO] TestAgent.server: cluster leadership acquired -11:47.210 [INFO] TestAgent.server: initializing acls -11:47.210 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:47.210 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:47.211 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:47.211 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:47.211 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:47.211 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:47.211 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-95afa584-2a2b-839f-61bd-267e7122fb00 -11:47.211 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-95afa584-2a2b-839f-61bd-267e7122fb00.dc1 -11:47.212 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:47.213 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:47.213 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:47.213 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:47.213 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:47.213 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:47.213 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-95afa584-2a2b-839f-61bd-267e7122fb00 -11:47.213 [INFO] TestAgent.server: member joined, marking health alive: member=Node-95afa584-2a2b-839f-61bd-267e7122fb00 -11:47.213 [INFO] TestAgent.server: New leader elected: payload=Node-95afa584-2a2b-839f-61bd-267e7122fb00 -11:47.214 [INFO] TestAgent.server: Updating LAN server: server="Node-95afa584-2a2b-839f-61bd-267e7122fb00 (Addr: tcp/127.0.0.1:16328) (DC: dc1)" -11:47.214 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-95afa584-2a2b-839f-61bd-267e7122fb00.dc1 area=wan -11:47.218 [INFO] TestAgent.server: federation state anti-entropy synced -11:47.218 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-95afa584-2a2b-839f-61bd-267e7122fb00 -11:47.386 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:47.386 [INFO] TestAgent: Synced node info -11:47.386 [DEBUG] TestAgent: Node info in sync -11:47.493 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-95afa584-2a2b-839f-61bd-267e7122fb00 -11:47.494 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-95afa584-2a2b-839f-61bd-267e7122fb00 -11:47.495 [INFO] TestAgent: Requesting shutdown -11:47.495 [INFO] TestAgent.server: shutting down server -11:47.495 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:47.495 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:47.495 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:47.495 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:47.495 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:47.495 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:47.495 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:47.495 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:47.495 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:47.495 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:47.495 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:47.495 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:47.495 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:47.495 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:47.495 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:47.495Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:47.495Z [INFO] agent.router.manager: shutting down -11:47.496 [INFO] TestAgent: consul server down -11:47.496 [INFO] TestAgent: shutdown complete -11:47.496 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16323 network=tcp -11:47.496 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16323 network=udp -11:47.496 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16324 network=tcp -11:47.996 [INFO] TestAgent: Waiting for endpoints to shut down -11:47.996 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:03.778Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:03.778Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:03.793Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:03.793Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:03.795 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6ab5deca-6166-0bea-1e86-1ce1d88e7e02 Address:127.0.0.1:23461}]" +48:03.795 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23461 [Follower]" leader= +48:03.795 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6ab5deca-6166-0bea-1e86-1ce1d88e7e02.dc1 127.0.0.1 +48:03.796 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6ab5deca-6166-0bea-1e86-1ce1d88e7e02 127.0.0.1 +2022-12-04T05:48:03.796Z [INFO] agent.router: Initializing LAN area manager +48:03.796 [INFO] TestAgent.server: Adding LAN server: server="Node-6ab5deca-6166-0bea-1e86-1ce1d88e7e02 (Addr: tcp/127.0.0.1:23461) (DC: dc1)" +48:03.796 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6ab5deca-6166-0bea-1e86-1ce1d88e7e02.dc1 area=wan +48:03.796 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23456 network=udp +48:03.796 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23456 network=tcp +48:03.797 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23457 network=tcp +48:03.797 [INFO] TestAgent: started state syncer +48:03.797 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23462 network=tcp +48:03.834 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:03.835 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23461 [Candidate]" term=2 +48:03.835 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:03.835 [DEBUG] TestAgent.server.raft: vote granted: from=6ab5deca-6166-0bea-1e86-1ce1d88e7e02 term=2 tally=1 +48:03.835 [INFO] TestAgent.server.raft: election won: tally=1 +48:03.835 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23461 [Leader]" +48:03.835 [INFO] TestAgent.server: cluster leadership acquired +48:03.835 [INFO] TestAgent.server: initializing acls +48:03.835 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:03.835 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:03.836 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:03.836 [INFO] TestAgent.server: New leader elected: payload=Node-6ab5deca-6166-0bea-1e86-1ce1d88e7e02 +48:03.836 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:03.836 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:03.836 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:03.836 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-6ab5deca-6166-0bea-1e86-1ce1d88e7e02 +48:03.836 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-6ab5deca-6166-0bea-1e86-1ce1d88e7e02.dc1 +48:03.837 [INFO] TestAgent.server: Updating LAN server: server="Node-6ab5deca-6166-0bea-1e86-1ce1d88e7e02 (Addr: tcp/127.0.0.1:23461) (DC: dc1)" +48:03.837 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-6ab5deca-6166-0bea-1e86-1ce1d88e7e02.dc1 area=wan +48:03.837 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:03.838 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:03.838 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:03.838 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:03.838 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:03.838 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:03.838 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6ab5deca-6166-0bea-1e86-1ce1d88e7e02 +48:03.838 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6ab5deca-6166-0bea-1e86-1ce1d88e7e02 +48:03.839 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6ab5deca-6166-0bea-1e86-1ce1d88e7e02 +48:03.839 [INFO] TestAgent.server: federation state anti-entropy synced +48:04.120 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:04.121 [INFO] TestAgent: Synced node info +48:04.208 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-6ab5deca-6166-0bea-1e86-1ce1d88e7e02 +48:04.208 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-6ab5deca-6166-0bea-1e86-1ce1d88e7e02 +48:04.209 [INFO] TestAgent: Requesting shutdown +48:04.209 [INFO] TestAgent.server: shutting down server +48:04.209 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:04.209 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:04.209 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:04.209 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:04.209 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:04.209 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:04.209 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:04.209 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:04.209 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:04.209 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:04.209 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:04.209 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:04.209 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:04.209 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:04.209 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:04.210Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:04.210Z [INFO] agent.router.manager: shutting down +48:04.210 [INFO] TestAgent: consul server down +48:04.210 [INFO] TestAgent: shutdown complete +48:04.210 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23456 network=tcp +48:04.210 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23456 network=udp +48:04.210 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23457 network=tcp +48:04.712 [INFO] TestAgent: Waiting for endpoints to shut down +48:04.712 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_not_for_overridden_sidecar +2022-12-04T05:48:04.541Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:04.541Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:04.553Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:04.553Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:04.554 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:53522ecd-db63-e975-00a9-137afb10e3af Address:127.0.0.1:23468}]" +48:04.555 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23468 [Follower]" leader= +48:04.555 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-53522ecd-db63-e975-00a9-137afb10e3af.dc1 127.0.0.1 +48:04.555 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-53522ecd-db63-e975-00a9-137afb10e3af 127.0.0.1 +2022-12-04T05:48:04.555Z [INFO] agent.router: Initializing LAN area manager +48:04.556 [INFO] TestAgent.server: Adding LAN server: server="Node-53522ecd-db63-e975-00a9-137afb10e3af (Addr: tcp/127.0.0.1:23468) (DC: dc1)" +48:04.556 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23463 network=udp +48:04.556 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-53522ecd-db63-e975-00a9-137afb10e3af.dc1 area=wan +48:04.556 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23463 network=tcp +48:04.556 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23464 network=tcp +48:04.556 [INFO] TestAgent: started state syncer +48:04.556 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23469 network=tcp +48:04.597 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:04.597 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23468 [Candidate]" term=2 +48:04.597 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:04.597 [DEBUG] TestAgent.server.raft: vote granted: from=53522ecd-db63-e975-00a9-137afb10e3af term=2 tally=1 +48:04.597 [INFO] TestAgent.server.raft: election won: tally=1 +48:04.597 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23468 [Leader]" +48:04.597 [INFO] TestAgent.server: cluster leadership acquired +48:04.598 [INFO] TestAgent.server: New leader elected: payload=Node-53522ecd-db63-e975-00a9-137afb10e3af +48:04.598 [INFO] TestAgent.server: initializing acls +48:04.598 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:04.598 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:04.598 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:04.598 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:04.598 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:04.598 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:04.599 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-53522ecd-db63-e975-00a9-137afb10e3af +48:04.599 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-53522ecd-db63-e975-00a9-137afb10e3af.dc1 +48:04.599 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:04.599 [INFO] TestAgent.server: Updating LAN server: server="Node-53522ecd-db63-e975-00a9-137afb10e3af (Addr: tcp/127.0.0.1:23468) (DC: dc1)" +48:04.600 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-53522ecd-db63-e975-00a9-137afb10e3af.dc1 area=wan +48:04.601 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:04.601 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:04.601 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:04.601 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:04.601 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:04.601 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-53522ecd-db63-e975-00a9-137afb10e3af +48:04.601 [INFO] TestAgent.server: member joined, marking health alive: member=Node-53522ecd-db63-e975-00a9-137afb10e3af +48:04.601 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-53522ecd-db63-e975-00a9-137afb10e3af +48:04.601 [INFO] TestAgent.server: federation state anti-entropy synced +48:04.717 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-53522ecd-db63-e975-00a9-137afb10e3af +48:04.719 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-53522ecd-db63-e975-00a9-137afb10e3af +48:04.722 [INFO] TestAgent: Requesting shutdown +48:04.722 [INFO] TestAgent.server: shutting down server +48:04.722 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:04.722 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:04.722 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:04.722 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:04.722 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:04.722 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:04.722 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:04.722 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:04.722 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:04.722 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:04.722 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:04.722 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:04.722 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:04.722 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:04.722 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:04.722 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:04.722Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:04.722Z [INFO] agent.router.manager: shutting down +48:04.722 [INFO] TestAgent: consul server down +48:04.722 [INFO] TestAgent: shutdown complete +48:04.722 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23463 network=tcp +48:04.722 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23463 network=udp +48:04.722 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23464 network=tcp +48:05.224 [INFO] TestAgent: Waiting for endpoints to shut down +48:05.224 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_and_overridden_for_sidecar -2021-10-31T23:11:47.533Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:47.533Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:47.565Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:47.565Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:47.567 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2794ea4d-f63d-4d64-3222-cc86dd57176b Address:127.0.0.1:16335}]" -11:47.567 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2794ea4d-f63d-4d64-3222-cc86dd57176b.dc1 127.0.0.1 -11:47.568 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2794ea4d-f63d-4d64-3222-cc86dd57176b 127.0.0.1 -2021-10-31T23:11:47.568Z [INFO] agent.router: Initializing LAN area manager -11:47.568 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16330 network=udp -11:47.568 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16335 [Follower]" leader= -11:47.568 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2794ea4d-f63d-4d64-3222-cc86dd57176b.dc1 area=wan -11:47.568 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16330 network=tcp -11:47.568 [INFO] TestAgent.server: Adding LAN server: server="Node-2794ea4d-f63d-4d64-3222-cc86dd57176b (Addr: tcp/127.0.0.1:16335) (DC: dc1)" -11:47.569 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16331 network=tcp -11:47.569 [INFO] TestAgent: started state syncer -11:47.569 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16336 network=tcp -11:47.609 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:47.609 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16335 [Candidate]" term=2 -11:47.610 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:47.610 [DEBUG] TestAgent.server.raft: vote granted: from=2794ea4d-f63d-4d64-3222-cc86dd57176b term=2 tally=1 -11:47.610 [INFO] TestAgent.server.raft: election won: tally=1 -11:47.610 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16335 [Leader]" -11:47.610 [INFO] TestAgent.server: cluster leadership acquired -11:47.610 [INFO] TestAgent.server: initializing acls -11:47.610 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:47.610 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:47.610 [INFO] TestAgent.server: New leader elected: payload=Node-2794ea4d-f63d-4d64-3222-cc86dd57176b -11:47.611 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:47.611 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:47.611 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:47.611 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:47.611 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-2794ea4d-f63d-4d64-3222-cc86dd57176b -11:47.611 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-2794ea4d-f63d-4d64-3222-cc86dd57176b.dc1 -11:47.612 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:47.612 [INFO] TestAgent.server: Updating LAN server: server="Node-2794ea4d-f63d-4d64-3222-cc86dd57176b (Addr: tcp/127.0.0.1:16335) (DC: dc1)" -11:47.613 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-2794ea4d-f63d-4d64-3222-cc86dd57176b.dc1 area=wan -11:47.613 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:47.613 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:47.613 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:47.613 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:47.613 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:47.613 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2794ea4d-f63d-4d64-3222-cc86dd57176b -11:47.613 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2794ea4d-f63d-4d64-3222-cc86dd57176b -11:47.614 [INFO] TestAgent.server: federation state anti-entropy synced -11:47.614 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2794ea4d-f63d-4d64-3222-cc86dd57176b -11:47.670 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:47.670 [INFO] TestAgent: Synced node info -11:47.990 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-2794ea4d-f63d-4d64-3222-cc86dd57176b -11:47.990 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-2794ea4d-f63d-4d64-3222-cc86dd57176b -11:47.992 [DEBUG] TestAgent: Node info in sync -11:47.993 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -11:47.993 [INFO] TestAgent: Synced service: service=web -11:47.993 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 -11:47.993 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 -11:48.002 [DEBUG] TestAgent: removed service: service=web -11:48.002 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 -11:48.002 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 -11:48.002 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -11:48.002 [DEBUG] TestAgent: Node info in sync -11:48.002 [INFO] TestAgent: Deregistered service: service=web -11:48.003 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -11:48.003 [INFO] TestAgent: Requesting shutdown -11:48.003 [INFO] TestAgent.server: shutting down server -11:48.003 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:48.003 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:48.003 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:48.003 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:48.003 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:48.003 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:48.003 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:48.003 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:48.003 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:48.003 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:48.003 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:48.003 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:48.003 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:48.003 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:48.003 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:48.003Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:48.003Z [INFO] agent.router.manager: shutting down -11:48.003 [INFO] TestAgent: consul server down -11:48.003 [INFO] TestAgent: shutdown complete -11:48.003 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16330 network=tcp -11:48.004 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16330 network=udp -11:48.004 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16331 network=tcp -11:48.504 [INFO] TestAgent: Waiting for endpoints to shut down -11:48.504 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:04.734Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:04.734Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:04.747Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:04.747Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:04.749 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:75e71f48-d142-197d-9ae1-0a61ab7f91d0 Address:127.0.0.1:23475}]" +48:04.749 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-75e71f48-d142-197d-9ae1-0a61ab7f91d0.dc1 127.0.0.1 +48:04.750 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-75e71f48-d142-197d-9ae1-0a61ab7f91d0 127.0.0.1 +2022-12-04T05:48:04.750Z [INFO] agent.router: Initializing LAN area manager +48:04.750 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23470 network=udp +48:04.750 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23475 [Follower]" leader= +48:04.750 [INFO] TestAgent.server: Adding LAN server: server="Node-75e71f48-d142-197d-9ae1-0a61ab7f91d0 (Addr: tcp/127.0.0.1:23475) (DC: dc1)" +48:04.750 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-75e71f48-d142-197d-9ae1-0a61ab7f91d0.dc1 area=wan +48:04.751 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23470 network=tcp +48:04.751 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23471 network=tcp +48:04.751 [INFO] TestAgent: started state syncer +48:04.751 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23476 network=tcp +48:04.802 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:04.806 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:04.806 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23475 [Candidate]" term=2 +48:04.806 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:04.806 [DEBUG] TestAgent.server.raft: vote granted: from=75e71f48-d142-197d-9ae1-0a61ab7f91d0 term=2 tally=1 +48:04.806 [INFO] TestAgent.server.raft: election won: tally=1 +48:04.806 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23475 [Leader]" +48:04.806 [INFO] TestAgent.server: cluster leadership acquired +48:04.807 [INFO] TestAgent.server: initializing acls +48:04.807 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:04.807 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:04.807 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:04.808 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:04.808 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:04.808 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:04.808 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-75e71f48-d142-197d-9ae1-0a61ab7f91d0 +48:04.808 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-75e71f48-d142-197d-9ae1-0a61ab7f91d0.dc1 +48:04.808 [INFO] TestAgent.server: New leader elected: payload=Node-75e71f48-d142-197d-9ae1-0a61ab7f91d0 +48:04.808 [INFO] TestAgent.server: Updating LAN server: server="Node-75e71f48-d142-197d-9ae1-0a61ab7f91d0 (Addr: tcp/127.0.0.1:23475) (DC: dc1)" +48:04.808 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-75e71f48-d142-197d-9ae1-0a61ab7f91d0.dc1 area=wan +48:04.808 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:04.810 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:04.810 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:04.810 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:04.810 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:04.810 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:04.810 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-75e71f48-d142-197d-9ae1-0a61ab7f91d0 +48:04.810 [INFO] TestAgent.server: member joined, marking health alive: member=Node-75e71f48-d142-197d-9ae1-0a61ab7f91d0 +48:04.811 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-75e71f48-d142-197d-9ae1-0a61ab7f91d0 +48:04.811 [INFO] TestAgent.server: federation state anti-entropy synced +48:04.831 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-75e71f48-d142-197d-9ae1-0a61ab7f91d0 +48:04.831 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-75e71f48-d142-197d-9ae1-0a61ab7f91d0 +48:04.833 [INFO] TestAgent: Requesting shutdown +48:04.833 [INFO] TestAgent.server: shutting down server +48:04.833 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:04.833 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:04.833 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:04.833 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:04.833 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:04.833 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:04.833 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:04.833 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:04.833 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:04.833 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:04.833 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:04.833 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:04.833 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:04.833 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:04.833 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:04.833 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:04.833Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:04.833Z [INFO] agent.router.manager: shutting down +48:04.833 [INFO] TestAgent: consul server down +48:04.833 [INFO] TestAgent: shutdown complete +48:04.833 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23470 network=tcp +48:04.833 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23470 network=udp +48:04.833 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23471 network=tcp +48:05.336 [INFO] TestAgent: Waiting for endpoints to shut down +48:05.336 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_and_overridden_for_sidecar +2022-12-04T05:48:05.358Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:05.358Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:05.388Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:05.388Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:05.389 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b5a10d26-96ba-91a0-b9f3-98f22487127b Address:127.0.0.1:23489}]" +48:05.390 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23489 [Follower]" leader= +48:05.390 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b5a10d26-96ba-91a0-b9f3-98f22487127b.dc1 127.0.0.1 +48:05.391 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b5a10d26-96ba-91a0-b9f3-98f22487127b 127.0.0.1 +2022-12-04T05:48:05.391Z [INFO] agent.router: Initializing LAN area manager +48:05.391 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23484 network=udp +48:05.391 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b5a10d26-96ba-91a0-b9f3-98f22487127b.dc1 area=wan +48:05.391 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23484 network=tcp +48:05.391 [INFO] TestAgent.server: Adding LAN server: server="Node-b5a10d26-96ba-91a0-b9f3-98f22487127b (Addr: tcp/127.0.0.1:23489) (DC: dc1)" +48:05.392 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23485 network=tcp +48:05.400 [INFO] TestAgent: started state syncer +48:05.400 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23490 network=tcp +48:05.442 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:05.457 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:05.457 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23489 [Candidate]" term=2 +48:05.457 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:05.457 [DEBUG] TestAgent.server.raft: vote granted: from=b5a10d26-96ba-91a0-b9f3-98f22487127b term=2 tally=1 +48:05.457 [INFO] TestAgent.server.raft: election won: tally=1 +48:05.457 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23489 [Leader]" +48:05.458 [INFO] TestAgent.server: cluster leadership acquired +48:05.458 [INFO] TestAgent.server: initializing acls +48:05.458 [INFO] TestAgent.server: New leader elected: payload=Node-b5a10d26-96ba-91a0-b9f3-98f22487127b +48:05.458 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:05.458 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:05.458 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:05.459 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:05.459 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:05.459 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:05.459 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b5a10d26-96ba-91a0-b9f3-98f22487127b +48:05.459 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b5a10d26-96ba-91a0-b9f3-98f22487127b.dc1 +48:05.459 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:05.460 [INFO] TestAgent.server: Updating LAN server: server="Node-b5a10d26-96ba-91a0-b9f3-98f22487127b (Addr: tcp/127.0.0.1:23489) (DC: dc1)" +48:05.460 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b5a10d26-96ba-91a0-b9f3-98f22487127b.dc1 area=wan +48:05.461 [INFO] TestAgent: Synced node info +48:05.461 [DEBUG] TestAgent: Node info in sync +48:05.464 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:05.464 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:05.464 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:05.464 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:05.464 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:05.464 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b5a10d26-96ba-91a0-b9f3-98f22487127b +48:05.464 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b5a10d26-96ba-91a0-b9f3-98f22487127b +48:05.464 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b5a10d26-96ba-91a0-b9f3-98f22487127b +48:05.464 [INFO] TestAgent.server: federation state anti-entropy synced +48:05.546 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b5a10d26-96ba-91a0-b9f3-98f22487127b +48:05.546 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b5a10d26-96ba-91a0-b9f3-98f22487127b +48:05.548 [DEBUG] TestAgent: Node info in sync +48:05.549 [INFO] TestAgent: Synced service: service=web +48:05.549 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +48:05.549 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +48:05.549 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +48:05.550 [DEBUG] TestAgent: removed service: service=web +48:05.550 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 +48:05.550 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 +48:05.550 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +48:05.550 [DEBUG] TestAgent: Node info in sync +48:05.550 [INFO] TestAgent: Deregistered service: service=web +48:05.551 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +48:05.551 [INFO] TestAgent: Requesting shutdown +48:05.551 [INFO] TestAgent.server: shutting down server +48:05.551 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:05.551 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:05.551 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:05.551 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:05.551 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:05.551 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:05.551 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:05.551 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:05.551 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:05.551 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:05.551 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:05.551 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:05.551 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:05.551 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:05.551 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:05.551Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:05.551Z [INFO] agent.router.manager: shutting down +48:05.551 [INFO] TestAgent: consul server down +48:05.551 [INFO] TestAgent: shutdown complete +48:05.552 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23484 network=tcp +48:05.552 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23484 network=udp +48:05.552 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23485 network=tcp +48:06.052 [INFO] TestAgent: Waiting for endpoints to shut down +48:06.052 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:05.246Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:05.246Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:05.260Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:05.260Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:05.273 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:acefb08e-8207-99f7-1bea-a20a64ea9bdd Address:127.0.0.1:23482}]" +48:05.273 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23482 [Follower]" leader= +48:05.274 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-acefb08e-8207-99f7-1bea-a20a64ea9bdd.dc1 127.0.0.1 +48:05.275 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-acefb08e-8207-99f7-1bea-a20a64ea9bdd 127.0.0.1 +2022-12-04T05:48:05.275Z [INFO] agent.router: Initializing LAN area manager +48:05.275 [INFO] TestAgent.server: Adding LAN server: server="Node-acefb08e-8207-99f7-1bea-a20a64ea9bdd (Addr: tcp/127.0.0.1:23482) (DC: dc1)" +48:05.275 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-acefb08e-8207-99f7-1bea-a20a64ea9bdd.dc1 area=wan +48:05.275 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23477 network=udp +48:05.275 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23477 network=tcp +48:05.280 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23478 network=tcp +48:05.280 [INFO] TestAgent: started state syncer +48:05.280 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23483 network=tcp +48:05.320 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:05.320 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23482 [Candidate]" term=2 +48:05.320 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:05.320 [DEBUG] TestAgent.server.raft: vote granted: from=acefb08e-8207-99f7-1bea-a20a64ea9bdd term=2 tally=1 +48:05.320 [INFO] TestAgent.server.raft: election won: tally=1 +48:05.320 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23482 [Leader]" +48:05.320 [INFO] TestAgent.server: cluster leadership acquired +48:05.321 [INFO] TestAgent.server: New leader elected: payload=Node-acefb08e-8207-99f7-1bea-a20a64ea9bdd +48:05.321 [INFO] TestAgent.server: initializing acls +48:05.321 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:05.321 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:05.322 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:05.322 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:05.322 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:05.322 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:05.322 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-acefb08e-8207-99f7-1bea-a20a64ea9bdd +48:05.322 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-acefb08e-8207-99f7-1bea-a20a64ea9bdd.dc1 +48:05.322 [INFO] TestAgent.server: Updating LAN server: server="Node-acefb08e-8207-99f7-1bea-a20a64ea9bdd (Addr: tcp/127.0.0.1:23482) (DC: dc1)" +48:05.322 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-acefb08e-8207-99f7-1bea-a20a64ea9bdd.dc1 area=wan +48:05.323 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:05.326 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:05.326 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:05.326 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:05.326 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:05.326 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:05.326 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-acefb08e-8207-99f7-1bea-a20a64ea9bdd +48:05.326 [INFO] TestAgent.server: member joined, marking health alive: member=Node-acefb08e-8207-99f7-1bea-a20a64ea9bdd +48:05.327 [INFO] TestAgent.server: federation state anti-entropy synced +48:05.327 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-acefb08e-8207-99f7-1bea-a20a64ea9bdd +48:05.534 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-acefb08e-8207-99f7-1bea-a20a64ea9bdd +48:05.535 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-acefb08e-8207-99f7-1bea-a20a64ea9bdd +48:05.540 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:05.540 [INFO] TestAgent: Synced node info +48:05.543 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy +48:05.543 [DEBUG] TestAgent: Node info in sync +48:05.543 [INFO] TestAgent: Synced service: service=web +48:05.544 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +48:05.544 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +48:05.544 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +48:05.548 [DEBUG] TestAgent: removed service: service=web +48:05.549 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 +48:05.549 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +48:05.549 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +48:05.549 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 +48:05.549 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +48:05.549 [DEBUG] TestAgent: Node info in sync +48:05.549 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +48:05.550 [INFO] TestAgent: Deregistered service: service=web +48:05.550 [INFO] TestAgent: Requesting shutdown +48:05.550 [INFO] TestAgent.server: shutting down server +48:05.550 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:05.550 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:05.550 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:05.550 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:05.550 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:05.550 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:05.550 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:05.550 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:05.550 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:05.550 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:05.550 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:05.550 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:05.550 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:05.550 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:05.550 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:05.550Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:05.550Z [INFO] agent.router.manager: shutting down +48:05.550 [INFO] TestAgent: consul server down +48:05.550 [INFO] TestAgent: shutdown complete +48:05.550 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23477 network=tcp +48:05.550 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23477 network=udp +48:05.550 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23478 network=tcp +48:06.053 [INFO] TestAgent: Waiting for endpoints to shut down +48:06.053 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_check_definition_in_sidecar -2021-10-31T23:11:48.024Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:48.024Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:48.051Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:48.051Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:48.053 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:18596f3b-029f-4530-e489-6e1a26e44fdd Address:127.0.0.1:16342}]" -11:48.053 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-18596f3b-029f-4530-e489-6e1a26e44fdd.dc1 127.0.0.1 -11:48.053 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16342 [Follower]" leader= -11:48.058 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-18596f3b-029f-4530-e489-6e1a26e44fdd 127.0.0.1 -2021-10-31T23:11:48.058Z [INFO] agent.router: Initializing LAN area manager -11:48.058 [INFO] TestAgent.server: Adding LAN server: server="Node-18596f3b-029f-4530-e489-6e1a26e44fdd (Addr: tcp/127.0.0.1:16342) (DC: dc1)" -11:48.058 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16337 network=udp -11:48.059 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-18596f3b-029f-4530-e489-6e1a26e44fdd.dc1 area=wan -11:48.059 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16337 network=tcp -11:48.059 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16338 network=tcp -11:48.059 [INFO] TestAgent: started state syncer -11:48.059 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16343 network=tcp -11:48.108 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:48.108 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16342 [Candidate]" term=2 -11:48.108 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:48.108 [DEBUG] TestAgent.server.raft: vote granted: from=18596f3b-029f-4530-e489-6e1a26e44fdd term=2 tally=1 -11:48.108 [INFO] TestAgent.server.raft: election won: tally=1 -11:48.108 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16342 [Leader]" -11:48.108 [INFO] TestAgent.server: cluster leadership acquired -11:48.109 [INFO] TestAgent.server: initializing acls -11:48.109 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:48.109 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:48.109 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:48.110 [INFO] TestAgent.server: New leader elected: payload=Node-18596f3b-029f-4530-e489-6e1a26e44fdd -11:48.110 [INFO] TestAgent.server: initializing acls -11:48.110 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:48.110 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:48.110 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:48.110 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:48.110 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -11:48.110 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-18596f3b-029f-4530-e489-6e1a26e44fdd -11:48.110 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-18596f3b-029f-4530-e489-6e1a26e44fdd.dc1 -11:48.110 [INFO] TestAgent.server: Updating LAN server: server="Node-18596f3b-029f-4530-e489-6e1a26e44fdd (Addr: tcp/127.0.0.1:16342) (DC: dc1)" -11:48.110 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-18596f3b-029f-4530-e489-6e1a26e44fdd.dc1 area=wan -11:48.110 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-18596f3b-029f-4530-e489-6e1a26e44fdd -11:48.111 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-18596f3b-029f-4530-e489-6e1a26e44fdd.dc1 -11:48.111 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-18596f3b-029f-4530-e489-6e1a26e44fdd.dc1 area=wan -11:48.111 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:48.111 [INFO] TestAgent.server: Updating LAN server: server="Node-18596f3b-029f-4530-e489-6e1a26e44fdd (Addr: tcp/127.0.0.1:16342) (DC: dc1)" -11:48.112 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:48.112 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:48.112 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:48.112 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:48.112 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:48.112 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-18596f3b-029f-4530-e489-6e1a26e44fdd -11:48.112 [INFO] TestAgent.server: member joined, marking health alive: member=Node-18596f3b-029f-4530-e489-6e1a26e44fdd -11:48.113 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-18596f3b-029f-4530-e489-6e1a26e44fdd -11:48.113 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-18596f3b-029f-4530-e489-6e1a26e44fdd -11:48.113 [INFO] TestAgent.server: federation state anti-entropy synced -11:48.234 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:48.235 [INFO] TestAgent: Synced node info -11:48.340 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-18596f3b-029f-4530-e489-6e1a26e44fdd -11:48.341 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-18596f3b-029f-4530-e489-6e1a26e44fdd -11:48.344 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy -11:48.344 [DEBUG] TestAgent: Node info in sync -11:48.344 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -11:48.345 [INFO] TestAgent: Synced service: service=web -11:48.345 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 -11:48.345 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 -11:48.345 [DEBUG] TestAgent: removed service: service=web -11:48.346 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 -11:48.346 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 -11:48.346 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -11:48.346 [DEBUG] TestAgent: Node info in sync -11:48.346 [INFO] TestAgent: Deregistered service: service=web -11:48.346 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -11:48.346 [INFO] TestAgent: Requesting shutdown -11:48.355 [INFO] TestAgent.server: shutting down server -11:48.355 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:48.355 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:48.355 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:48.355 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:48.355 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:48.355 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:48.355 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:48.355 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:48.355 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:48.355 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:48.355 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:48.355 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:48.355 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:48.355 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:48.355 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:48.355Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:48.355Z [INFO] agent.router.manager: shutting down -11:48.356 [INFO] TestAgent: consul server down -11:48.356 [INFO] TestAgent: shutdown complete -11:48.356 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16337 network=tcp -11:48.356 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16337 network=udp -11:48.356 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16338 network=tcp -11:48.857 [INFO] TestAgent: Waiting for endpoints to shut down -11:48.857 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_check_definition_in_sidecar -2021-10-31T23:11:48.531Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:48.555Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:48.557 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dc33c70a-b989-dea5-96d5-4104bf90b3ad Address:127.0.0.1:16349}]" -11:48.557 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dc33c70a-b989-dea5-96d5-4104bf90b3ad.dc1 127.0.0.1 -11:48.557 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16349 [Follower]" leader= -11:48.558 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dc33c70a-b989-dea5-96d5-4104bf90b3ad 127.0.0.1 -2021-10-31T23:11:48.558Z [INFO] agent.router: Initializing LAN area manager -11:48.558 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16344 network=udp -11:48.558 [INFO] TestAgent.server: Adding LAN server: server="Node-dc33c70a-b989-dea5-96d5-4104bf90b3ad (Addr: tcp/127.0.0.1:16349) (DC: dc1)" -11:48.558 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dc33c70a-b989-dea5-96d5-4104bf90b3ad.dc1 area=wan -11:48.558 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16344 network=tcp -11:48.559 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16345 network=tcp -11:48.559 [INFO] TestAgent: started state syncer -11:48.559 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16350 network=tcp -11:48.628 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:48.628 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16349 [Candidate]" term=2 -11:48.628 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:48.628 [DEBUG] TestAgent.server.raft: vote granted: from=dc33c70a-b989-dea5-96d5-4104bf90b3ad term=2 tally=1 -11:48.628 [INFO] TestAgent.server.raft: election won: tally=1 -11:48.628 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16349 [Leader]" -11:48.628 [INFO] TestAgent.server: cluster leadership acquired -11:48.628 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16349 -11:48.629 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:48.629 [INFO] TestAgent.server: New leader elected: payload=Node-dc33c70a-b989-dea5-96d5-4104bf90b3ad -11:48.631 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:48.631 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:48.631 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:48.631 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:48.631 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:48.631 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dc33c70a-b989-dea5-96d5-4104bf90b3ad -11:48.631 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dc33c70a-b989-dea5-96d5-4104bf90b3ad -11:48.631 [INFO] TestAgent.server: federation state anti-entropy synced -11:48.658 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:48.659 [INFO] TestAgent: Synced node info -11:48.846 [INFO] TestAgent: Requesting shutdown -11:48.846 [INFO] TestAgent.server: shutting down server -11:48.846 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:48.846 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:48.846 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:48.846 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:48.846 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:48.846 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:48.846 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:48.846 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:48.846 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:48.846 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:48.846 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:48.847Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:48.847Z [INFO] agent.router.manager: shutting down -11:48.847 [INFO] TestAgent: consul server down -11:48.847 [INFO] TestAgent: shutdown complete -11:48.847 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16344 network=tcp -11:48.847 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16344 network=udp -11:48.847 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16345 network=tcp -11:49.349 [INFO] TestAgent: Waiting for endpoints to shut down -11:49.349 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:06.083Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:06.100Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:06.103 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c2ee6a9a-70bb-1ee9-8390-f3b75b7a397e Address:127.0.0.1:23496}]" +48:06.103 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23496 [Follower]" leader= +48:06.104 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c2ee6a9a-70bb-1ee9-8390-f3b75b7a397e.dc1 127.0.0.1 +48:06.104 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c2ee6a9a-70bb-1ee9-8390-f3b75b7a397e 127.0.0.1 +2022-12-04T05:48:06.104Z [INFO] agent.router: Initializing LAN area manager +48:06.104 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c2ee6a9a-70bb-1ee9-8390-f3b75b7a397e.dc1 area=wan +48:06.105 [INFO] TestAgent.server: Adding LAN server: server="Node-c2ee6a9a-70bb-1ee9-8390-f3b75b7a397e (Addr: tcp/127.0.0.1:23496) (DC: dc1)" +48:06.105 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23491 network=udp +48:06.105 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23491 network=tcp +48:06.105 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23492 network=tcp +48:06.105 [INFO] TestAgent: started state syncer +48:06.105 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23497 network=tcp +48:06.142 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:06.142 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23496 [Candidate]" term=2 +48:06.142 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:06.142 [DEBUG] TestAgent.server.raft: vote granted: from=c2ee6a9a-70bb-1ee9-8390-f3b75b7a397e term=2 tally=1 +48:06.142 [INFO] TestAgent.server.raft: election won: tally=1 +48:06.142 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23496 [Leader]" +48:06.142 [INFO] TestAgent.server: cluster leadership acquired +48:06.142 [INFO] TestAgent.server: New leader elected: payload=Node-c2ee6a9a-70bb-1ee9-8390-f3b75b7a397e +48:06.142 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23496 +48:06.143 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:06.144 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:06.144 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:06.144 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:06.144 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:06.144 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:06.144 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c2ee6a9a-70bb-1ee9-8390-f3b75b7a397e +48:06.144 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c2ee6a9a-70bb-1ee9-8390-f3b75b7a397e +48:06.145 [INFO] TestAgent.server: federation state anti-entropy synced +48:06.269 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:06.270 [INFO] TestAgent: Synced node info +48:06.270 [DEBUG] TestAgent: Node info in sync +48:06.384 [INFO] TestAgent: Requesting shutdown +48:06.384 [INFO] TestAgent.server: shutting down server +48:06.384 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:06.384 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:06.384 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:06.384 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:06.384 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:06.384 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:06.384 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:06.384 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:06.384 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:06.384 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:06.384 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:06.384Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:06.384Z [INFO] agent.router.manager: shutting down +48:06.385 [INFO] TestAgent: consul server down +48:06.385 [INFO] TestAgent: shutdown complete +48:06.385 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23491 network=tcp +48:06.385 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23491 network=udp +48:06.385 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23492 network=tcp +48:06.886 [INFO] TestAgent: Waiting for endpoints to shut down +48:06.886 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_checks_definitions_in_sidecar -2021-10-31T23:11:48.896Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:48.931Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:48.934 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e18d1271-cdc3-3ea5-84b0-9d945d80c027 Address:127.0.0.1:16356}]" -11:48.935 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e18d1271-cdc3-3ea5-84b0-9d945d80c027.dc1 127.0.0.1 -11:48.935 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e18d1271-cdc3-3ea5-84b0-9d945d80c027 127.0.0.1 -2021-10-31T23:11:48.935Z [INFO] agent.router: Initializing LAN area manager -11:48.936 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16351 network=udp -11:48.936 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16356 [Follower]" leader= -11:48.936 [INFO] TestAgent.server: Adding LAN server: server="Node-e18d1271-cdc3-3ea5-84b0-9d945d80c027 (Addr: tcp/127.0.0.1:16356) (DC: dc1)" -11:48.936 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e18d1271-cdc3-3ea5-84b0-9d945d80c027.dc1 area=wan -11:48.936 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16351 network=tcp -11:48.937 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16352 network=tcp -11:48.950 [INFO] TestAgent: started state syncer -11:48.950 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16357 network=tcp -11:49.001 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:49.001 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16356 [Candidate]" term=2 -11:49.002 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:49.002 [DEBUG] TestAgent.server.raft: vote granted: from=e18d1271-cdc3-3ea5-84b0-9d945d80c027 term=2 tally=1 -11:49.002 [INFO] TestAgent.server.raft: election won: tally=1 -11:49.002 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16356 [Leader]" -11:49.002 [INFO] TestAgent.server: cluster leadership acquired -11:49.002 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16356 -11:49.003 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:49.004 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:49.004 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:49.004 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:49.004 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:49.004 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:49.004 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e18d1271-cdc3-3ea5-84b0-9d945d80c027 -11:49.004 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e18d1271-cdc3-3ea5-84b0-9d945d80c027 -11:49.004 [INFO] TestAgent.server: New leader elected: payload=Node-e18d1271-cdc3-3ea5-84b0-9d945d80c027 -11:49.005 [INFO] TestAgent.server: federation state anti-entropy synced -11:49.258 [INFO] TestAgent: Requesting shutdown -11:49.259 [INFO] TestAgent.server: shutting down server -11:49.259 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:49.259 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:49.259 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:49.259 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:49.259 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:49.259 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:49.259 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:49.259 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:49.259 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:49.259 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:49.259 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:49.259 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:49.259Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:49.259Z [INFO] agent.router.manager: shutting down -11:49.259 [INFO] TestAgent: consul server down -11:49.259 [INFO] TestAgent: shutdown complete -11:49.259 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16351 network=tcp -11:49.259 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16351 network=udp -11:49.259 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16352 network=tcp -11:49.759 [INFO] TestAgent: Waiting for endpoints to shut down -11:49.759 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:06.088Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:06.105Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:06.107 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f4cb9f1d-93d8-38f2-a616-71892b292745 Address:127.0.0.1:23503}]" +48:06.108 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f4cb9f1d-93d8-38f2-a616-71892b292745.dc1 127.0.0.1 +48:06.108 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23503 [Follower]" leader= +48:06.108 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f4cb9f1d-93d8-38f2-a616-71892b292745 127.0.0.1 +2022-12-04T05:48:06.108Z [INFO] agent.router: Initializing LAN area manager +48:06.109 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23498 network=udp +48:06.109 [INFO] TestAgent.server: Adding LAN server: server="Node-f4cb9f1d-93d8-38f2-a616-71892b292745 (Addr: tcp/127.0.0.1:23503) (DC: dc1)" +48:06.109 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f4cb9f1d-93d8-38f2-a616-71892b292745.dc1 area=wan +48:06.109 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23498 network=tcp +48:06.109 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23499 network=tcp +48:06.109 [INFO] TestAgent: started state syncer +48:06.109 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23504 network=tcp +48:06.167 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:06.167 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23503 [Candidate]" term=2 +48:06.167 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:06.167 [DEBUG] TestAgent.server.raft: vote granted: from=f4cb9f1d-93d8-38f2-a616-71892b292745 term=2 tally=1 +48:06.167 [INFO] TestAgent.server.raft: election won: tally=1 +48:06.167 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23503 [Leader]" +48:06.167 [INFO] TestAgent.server: cluster leadership acquired +48:06.167 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23503 +48:06.167 [INFO] TestAgent.server: New leader elected: payload=Node-f4cb9f1d-93d8-38f2-a616-71892b292745 +48:06.168 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:06.170 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:06.170 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:06.170 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:06.170 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:06.170 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:06.170 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f4cb9f1d-93d8-38f2-a616-71892b292745 +48:06.170 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f4cb9f1d-93d8-38f2-a616-71892b292745 +48:06.170 [INFO] TestAgent.server: federation state anti-entropy synced +48:06.284 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:06.284 [INFO] TestAgent: Synced node info +48:06.284 [DEBUG] TestAgent: Node info in sync +48:06.464 [INFO] TestAgent: Requesting shutdown +48:06.464 [INFO] TestAgent.server: shutting down server +48:06.464 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:06.464 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:06.464 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:06.464 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:06.464 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:06.464 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:06.464 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:06.464 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:06.464 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:06.464 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:06.464 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:06.464Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:06.464Z [INFO] agent.router.manager: shutting down +48:06.464 [INFO] TestAgent: consul server down +48:06.464 [INFO] TestAgent: shutdown complete +48:06.464 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23498 network=tcp +48:06.464 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23498 network=udp +48:06.464 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23499 network=tcp +48:06.966 [INFO] TestAgent: Waiting for endpoints to shut down +48:06.966 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_checks_definitions_in_sidecar -2021-10-31T23:11:49.372Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:49.393Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:49.399 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7211c768-994f-4cf1-d1b0-9f01f2c9bb96 Address:127.0.0.1:16363}]" -11:49.399 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16363 [Follower]" leader= -11:49.400 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7211c768-994f-4cf1-d1b0-9f01f2c9bb96.dc1 127.0.0.1 -11:49.400 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7211c768-994f-4cf1-d1b0-9f01f2c9bb96 127.0.0.1 -2021-10-31T23:11:49.400Z [INFO] agent.router: Initializing LAN area manager -11:49.401 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16358 network=udp -11:49.401 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7211c768-994f-4cf1-d1b0-9f01f2c9bb96.dc1 area=wan -11:49.401 [INFO] TestAgent.server: Adding LAN server: server="Node-7211c768-994f-4cf1-d1b0-9f01f2c9bb96 (Addr: tcp/127.0.0.1:16363) (DC: dc1)" -11:49.401 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16358 network=tcp -11:49.405 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16359 network=tcp -11:49.406 [INFO] TestAgent: started state syncer -11:49.406 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16364 network=tcp -11:49.453 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:49.453 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16363 [Candidate]" term=2 -11:49.454 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:49.454 [DEBUG] TestAgent.server.raft: vote granted: from=7211c768-994f-4cf1-d1b0-9f01f2c9bb96 term=2 tally=1 -11:49.454 [INFO] TestAgent.server.raft: election won: tally=1 -11:49.454 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16363 [Leader]" -11:49.454 [INFO] TestAgent.server: cluster leadership acquired -11:49.454 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16363 -11:49.455 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:49.456 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:49.456 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:49.456 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:49.456 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:49.456 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:49.456 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7211c768-994f-4cf1-d1b0-9f01f2c9bb96 -11:49.456 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7211c768-994f-4cf1-d1b0-9f01f2c9bb96 -11:49.457 [INFO] TestAgent.server: New leader elected: payload=Node-7211c768-994f-4cf1-d1b0-9f01f2c9bb96 -11:49.457 [INFO] TestAgent.server: federation state anti-entropy synced -11:49.542 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:49.542 [INFO] TestAgent: Synced node info -11:49.542 [DEBUG] TestAgent: Node info in sync -11:49.618 [INFO] TestAgent: Requesting shutdown -11:49.618 [INFO] TestAgent.server: shutting down server -11:49.618 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:49.618 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:49.618 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:49.618 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:49.618 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:49.618 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:49.618 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:49.618 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:49.618 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:49.618 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:49.618 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:49.618Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:49.618Z [INFO] agent.router.manager: shutting down -11:49.618 [INFO] TestAgent: consul server down -11:49.618 [INFO] TestAgent: shutdown complete -11:49.618 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16358 network=tcp -11:49.618 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16358 network=udp -11:49.618 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16359 network=tcp -11:50.122 [INFO] TestAgent: Waiting for endpoints to shut down -11:50.122 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:06.899Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:06.912Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:06.914 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:be53ddae-0b24-08df-622e-cd67c0257540 Address:127.0.0.1:23510}]" +48:06.914 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23510 [Follower]" leader= +48:06.915 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-be53ddae-0b24-08df-622e-cd67c0257540.dc1 127.0.0.1 +48:06.915 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-be53ddae-0b24-08df-622e-cd67c0257540 127.0.0.1 +2022-12-04T05:48:06.915Z [INFO] agent.router: Initializing LAN area manager +48:06.916 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23505 network=udp +48:06.916 [INFO] TestAgent.server: Adding LAN server: server="Node-be53ddae-0b24-08df-622e-cd67c0257540 (Addr: tcp/127.0.0.1:23510) (DC: dc1)" +48:06.916 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23505 network=tcp +48:06.916 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-be53ddae-0b24-08df-622e-cd67c0257540.dc1 area=wan +48:06.917 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23506 network=tcp +48:06.917 [INFO] TestAgent: started state syncer +48:06.917 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23511 network=tcp +48:06.955 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:06.955 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23510 [Candidate]" term=2 +48:06.955 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:06.955 [DEBUG] TestAgent.server.raft: vote granted: from=be53ddae-0b24-08df-622e-cd67c0257540 term=2 tally=1 +48:06.955 [INFO] TestAgent.server.raft: election won: tally=1 +48:06.955 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23510 [Leader]" +48:06.955 [INFO] TestAgent.server: cluster leadership acquired +48:06.955 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23510 +48:06.956 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:06.957 [INFO] TestAgent.server: New leader elected: payload=Node-be53ddae-0b24-08df-622e-cd67c0257540 +48:06.958 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:06.958 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:06.958 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:06.958 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:06.958 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:06.958 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-be53ddae-0b24-08df-622e-cd67c0257540 +48:06.958 [INFO] TestAgent.server: member joined, marking health alive: member=Node-be53ddae-0b24-08df-622e-cd67c0257540 +48:06.958 [INFO] TestAgent.server: federation state anti-entropy synced +48:07.050 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:07.051 [INFO] TestAgent: Synced node info +48:07.051 [DEBUG] TestAgent: Node info in sync +48:07.122 [INFO] TestAgent: Requesting shutdown +48:07.122 [INFO] TestAgent.server: shutting down server +48:07.122 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:07.123 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:07.123 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:07.123 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:07.123 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:07.123 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:07.123 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:07.123 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:07.123 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:07.123 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:07.123 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:07.123Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:07.123Z [INFO] agent.router.manager: shutting down +48:07.123 [INFO] TestAgent: consul server down +48:07.123 [INFO] TestAgent: shutdown complete +48:07.123 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23505 network=tcp +48:07.123 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23505 network=udp +48:07.123 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23506 network=tcp +48:07.624 [INFO] TestAgent: Waiting for endpoints to shut down +48:07.624 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_check_status_in_sidecar -2021-10-31T23:11:49.776Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:49.792Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:49.795 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:06516fbd-1ae6-d789-9d0f-eb3a7796c6f3 Address:127.0.0.1:16370}]" -11:49.795 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-06516fbd-1ae6-d789-9d0f-eb3a7796c6f3.dc1 127.0.0.1 -11:49.796 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-06516fbd-1ae6-d789-9d0f-eb3a7796c6f3 127.0.0.1 -2021-10-31T23:11:49.796Z [INFO] agent.router: Initializing LAN area manager -11:49.796 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16365 network=udp -11:49.796 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16370 [Follower]" leader= -11:49.796 [INFO] TestAgent.server: Adding LAN server: server="Node-06516fbd-1ae6-d789-9d0f-eb3a7796c6f3 (Addr: tcp/127.0.0.1:16370) (DC: dc1)" -11:49.796 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-06516fbd-1ae6-d789-9d0f-eb3a7796c6f3.dc1 area=wan -11:49.796 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16365 network=tcp -11:49.809 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16366 network=tcp -11:49.809 [INFO] TestAgent: started state syncer -11:49.809 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16371 network=tcp -11:49.856 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:49.856 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16370 [Candidate]" term=2 -11:49.856 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:49.856 [DEBUG] TestAgent.server.raft: vote granted: from=06516fbd-1ae6-d789-9d0f-eb3a7796c6f3 term=2 tally=1 -11:49.856 [INFO] TestAgent.server.raft: election won: tally=1 -11:49.856 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16370 [Leader]" -11:49.856 [INFO] TestAgent.server: cluster leadership acquired -11:49.857 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16370 -11:49.857 [INFO] TestAgent.server: New leader elected: payload=Node-06516fbd-1ae6-d789-9d0f-eb3a7796c6f3 -11:49.857 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:49.859 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:49.859 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:49.859 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:49.859 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:49.859 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:49.859 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-06516fbd-1ae6-d789-9d0f-eb3a7796c6f3 -11:49.859 [INFO] TestAgent.server: member joined, marking health alive: member=Node-06516fbd-1ae6-d789-9d0f-eb3a7796c6f3 -11:49.860 [INFO] TestAgent.server: federation state anti-entropy synced -11:49.873 [INFO] TestAgent: Requesting shutdown -11:49.874 [INFO] TestAgent.server: shutting down server -11:49.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:49.874 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:49.874 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:49.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:49.874 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:49.874 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:49.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:49.874 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:49.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:49.874 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:49.874 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:49.874 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:49.874Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:49.874Z [INFO] agent.router.manager: shutting down -11:49.874 [INFO] TestAgent: consul server down -11:49.874 [INFO] TestAgent: shutdown complete -11:49.874 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16365 network=tcp -11:49.874 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16365 network=udp -11:49.874 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16366 network=tcp -11:50.377 [INFO] TestAgent: Waiting for endpoints to shut down -11:50.378 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:06.982Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:06.997Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:07.000 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dcfb8ecf-b467-b837-06c1-d7d47e6f1896 Address:127.0.0.1:23517}]" +48:07.005 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dcfb8ecf-b467-b837-06c1-d7d47e6f1896.dc1 127.0.0.1 +48:07.007 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dcfb8ecf-b467-b837-06c1-d7d47e6f1896 127.0.0.1 +2022-12-04T05:48:07.007Z [INFO] agent.router: Initializing LAN area manager +48:07.008 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23517 [Follower]" leader= +48:07.009 [INFO] TestAgent.server: Adding LAN server: server="Node-dcfb8ecf-b467-b837-06c1-d7d47e6f1896 (Addr: tcp/127.0.0.1:23517) (DC: dc1)" +48:07.009 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dcfb8ecf-b467-b837-06c1-d7d47e6f1896.dc1 area=wan +48:07.009 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23512 network=tcp +48:07.010 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23512 network=udp +48:07.014 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23513 network=tcp +48:07.014 [INFO] TestAgent: started state syncer +48:07.014 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23518 network=tcp +48:07.066 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:07.067 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23517 [Candidate]" term=2 +48:07.067 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:07.067 [DEBUG] TestAgent.server.raft: vote granted: from=dcfb8ecf-b467-b837-06c1-d7d47e6f1896 term=2 tally=1 +48:07.067 [INFO] TestAgent.server.raft: election won: tally=1 +48:07.067 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23517 [Leader]" +48:07.067 [INFO] TestAgent.server: cluster leadership acquired +48:07.067 [INFO] TestAgent.server: New leader elected: payload=Node-dcfb8ecf-b467-b837-06c1-d7d47e6f1896 +48:07.067 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23517 +48:07.070 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:07.071 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:07.071 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:07.071 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:07.071 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:07.071 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:07.072 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dcfb8ecf-b467-b837-06c1-d7d47e6f1896 +48:07.072 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dcfb8ecf-b467-b837-06c1-d7d47e6f1896 +48:07.072 [INFO] TestAgent.server: federation state anti-entropy synced +48:07.211 [INFO] TestAgent: Requesting shutdown +48:07.211 [INFO] TestAgent.server: shutting down server +48:07.211 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:07.211 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:07.211 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:07.211 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:07.211 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:07.211 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:07.211 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:07.211 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:07.211 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:07.211 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:07.211 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:07.211 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:07.211Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:07.211Z [INFO] agent.router.manager: shutting down +48:07.211 [INFO] TestAgent: consul server down +48:07.211 [INFO] TestAgent: shutdown complete +48:07.211 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23512 network=tcp +48:07.211 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23512 network=udp +48:07.211 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23513 network=tcp +48:07.712 [INFO] TestAgent: Waiting for endpoints to shut down +48:07.712 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_check_status_in_sidecar -2021-10-31T23:11:50.145Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:50.182Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:50.184 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:54604c9a-0623-ff26-7069-75698d5215c0 Address:127.0.0.1:16377}]" -11:50.184 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-54604c9a-0623-ff26-7069-75698d5215c0.dc1 127.0.0.1 -11:50.185 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-54604c9a-0623-ff26-7069-75698d5215c0 127.0.0.1 -2021-10-31T23:11:50.185Z [INFO] agent.router: Initializing LAN area manager -11:50.185 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16372 network=udp -11:50.185 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16377 [Follower]" leader= -11:50.186 [INFO] TestAgent.server: Adding LAN server: server="Node-54604c9a-0623-ff26-7069-75698d5215c0 (Addr: tcp/127.0.0.1:16377) (DC: dc1)" -11:50.186 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-54604c9a-0623-ff26-7069-75698d5215c0.dc1 area=wan -11:50.186 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16372 network=tcp -11:50.194 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16373 network=tcp -11:50.194 [INFO] TestAgent: started state syncer -11:50.194 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16378 network=tcp -11:50.241 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:50.241 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16377 [Candidate]" term=2 -11:50.242 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:50.242 [DEBUG] TestAgent.server.raft: vote granted: from=54604c9a-0623-ff26-7069-75698d5215c0 term=2 tally=1 -11:50.242 [INFO] TestAgent.server.raft: election won: tally=1 -11:50.242 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16377 [Leader]" -11:50.242 [INFO] TestAgent.server: cluster leadership acquired -11:50.242 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16377 -11:50.243 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:50.244 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:50.244 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:50.244 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:50.244 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:50.244 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:50.244 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-54604c9a-0623-ff26-7069-75698d5215c0 -11:50.244 [INFO] TestAgent.server: member joined, marking health alive: member=Node-54604c9a-0623-ff26-7069-75698d5215c0 -11:50.245 [INFO] TestAgent.server: New leader elected: payload=Node-54604c9a-0623-ff26-7069-75698d5215c0 -11:50.245 [INFO] TestAgent.server: federation state anti-entropy synced -11:50.347 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:50.348 [INFO] TestAgent: Synced node info -11:50.518 [INFO] TestAgent: Requesting shutdown -11:50.518 [INFO] TestAgent.server: shutting down server -11:50.518 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:50.518 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:50.518 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:50.518 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:50.518 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:50.518 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:50.518 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:50.518 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:50.518 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:50.518 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:50.518 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:50.518Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:50.518Z [INFO] agent.router.manager: shutting down -11:50.518 [INFO] TestAgent: consul server down -11:50.518 [INFO] TestAgent: shutdown complete -11:50.518 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16372 network=tcp -11:50.518 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16372 network=udp -11:50.518 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16373 network=tcp -11:51.021 [INFO] TestAgent: Waiting for endpoints to shut down -11:51.021 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:07.638Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:07.651Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:07.653 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d95a662a-8792-ebe1-1b88-d20af5ff7659 Address:127.0.0.1:23524}]" +48:07.653 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23524 [Follower]" leader= +48:07.654 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d95a662a-8792-ebe1-1b88-d20af5ff7659.dc1 127.0.0.1 +48:07.654 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d95a662a-8792-ebe1-1b88-d20af5ff7659 127.0.0.1 +2022-12-04T05:48:07.654Z [INFO] agent.router: Initializing LAN area manager +48:07.655 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23519 network=udp +48:07.655 [INFO] TestAgent.server: Adding LAN server: server="Node-d95a662a-8792-ebe1-1b88-d20af5ff7659 (Addr: tcp/127.0.0.1:23524) (DC: dc1)" +48:07.655 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d95a662a-8792-ebe1-1b88-d20af5ff7659.dc1 area=wan +48:07.655 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23519 network=tcp +48:07.656 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23520 network=tcp +48:07.656 [INFO] TestAgent: started state syncer +48:07.656 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23525 network=tcp +48:07.696 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:07.696 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23524 [Candidate]" term=2 +48:07.697 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:07.697 [DEBUG] TestAgent.server.raft: vote granted: from=d95a662a-8792-ebe1-1b88-d20af5ff7659 term=2 tally=1 +48:07.697 [INFO] TestAgent.server.raft: election won: tally=1 +48:07.697 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23524 [Leader]" +48:07.697 [INFO] TestAgent.server: cluster leadership acquired +48:07.697 [INFO] TestAgent.server: New leader elected: payload=Node-d95a662a-8792-ebe1-1b88-d20af5ff7659 +48:07.697 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23524 +48:07.698 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:07.700 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:07.700 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:07.700 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:07.700 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:07.700 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:07.700 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d95a662a-8792-ebe1-1b88-d20af5ff7659 +48:07.700 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d95a662a-8792-ebe1-1b88-d20af5ff7659 +48:07.700 [INFO] TestAgent.server: federation state anti-entropy synced +48:07.788 [INFO] TestAgent: Requesting shutdown +48:07.788 [INFO] TestAgent.server: shutting down server +48:07.788 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:07.788 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:07.788 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:07.788 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:07.788 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:07.788 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:07.788 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:07.788 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:07.788 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:07.788 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:07.788 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:07.788 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:48:07.788Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:07.788Z [INFO] agent.router.manager: shutting down +48:07.788 [INFO] TestAgent: consul server down +48:07.788 [INFO] TestAgent: shutdown complete +48:07.788 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23519 network=tcp +48:07.788 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23519 network=udp +48:07.788 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23520 network=tcp +48:08.289 [INFO] TestAgent: Waiting for endpoints to shut down +48:08.289 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_checks_status_in_sidecar -2021-10-31T23:11:50.405Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:50.432Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:50.434 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:edeb8dee-34ae-cb5b-c6ea-5af50c44ab5d Address:127.0.0.1:16384}]" -11:50.435 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-edeb8dee-34ae-cb5b-c6ea-5af50c44ab5d.dc1 127.0.0.1 -11:50.436 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-edeb8dee-34ae-cb5b-c6ea-5af50c44ab5d 127.0.0.1 -2021-10-31T23:11:50.436Z [INFO] agent.router: Initializing LAN area manager -11:50.436 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16379 network=udp -11:50.436 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16384 [Follower]" leader= -11:50.436 [INFO] TestAgent.server: Adding LAN server: server="Node-edeb8dee-34ae-cb5b-c6ea-5af50c44ab5d (Addr: tcp/127.0.0.1:16384) (DC: dc1)" -11:50.436 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-edeb8dee-34ae-cb5b-c6ea-5af50c44ab5d.dc1 area=wan -11:50.436 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16379 network=tcp -11:50.437 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16380 network=tcp -11:50.437 [INFO] TestAgent: started state syncer -11:50.437 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16385 network=tcp -11:50.473 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:50.473 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16384 [Candidate]" term=2 -11:50.474 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:50.474 [DEBUG] TestAgent.server.raft: vote granted: from=edeb8dee-34ae-cb5b-c6ea-5af50c44ab5d term=2 tally=1 -11:50.474 [INFO] TestAgent.server.raft: election won: tally=1 -11:50.474 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16384 [Leader]" -11:50.474 [INFO] TestAgent.server: cluster leadership acquired -11:50.489 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16384 -11:50.491 [INFO] TestAgent.server: New leader elected: payload=Node-edeb8dee-34ae-cb5b-c6ea-5af50c44ab5d -11:50.491 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:50.493 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:50.493 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:50.493 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:50.493 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:50.493 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:50.493 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-edeb8dee-34ae-cb5b-c6ea-5af50c44ab5d -11:50.493 [INFO] TestAgent.server: member joined, marking health alive: member=Node-edeb8dee-34ae-cb5b-c6ea-5af50c44ab5d -11:50.498 [INFO] TestAgent.server: federation state anti-entropy synced -11:50.606 [INFO] TestAgent: Requesting shutdown -11:50.606 [INFO] TestAgent.server: shutting down server -11:50.606 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:50.606 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:50.606 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:50.606 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:50.606 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:50.606 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:50.606 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:50.606 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:50.606 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:50.606 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:50.606 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:50.606 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:50.606Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:50.606Z [INFO] agent.router.manager: shutting down -11:50.606 [INFO] TestAgent: consul server down -11:50.606 [INFO] TestAgent: shutdown complete -11:50.606 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16379 network=tcp -11:50.606 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16379 network=udp -11:50.606 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16380 network=tcp -11:51.109 [INFO] TestAgent: Waiting for endpoints to shut down -11:51.109 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:07.726Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:07.739Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:07.741 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:05e18784-5249-7974-f143-688b928e0261 Address:127.0.0.1:23531}]" +48:07.741 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23531 [Follower]" leader= +48:07.742 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-05e18784-5249-7974-f143-688b928e0261.dc1 127.0.0.1 +48:07.743 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-05e18784-5249-7974-f143-688b928e0261 127.0.0.1 +2022-12-04T05:48:07.743Z [INFO] agent.router: Initializing LAN area manager +48:07.743 [INFO] TestAgent.server: Adding LAN server: server="Node-05e18784-5249-7974-f143-688b928e0261 (Addr: tcp/127.0.0.1:23531) (DC: dc1)" +48:07.743 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-05e18784-5249-7974-f143-688b928e0261.dc1 area=wan +48:07.743 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23526 network=udp +48:07.743 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23526 network=tcp +48:07.748 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23527 network=tcp +48:07.748 [INFO] TestAgent: started state syncer +48:07.752 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23532 network=tcp +48:07.807 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:07.807 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23531 [Candidate]" term=2 +48:07.807 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:07.807 [DEBUG] TestAgent.server.raft: vote granted: from=05e18784-5249-7974-f143-688b928e0261 term=2 tally=1 +48:07.807 [INFO] TestAgent.server.raft: election won: tally=1 +48:07.807 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23531 [Leader]" +48:07.807 [INFO] TestAgent.server: cluster leadership acquired +48:07.807 [INFO] TestAgent.server: New leader elected: payload=Node-05e18784-5249-7974-f143-688b928e0261 +48:07.807 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23531 +48:07.808 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:07.809 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:07.809 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:07.809 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:07.809 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:07.809 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:07.809 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-05e18784-5249-7974-f143-688b928e0261 +48:07.809 [INFO] TestAgent.server: member joined, marking health alive: member=Node-05e18784-5249-7974-f143-688b928e0261 +48:07.810 [INFO] TestAgent.server: federation state anti-entropy synced +48:07.891 [INFO] TestAgent: Requesting shutdown +48:07.892 [INFO] TestAgent.server: shutting down server +48:07.892 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:07.892 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:07.892 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:07.892 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:07.892 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:07.892 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:07.892 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:07.892 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:07.892 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:07.892 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:07.892 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:07.892 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:07.892Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:07.892Z [INFO] agent.router.manager: shutting down +48:07.892 [INFO] TestAgent: consul server down +48:07.892 [INFO] TestAgent: shutdown complete +48:07.892 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23526 network=tcp +48:07.892 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23526 network=udp +48:07.892 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23527 network=tcp +48:08.392 [INFO] TestAgent: Waiting for endpoints to shut down +48:08.392 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_checks_status_in_sidecar -2021-10-31T23:11:51.059Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:51.096Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:51.098 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:484d4b86-3c8e-b74f-118e-8eb1c520e0f4 Address:127.0.0.1:16391}]" -11:51.099 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-484d4b86-3c8e-b74f-118e-8eb1c520e0f4.dc1 127.0.0.1 -11:51.100 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-484d4b86-3c8e-b74f-118e-8eb1c520e0f4 127.0.0.1 -2021-10-31T23:11:51.100Z [INFO] agent.router: Initializing LAN area manager -11:51.100 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16386 network=udp -11:51.100 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16391 [Follower]" leader= -11:51.100 [INFO] TestAgent.server: Adding LAN server: server="Node-484d4b86-3c8e-b74f-118e-8eb1c520e0f4 (Addr: tcp/127.0.0.1:16391) (DC: dc1)" -11:51.100 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-484d4b86-3c8e-b74f-118e-8eb1c520e0f4.dc1 area=wan -11:51.100 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16386 network=tcp -11:51.101 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16387 network=tcp -11:51.101 [INFO] TestAgent: started state syncer -11:51.101 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16392 network=tcp -11:51.146 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:51.146 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16391 [Candidate]" term=2 -11:51.146 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:51.146 [DEBUG] TestAgent.server.raft: vote granted: from=484d4b86-3c8e-b74f-118e-8eb1c520e0f4 term=2 tally=1 -11:51.146 [INFO] TestAgent.server.raft: election won: tally=1 -11:51.146 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16391 [Leader]" -11:51.146 [INFO] TestAgent.server: cluster leadership acquired -11:51.146 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16391 -11:51.147 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:51.148 [INFO] TestAgent.server: New leader elected: payload=Node-484d4b86-3c8e-b74f-118e-8eb1c520e0f4 -11:51.151 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:51.151 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:51.151 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:51.151 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:51.151 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:51.151 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-484d4b86-3c8e-b74f-118e-8eb1c520e0f4 -11:51.151 [INFO] TestAgent.server: member joined, marking health alive: member=Node-484d4b86-3c8e-b74f-118e-8eb1c520e0f4 -11:51.152 [INFO] TestAgent.server: federation state anti-entropy synced -11:51.318 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:51.318 [INFO] TestAgent: Synced node info -11:51.318 [DEBUG] TestAgent: Node info in sync -11:51.447 [INFO] TestAgent: Requesting shutdown -11:51.447 [INFO] TestAgent.server: shutting down server -11:51.447 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:51.447 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:51.447 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:51.447 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:51.447 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:51.447 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:51.447 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:51.447 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:51.447 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:51.447 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:51.448 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:51.448Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:51.448Z [INFO] agent.router.manager: shutting down -11:51.448 [INFO] TestAgent: consul server down -11:51.448 [INFO] TestAgent: shutdown complete -11:51.448 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16386 network=tcp -11:51.448 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16386 network=udp -11:51.448 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16387 network=tcp -11:51.949 [INFO] TestAgent: Waiting for endpoints to shut down -11:51.949 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:08.299Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:08.312Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:08.313 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d5f6c27e-9724-c446-a322-f2b5d3fc2bae Address:127.0.0.1:23538}]" +48:08.313 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23538 [Follower]" leader= +48:08.313 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d5f6c27e-9724-c446-a322-f2b5d3fc2bae.dc1 127.0.0.1 +48:08.314 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d5f6c27e-9724-c446-a322-f2b5d3fc2bae 127.0.0.1 +2022-12-04T05:48:08.314Z [INFO] agent.router: Initializing LAN area manager +48:08.314 [INFO] TestAgent.server: Adding LAN server: server="Node-d5f6c27e-9724-c446-a322-f2b5d3fc2bae (Addr: tcp/127.0.0.1:23538) (DC: dc1)" +48:08.314 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23533 network=udp +48:08.314 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d5f6c27e-9724-c446-a322-f2b5d3fc2bae.dc1 area=wan +48:08.314 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23533 network=tcp +48:08.314 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23534 network=tcp +48:08.314 [INFO] TestAgent: started state syncer +48:08.314 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23539 network=tcp +48:08.363 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:08.363 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23538 [Candidate]" term=2 +48:08.363 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:08.363 [DEBUG] TestAgent.server.raft: vote granted: from=d5f6c27e-9724-c446-a322-f2b5d3fc2bae term=2 tally=1 +48:08.363 [INFO] TestAgent.server.raft: election won: tally=1 +48:08.363 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23538 [Leader]" +48:08.363 [INFO] TestAgent.server: cluster leadership acquired +48:08.364 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23538 +48:08.364 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:08.365 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:08.365 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:08.365 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:08.365 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:08.365 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:08.365 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d5f6c27e-9724-c446-a322-f2b5d3fc2bae +48:08.365 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d5f6c27e-9724-c446-a322-f2b5d3fc2bae +48:08.366 [INFO] TestAgent.server: New leader elected: payload=Node-d5f6c27e-9724-c446-a322-f2b5d3fc2bae +48:08.366 [INFO] TestAgent.server: federation state anti-entropy synced +48:08.481 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:08.482 [INFO] TestAgent: Synced node info +48:08.594 [INFO] TestAgent: Requesting shutdown +48:08.594 [INFO] TestAgent.server: shutting down server +48:08.594 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:08.594 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:08.594 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:08.594 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:08.594 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:08.594 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:08.594 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:08.594 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:08.594 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:08.594 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:08.594 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:08.594Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:08.594Z [INFO] agent.router.manager: shutting down +48:08.594 [INFO] TestAgent: consul server down +48:08.594 [INFO] TestAgent: shutdown complete +48:08.594 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23533 network=tcp +48:08.594 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23533 network=udp +48:08.594 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23534 network=tcp +48:09.095 [INFO] TestAgent: Waiting for endpoints to shut down +48:09.095 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/another_service_registered_with_same_ID_as_a_sidecar_should_not_be_deregistered -2021-10-31T23:11:51.137Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:51.165Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:51.174 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:34197374-8c17-525f-3bd0-7648f20b45f6 Address:127.0.0.1:16398}]" -11:51.175 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-34197374-8c17-525f-3bd0-7648f20b45f6.dc1 127.0.0.1 -11:51.175 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16398 [Follower]" leader= -11:51.175 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-34197374-8c17-525f-3bd0-7648f20b45f6 127.0.0.1 -2021-10-31T23:11:51.176Z [INFO] agent.router: Initializing LAN area manager -11:51.176 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-34197374-8c17-525f-3bd0-7648f20b45f6.dc1 area=wan -11:51.176 [INFO] TestAgent.server: Adding LAN server: server="Node-34197374-8c17-525f-3bd0-7648f20b45f6 (Addr: tcp/127.0.0.1:16398) (DC: dc1)" -11:51.176 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16393 network=udp -11:51.176 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16393 network=tcp -11:51.177 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16394 network=tcp -11:51.177 [INFO] TestAgent: started state syncer -11:51.177 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16399 network=tcp -11:51.225 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:51.225 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16398 [Candidate]" term=2 -11:51.226 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:51.226 [DEBUG] TestAgent.server.raft: vote granted: from=34197374-8c17-525f-3bd0-7648f20b45f6 term=2 tally=1 -11:51.226 [INFO] TestAgent.server.raft: election won: tally=1 -11:51.226 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16398 [Leader]" -11:51.226 [INFO] TestAgent.server: cluster leadership acquired -11:51.226 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16398 -11:51.227 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:51.228 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:51.228 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:51.228 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:51.228 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:51.228 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:51.228 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-34197374-8c17-525f-3bd0-7648f20b45f6 -11:51.228 [INFO] TestAgent.server: member joined, marking health alive: member=Node-34197374-8c17-525f-3bd0-7648f20b45f6 -11:51.229 [INFO] TestAgent.server: New leader elected: payload=Node-34197374-8c17-525f-3bd0-7648f20b45f6 -11:51.229 [INFO] TestAgent.server: federation state anti-entropy synced -11:51.330 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:51.330 [INFO] TestAgent: Synced node info -11:51.330 [DEBUG] TestAgent: Node info in sync -11:51.562 [INFO] TestAgent: Requesting shutdown -11:51.562 [INFO] TestAgent.server: shutting down server -11:51.562 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:51.562 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:51.562 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:51.562 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:51.562 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:51.562 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:51.562 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:51.562 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:51.562 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:51.562 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:51.562 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:51.562Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:51.562Z [INFO] agent.router.manager: shutting down -11:51.562 [INFO] TestAgent: consul server down -11:51.562 [INFO] TestAgent: shutdown complete -11:51.562 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16393 network=tcp -11:51.562 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16393 network=udp -11:51.562 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16394 network=tcp -11:52.065 [INFO] TestAgent: Waiting for endpoints to shut down -11:52.065 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:08.403Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:08.413Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:08.414 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9036bae9-d6fd-2ef2-4ee2-49eb3913098e Address:127.0.0.1:23545}]" +48:08.415 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23545 [Follower]" leader= +48:08.415 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9036bae9-d6fd-2ef2-4ee2-49eb3913098e.dc1 127.0.0.1 +48:08.416 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9036bae9-d6fd-2ef2-4ee2-49eb3913098e 127.0.0.1 +2022-12-04T05:48:08.416Z [INFO] agent.router: Initializing LAN area manager +48:08.416 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23540 network=udp +48:08.416 [INFO] TestAgent.server: Adding LAN server: server="Node-9036bae9-d6fd-2ef2-4ee2-49eb3913098e (Addr: tcp/127.0.0.1:23545) (DC: dc1)" +48:08.416 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9036bae9-d6fd-2ef2-4ee2-49eb3913098e.dc1 area=wan +48:08.416 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23540 network=tcp +48:08.416 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23541 network=tcp +48:08.417 [INFO] TestAgent: started state syncer +48:08.417 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23546 network=tcp +48:08.471 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:08.472 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23545 [Candidate]" term=2 +48:08.472 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:08.472 [DEBUG] TestAgent.server.raft: vote granted: from=9036bae9-d6fd-2ef2-4ee2-49eb3913098e term=2 tally=1 +48:08.472 [INFO] TestAgent.server.raft: election won: tally=1 +48:08.472 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23545 [Leader]" +48:08.472 [INFO] TestAgent.server: cluster leadership acquired +48:08.472 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23545 +48:08.484 [INFO] TestAgent.server: New leader elected: payload=Node-9036bae9-d6fd-2ef2-4ee2-49eb3913098e +48:08.485 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:08.487 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:08.487 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:08.487 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:08.487 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:08.487 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:08.488 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9036bae9-d6fd-2ef2-4ee2-49eb3913098e +48:08.488 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9036bae9-d6fd-2ef2-4ee2-49eb3913098e +48:08.488 [INFO] TestAgent.server: federation state anti-entropy synced +48:08.544 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:08.545 [INFO] TestAgent: Synced node info +48:08.545 [DEBUG] TestAgent: Node info in sync +48:08.595 [INFO] TestAgent: Requesting shutdown +48:08.596 [INFO] TestAgent.server: shutting down server +48:08.596 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:08.596 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:08.596 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:08.596 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:08.596 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:08.596 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:08.596 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:08.596 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:08.596 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:08.596 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:08.596 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:08.596Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:08.596Z [INFO] agent.router.manager: shutting down +48:08.596 [INFO] TestAgent: consul server down +48:08.596 [INFO] TestAgent: shutdown complete +48:08.596 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23540 network=tcp +48:08.596 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23540 network=udp +48:08.596 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23541 network=tcp +48:09.100 [INFO] TestAgent: Waiting for endpoints to shut down +48:09.100 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/another_service_registered_with_same_ID_as_a_sidecar_should_not_be_deregistered -2021-10-31T23:11:51.996Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:52.011Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:52.013 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ba90ac96-e897-81ee-c258-b25a56bebbac Address:127.0.0.1:16405}]" -11:52.013 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16405 [Follower]" leader= -11:52.013 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ba90ac96-e897-81ee-c258-b25a56bebbac.dc1 127.0.0.1 -11:52.014 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ba90ac96-e897-81ee-c258-b25a56bebbac 127.0.0.1 -2021-10-31T23:11:52.014Z [INFO] agent.router: Initializing LAN area manager -11:52.014 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16400 network=udp -11:52.015 [INFO] TestAgent.server: Adding LAN server: server="Node-ba90ac96-e897-81ee-c258-b25a56bebbac (Addr: tcp/127.0.0.1:16405) (DC: dc1)" -11:52.015 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ba90ac96-e897-81ee-c258-b25a56bebbac.dc1 area=wan -11:52.015 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16400 network=tcp -11:52.015 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16401 network=tcp -11:52.016 [INFO] TestAgent: started state syncer -11:52.016 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16406 network=tcp -11:52.057 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:52.058 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16405 [Candidate]" term=2 -11:52.058 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:52.058 [DEBUG] TestAgent.server.raft: vote granted: from=ba90ac96-e897-81ee-c258-b25a56bebbac term=2 tally=1 -11:52.058 [INFO] TestAgent.server.raft: election won: tally=1 -11:52.058 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16405 [Leader]" -11:52.058 [INFO] TestAgent.server: cluster leadership acquired -11:52.058 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16405 -11:52.059 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:52.060 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:52.060 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:52.060 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:52.060 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:52.060 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:52.061 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba90ac96-e897-81ee-c258-b25a56bebbac -11:52.061 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ba90ac96-e897-81ee-c258-b25a56bebbac -11:52.061 [INFO] TestAgent.server: New leader elected: payload=Node-ba90ac96-e897-81ee-c258-b25a56bebbac -11:52.065 [INFO] TestAgent.server: federation state anti-entropy synced -11:52.157 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:52.158 [INFO] TestAgent: Synced node info -11:52.158 [DEBUG] TestAgent: Node info in sync -11:52.358 [DEBUG] TestAgent: Node info in sync -11:52.359 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -11:52.359 [INFO] TestAgent: Synced service: service=web -11:52.359 [DEBUG] TestAgent: removed service: service=web -11:52.359 [DEBUG] TestAgent: Node info in sync -11:52.359 [DEBUG] TestAgent: Service in sync: service=web-sidecar-proxy -11:52.360 [INFO] TestAgent: Deregistered service: service=web -11:52.360 [INFO] TestAgent: Requesting shutdown -11:52.360 [INFO] TestAgent.server: shutting down server -11:52.360 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:52.360 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:52.360 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:52.360 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:52.360 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:52.360 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:52.360 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:52.360 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:52.360 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:52.360 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:52.360 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:52.360Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:52.360Z [INFO] agent.router.manager: shutting down -11:52.360 [INFO] TestAgent: consul server down -11:52.360 [INFO] TestAgent: shutdown complete -11:52.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16400 network=tcp -11:52.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16400 network=udp -11:52.360 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16401 network=tcp -11:52.860 [INFO] TestAgent: Waiting for endpoints to shut down -11:52.861 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:09.110Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:09.120Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:09.122 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c1788f05-487d-536a-0447-3aa364f75c5a Address:127.0.0.1:23552}]" +48:09.122 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c1788f05-487d-536a-0447-3aa364f75c5a.dc1 127.0.0.1 +48:09.123 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c1788f05-487d-536a-0447-3aa364f75c5a 127.0.0.1 +2022-12-04T05:48:09.123Z [INFO] agent.router: Initializing LAN area manager +48:09.123 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23547 network=udp +48:09.123 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23552 [Follower]" leader= +48:09.123 [INFO] TestAgent.server: Adding LAN server: server="Node-c1788f05-487d-536a-0447-3aa364f75c5a (Addr: tcp/127.0.0.1:23552) (DC: dc1)" +48:09.123 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c1788f05-487d-536a-0447-3aa364f75c5a.dc1 area=wan +48:09.123 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23547 network=tcp +48:09.124 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23548 network=tcp +48:09.124 [INFO] TestAgent: started state syncer +48:09.124 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23553 network=tcp +48:09.179 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:09.179 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23552 [Candidate]" term=2 +48:09.180 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:09.180 [DEBUG] TestAgent.server.raft: vote granted: from=c1788f05-487d-536a-0447-3aa364f75c5a term=2 tally=1 +48:09.180 [INFO] TestAgent.server.raft: election won: tally=1 +48:09.180 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23552 [Leader]" +48:09.180 [INFO] TestAgent.server: cluster leadership acquired +48:09.180 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23552 +48:09.180 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:09.182 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:09.182 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:09.182 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:09.182 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:09.182 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:09.182 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c1788f05-487d-536a-0447-3aa364f75c5a +48:09.182 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c1788f05-487d-536a-0447-3aa364f75c5a +48:09.182 [INFO] TestAgent.server: New leader elected: payload=Node-c1788f05-487d-536a-0447-3aa364f75c5a +48:09.183 [INFO] TestAgent.server: federation state anti-entropy synced +48:09.304 [INFO] TestAgent: Synced node info +48:09.304 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +48:09.305 [INFO] TestAgent: Synced service: service=web +48:09.305 [DEBUG] TestAgent: removed service: service=web +48:09.305 [DEBUG] TestAgent: Node info in sync +48:09.305 [DEBUG] TestAgent: Service in sync: service=web-sidecar-proxy +48:09.305 [INFO] TestAgent: Deregistered service: service=web +48:09.305 [INFO] TestAgent: Requesting shutdown +48:09.305 [INFO] TestAgent.server: shutting down server +48:09.305 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:09.305 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:09.305 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:09.305 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:09.305 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:09.305 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:09.305 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:09.305 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:09.305 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:09.305 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:09.305 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:09.305 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:09.305Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:09.305Z [INFO] agent.router.manager: shutting down +48:09.306 [INFO] TestAgent: consul server down +48:09.306 [INFO] TestAgent: shutdown complete +48:09.306 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23547 network=tcp +48:09.306 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23547 network=udp +48:09.306 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23548 network=tcp +48:09.806 [INFO] TestAgent: Waiting for endpoints to shut down +48:09.806 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/updates_to_sidecar_should_work -2021-10-31T23:11:52.088Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:52.114Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:52.116 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4874289f-b0e6-8c35-4465-a06d62716e22 Address:127.0.0.1:16412}]" -11:52.117 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16412 [Follower]" leader= -11:52.117 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4874289f-b0e6-8c35-4465-a06d62716e22.dc1 127.0.0.1 -11:52.118 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4874289f-b0e6-8c35-4465-a06d62716e22 127.0.0.1 -2021-10-31T23:11:52.118Z [INFO] agent.router: Initializing LAN area manager -11:52.118 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16407 network=udp -11:52.118 [INFO] TestAgent.server: Adding LAN server: server="Node-4874289f-b0e6-8c35-4465-a06d62716e22 (Addr: tcp/127.0.0.1:16412) (DC: dc1)" -11:52.118 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4874289f-b0e6-8c35-4465-a06d62716e22.dc1 area=wan -11:52.118 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16407 network=tcp -11:52.119 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16408 network=tcp -11:52.119 [INFO] TestAgent: started state syncer -11:52.119 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16413 network=tcp -11:52.161 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:52.161 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16412 [Candidate]" term=2 -11:52.162 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:52.162 [DEBUG] TestAgent.server.raft: vote granted: from=4874289f-b0e6-8c35-4465-a06d62716e22 term=2 tally=1 -11:52.162 [INFO] TestAgent.server.raft: election won: tally=1 -11:52.162 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16412 [Leader]" -11:52.162 [INFO] TestAgent.server: cluster leadership acquired -11:52.162 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16412 -11:52.163 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:52.164 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:52.164 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:52.164 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:52.164 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:52.164 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:52.164 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4874289f-b0e6-8c35-4465-a06d62716e22 -11:52.164 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4874289f-b0e6-8c35-4465-a06d62716e22 -11:52.165 [INFO] TestAgent.server: New leader elected: payload=Node-4874289f-b0e6-8c35-4465-a06d62716e22 -11:52.165 [INFO] TestAgent.server: federation state anti-entropy synced -11:52.201 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:52.201 [INFO] TestAgent: Synced node info -11:52.201 [DEBUG] TestAgent: Node info in sync -11:52.430 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:52.430 [DEBUG] TestAgent: Node info in sync -11:52.458 [DEBUG] TestAgent: Node info in sync -11:52.459 [INFO] TestAgent: Synced service: service=web -11:52.459 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -11:52.459 [DEBUG] TestAgent: removed service: service=web -11:52.459 [DEBUG] TestAgent: Node info in sync -11:52.459 [DEBUG] TestAgent: Service in sync: service=web-sidecar-proxy -11:52.459 [INFO] TestAgent: Deregistered service: service=web -11:52.460 [INFO] TestAgent: Requesting shutdown -11:52.460 [INFO] TestAgent.server: shutting down server -11:52.460 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:52.460 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:52.460 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:52.460 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:52.460 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:52.460 [DEBUG] TestAgent: Node info in sync -11:52.460 [DEBUG] TestAgent: Service in sync: service=web-sidecar-proxy -11:52.460 [DEBUG] TestAgent: Node info in sync -11:52.460 [DEBUG] TestAgent: Service in sync: service=web-sidecar-proxy -11:52.460 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:52.460 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:52.460 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:52.460 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:52.460 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:52.460 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:52.460Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:52.460Z [INFO] agent.router.manager: shutting down -11:52.460 [INFO] TestAgent: consul server down -11:52.460 [INFO] TestAgent: shutdown complete -11:52.460 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16407 network=tcp -11:52.460 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16407 network=udp -11:52.460 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16408 network=tcp -11:52.961 [INFO] TestAgent: Waiting for endpoints to shut down -11:52.961 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:09.112Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:09.124Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:09.125 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c62ca8c1-a343-2b9a-8067-77cbbc418da0 Address:127.0.0.1:23559}]" +48:09.125 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23559 [Follower]" leader= +48:09.126 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c62ca8c1-a343-2b9a-8067-77cbbc418da0.dc1 127.0.0.1 +48:09.126 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c62ca8c1-a343-2b9a-8067-77cbbc418da0 127.0.0.1 +2022-12-04T05:48:09.126Z [INFO] agent.router: Initializing LAN area manager +48:09.127 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23554 network=udp +48:09.127 [INFO] TestAgent.server: Adding LAN server: server="Node-c62ca8c1-a343-2b9a-8067-77cbbc418da0 (Addr: tcp/127.0.0.1:23559) (DC: dc1)" +48:09.127 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c62ca8c1-a343-2b9a-8067-77cbbc418da0.dc1 area=wan +48:09.127 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23554 network=tcp +48:09.128 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23555 network=tcp +48:09.128 [INFO] TestAgent: started state syncer +48:09.128 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23560 network=tcp +48:09.169 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:09.170 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23559 [Candidate]" term=2 +48:09.170 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:09.170 [DEBUG] TestAgent.server.raft: vote granted: from=c62ca8c1-a343-2b9a-8067-77cbbc418da0 term=2 tally=1 +48:09.170 [INFO] TestAgent.server.raft: election won: tally=1 +48:09.170 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23559 [Leader]" +48:09.170 [INFO] TestAgent.server: cluster leadership acquired +48:09.170 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23559 +48:09.171 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:09.172 [INFO] TestAgent.server: New leader elected: payload=Node-c62ca8c1-a343-2b9a-8067-77cbbc418da0 +48:09.172 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:09.172 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:09.172 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:09.172 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:09.172 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:09.172 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c62ca8c1-a343-2b9a-8067-77cbbc418da0 +48:09.172 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c62ca8c1-a343-2b9a-8067-77cbbc418da0 +48:09.173 [INFO] TestAgent.server: federation state anti-entropy synced +48:09.314 [INFO] TestAgent: Synced node info +48:09.314 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +48:09.315 [INFO] TestAgent: Synced service: service=web +48:09.315 [DEBUG] TestAgent: removed service: service=web +48:09.315 [DEBUG] TestAgent: Node info in sync +48:09.315 [DEBUG] TestAgent: Service in sync: service=web-sidecar-proxy +48:09.327 [INFO] TestAgent: Deregistered service: service=web +48:09.327 [INFO] TestAgent: Requesting shutdown +48:09.329 [INFO] TestAgent.server: shutting down server +48:09.329 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:09.329 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:09.329 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:09.329 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:09.329 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:09.329 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:09.329 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:09.329 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:09.329 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:09.329 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:09.329 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:09.329 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:48:09.330Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:09.330Z [INFO] agent.router.manager: shutting down +48:09.330 [INFO] TestAgent: consul server down +48:09.330 [INFO] TestAgent: shutdown complete +48:09.330 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23554 network=tcp +48:09.330 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23554 network=udp +48:09.330 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23555 network=tcp +48:09.832 [INFO] TestAgent: Waiting for endpoints to shut down +48:09.832 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/updates_to_sidecar_should_work -2021-10-31T23:11:52.929Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:52.963Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:52.965 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a5f9172c-28a3-169a-dcf4-2b133b62c983 Address:127.0.0.1:16419}]" -11:52.965 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16419 [Follower]" leader= -11:52.965 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a5f9172c-28a3-169a-dcf4-2b133b62c983.dc1 127.0.0.1 -11:52.966 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a5f9172c-28a3-169a-dcf4-2b133b62c983 127.0.0.1 -2021-10-31T23:11:52.966Z [INFO] agent.router: Initializing LAN area manager -11:52.966 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16414 network=udp -11:52.966 [INFO] TestAgent.server: Adding LAN server: server="Node-a5f9172c-28a3-169a-dcf4-2b133b62c983 (Addr: tcp/127.0.0.1:16419) (DC: dc1)" -11:52.966 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a5f9172c-28a3-169a-dcf4-2b133b62c983.dc1 area=wan -11:52.967 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16414 network=tcp -11:52.967 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16415 network=tcp -11:52.967 [INFO] TestAgent: started state syncer -11:52.968 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16420 network=tcp -11:53.013 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:53.013 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16419 [Candidate]" term=2 -11:53.014 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:53.014 [DEBUG] TestAgent.server.raft: vote granted: from=a5f9172c-28a3-169a-dcf4-2b133b62c983 term=2 tally=1 -11:53.014 [INFO] TestAgent.server.raft: election won: tally=1 -11:53.014 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16419 [Leader]" -11:53.014 [INFO] TestAgent.server: cluster leadership acquired -11:53.014 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16419 -11:53.014 [INFO] TestAgent.server: New leader elected: payload=Node-a5f9172c-28a3-169a-dcf4-2b133b62c983 -11:53.015 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:53.016 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:53.016 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:53.016 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:53.016 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:53.016 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:53.016 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a5f9172c-28a3-169a-dcf4-2b133b62c983 -11:53.016 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a5f9172c-28a3-169a-dcf4-2b133b62c983 -11:53.017 [INFO] TestAgent.server: federation state anti-entropy synced -11:53.159 [INFO] TestAgent: Synced node info -11:53.159 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -11:53.160 [INFO] TestAgent: Synced service: service=web -11:53.160 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 -11:53.160 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 -11:53.160 [DEBUG] TestAgent: removed service: service=web -11:53.160 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 -11:53.160 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 -11:53.160 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -11:53.160 [DEBUG] TestAgent: Node info in sync -11:53.160 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -11:53.161 [INFO] TestAgent: Deregistered service: service=web -11:53.161 [INFO] TestAgent: Requesting shutdown -11:53.161 [INFO] TestAgent.server: shutting down server -11:53.161 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:53.161 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:53.161 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:53.161 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:53.161 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:53.161 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:53.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:53.161 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:53.161 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:53.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:53.161 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:53.161 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:53.161Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:53.161Z [INFO] agent.router.manager: shutting down -11:53.161 [INFO] TestAgent: consul server down -11:53.161 [INFO] TestAgent: shutdown complete -11:53.161 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16414 network=tcp -11:53.161 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16414 network=udp -11:53.161 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16415 network=tcp -11:53.665 [INFO] TestAgent: Waiting for endpoints to shut down -11:53.665 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:09.820Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:09.837Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:09.839 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6c0ad4f4-6d33-2142-bd93-c4f71819ab07 Address:127.0.0.1:23566}]" +48:09.839 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6c0ad4f4-6d33-2142-bd93-c4f71819ab07.dc1 127.0.0.1 +48:09.840 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23566 [Follower]" leader= +48:09.840 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6c0ad4f4-6d33-2142-bd93-c4f71819ab07 127.0.0.1 +2022-12-04T05:48:09.840Z [INFO] agent.router: Initializing LAN area manager +48:09.841 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23561 network=udp +48:09.841 [INFO] TestAgent.server: Adding LAN server: server="Node-6c0ad4f4-6d33-2142-bd93-c4f71819ab07 (Addr: tcp/127.0.0.1:23566) (DC: dc1)" +48:09.841 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6c0ad4f4-6d33-2142-bd93-c4f71819ab07.dc1 area=wan +48:09.841 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23561 network=tcp +48:09.841 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23562 network=tcp +48:09.841 [INFO] TestAgent: started state syncer +48:09.841 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23567 network=tcp +48:09.879 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:09.879 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23566 [Candidate]" term=2 +48:09.880 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:09.880 [DEBUG] TestAgent.server.raft: vote granted: from=6c0ad4f4-6d33-2142-bd93-c4f71819ab07 term=2 tally=1 +48:09.880 [INFO] TestAgent.server.raft: election won: tally=1 +48:09.880 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23566 [Leader]" +48:09.880 [INFO] TestAgent.server: cluster leadership acquired +48:09.880 [INFO] TestAgent.server: New leader elected: payload=Node-6c0ad4f4-6d33-2142-bd93-c4f71819ab07 +48:09.880 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23566 +48:09.881 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:09.882 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:09.882 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:09.882 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:09.882 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:09.882 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:09.882 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6c0ad4f4-6d33-2142-bd93-c4f71819ab07 +48:09.882 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6c0ad4f4-6d33-2142-bd93-c4f71819ab07 +48:09.882 [INFO] TestAgent.server: federation state anti-entropy synced +48:10.042 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:10.042 [INFO] TestAgent: Synced node info +48:10.042 [DEBUG] TestAgent: Node info in sync +48:10.093 [DEBUG] TestAgent: Node info in sync +48:10.094 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +48:10.094 [INFO] TestAgent: Synced service: service=web +48:10.094 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +48:10.094 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +48:10.094 [DEBUG] TestAgent: removed service: service=web +48:10.095 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 +48:10.095 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 +48:10.095 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +48:10.095 [DEBUG] TestAgent: Node info in sync +48:10.095 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +48:10.095 [INFO] TestAgent: Deregistered service: service=web +48:10.095 [INFO] TestAgent: Requesting shutdown +48:10.096 [INFO] TestAgent.server: shutting down server +48:10.096 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:10.096 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:10.096 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:10.096 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:10.096 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:10.096 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:10.096 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:10.096 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:10.096 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:10.096Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:10.096Z [INFO] agent.router.manager: shutting down +48:10.096 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" +48:10.096 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:10.096 [INFO] TestAgent: consul server down +48:10.096 [INFO] TestAgent: shutdown complete +48:10.096 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23561 network=tcp +48:10.096 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23561 network=udp +48:10.096 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23562 network=tcp +48:10.596 [INFO] TestAgent: Waiting for endpoints to shut down +48:10.596 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/normal/update_that_removes_sidecar_should_NOT_deregister_it -2021-10-31T23:11:52.992Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:53.027Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:53.029 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ece57bc8-0ab8-00ed-63d7-c41725e5da82 Address:127.0.0.1:16426}]" -11:53.029 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16426 [Follower]" leader= -11:53.034 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ece57bc8-0ab8-00ed-63d7-c41725e5da82.dc1 127.0.0.1 -11:53.035 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ece57bc8-0ab8-00ed-63d7-c41725e5da82 127.0.0.1 -2021-10-31T23:11:53.035Z [INFO] agent.router: Initializing LAN area manager -11:53.035 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16421 network=udp -11:53.046 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ece57bc8-0ab8-00ed-63d7-c41725e5da82.dc1 area=wan -11:53.046 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16421 network=tcp -11:53.047 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16422 network=tcp -11:53.047 [INFO] TestAgent: started state syncer -11:53.047 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16427 network=tcp -11:53.048 [INFO] TestAgent.server: Adding LAN server: server="Node-ece57bc8-0ab8-00ed-63d7-c41725e5da82 (Addr: tcp/127.0.0.1:16426) (DC: dc1)" -11:53.085 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:53.086 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16426 [Candidate]" term=2 -11:53.086 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:53.086 [DEBUG] TestAgent.server.raft: vote granted: from=ece57bc8-0ab8-00ed-63d7-c41725e5da82 term=2 tally=1 -11:53.086 [INFO] TestAgent.server.raft: election won: tally=1 -11:53.086 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16426 [Leader]" -11:53.086 [INFO] TestAgent.server: cluster leadership acquired -11:53.086 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16426 -11:53.087 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:53.087 [INFO] TestAgent.server: New leader elected: payload=Node-ece57bc8-0ab8-00ed-63d7-c41725e5da82 -11:53.088 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:53.088 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:53.088 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:53.088 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:53.088 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:53.088 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ece57bc8-0ab8-00ed-63d7-c41725e5da82 -11:53.088 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ece57bc8-0ab8-00ed-63d7-c41725e5da82 -11:53.094 [INFO] TestAgent.server: federation state anti-entropy synced -11:53.398 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy -11:53.399 [INFO] TestAgent: Synced node info -11:53.399 [INFO] TestAgent: Synced service: service=web -11:53.400 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -11:53.400 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 -11:53.400 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 -11:53.401 [DEBUG] TestAgent: removed service: service=web -11:53.402 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 -11:53.402 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 -11:53.402 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -11:53.402 [DEBUG] TestAgent: Node info in sync -11:53.402 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -11:53.402 [INFO] TestAgent: Deregistered service: service=web -11:53.402 [INFO] TestAgent: Requesting shutdown -11:53.403 [INFO] TestAgent.server: shutting down server -11:53.403 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:53.403 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:53.403 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:53.403 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:53.403 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:53.403 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:53.403 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:53.403 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:53.403 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:53.403 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:53.403 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:53.403 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:53.403Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:53.403Z [INFO] agent.router.manager: shutting down -11:53.403 [INFO] TestAgent: consul server down -11:53.403 [INFO] TestAgent: shutdown complete -11:53.403 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16421 network=tcp -11:53.403 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16421 network=udp -11:53.403 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16422 network=tcp -11:53.907 [INFO] TestAgent: Waiting for endpoints to shut down -11:53.907 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:09.848Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:09.857Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:09.858 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e5e594f2-012d-7a7d-2143-6361581d5443 Address:127.0.0.1:23573}]" +48:09.858 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23573 [Follower]" leader= +48:09.859 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e5e594f2-012d-7a7d-2143-6361581d5443.dc1 127.0.0.1 +48:09.859 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e5e594f2-012d-7a7d-2143-6361581d5443 127.0.0.1 +2022-12-04T05:48:09.859Z [INFO] agent.router: Initializing LAN area manager +48:09.859 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e5e594f2-012d-7a7d-2143-6361581d5443.dc1 area=wan +48:09.859 [INFO] TestAgent.server: Adding LAN server: server="Node-e5e594f2-012d-7a7d-2143-6361581d5443 (Addr: tcp/127.0.0.1:23573) (DC: dc1)" +48:09.859 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23568 network=udp +48:09.859 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23568 network=tcp +48:09.860 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23569 network=tcp +48:09.860 [INFO] TestAgent: started state syncer +48:09.860 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23574 network=tcp +48:09.898 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:09.898 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23573 [Candidate]" term=2 +48:09.899 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:09.899 [DEBUG] TestAgent.server.raft: vote granted: from=e5e594f2-012d-7a7d-2143-6361581d5443 term=2 tally=1 +48:09.899 [INFO] TestAgent.server.raft: election won: tally=1 +48:09.899 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23573 [Leader]" +48:09.899 [INFO] TestAgent.server: cluster leadership acquired +48:09.899 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23573 +48:09.899 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:09.900 [INFO] TestAgent.server: New leader elected: payload=Node-e5e594f2-012d-7a7d-2143-6361581d5443 +48:09.900 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:09.901 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:09.901 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:09.901 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:09.901 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:09.901 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e5e594f2-012d-7a7d-2143-6361581d5443 +48:09.901 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e5e594f2-012d-7a7d-2143-6361581d5443 +48:09.908 [INFO] TestAgent.server: federation state anti-entropy synced +48:10.257 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:10.257 [INFO] TestAgent: Synced node info +48:10.289 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy +48:10.289 [DEBUG] TestAgent: Node info in sync +48:10.289 [INFO] TestAgent: Synced service: service=web +48:10.290 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +48:10.290 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:1 +48:10.290 [DEBUG] TestAgent: Check in sync: check=service:web-sidecar-proxy:2 +48:10.290 [DEBUG] TestAgent: removed service: service=web +48:10.290 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:1 +48:10.290 [DEBUG] TestAgent: removed check: check=service:web-sidecar-proxy:2 +48:10.290 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +48:10.290 [DEBUG] TestAgent: Node info in sync +48:10.291 [INFO] TestAgent: Deregistered service: service=web +48:10.291 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +48:10.291 [INFO] TestAgent: Requesting shutdown +48:10.291 [INFO] TestAgent.server: shutting down server +48:10.291 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:10.291 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:10.291 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:10.291 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:10.291 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:10.291 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:10.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:10.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:10.291 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:10.291 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:10.291 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:10.291Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:10.291Z [INFO] agent.router.manager: shutting down +48:10.292 [INFO] TestAgent: consul server down +48:10.292 [INFO] TestAgent: shutdown complete +48:10.292 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23568 network=tcp +48:10.292 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23568 network=udp +48:10.292 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23569 network=tcp +48:10.792 [INFO] TestAgent: Waiting for endpoints to shut down +48:10.792 [INFO] TestAgent: Endpoints down === RUN TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/update_that_removes_sidecar_should_NOT_deregister_it -2021-10-31T23:11:53.698Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:53.730Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:53.732 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a1a04569-20b8-950d-4ac9-d67798b560a5 Address:127.0.0.1:16433}]" -11:53.733 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a1a04569-20b8-950d-4ac9-d67798b560a5.dc1 127.0.0.1 -11:53.733 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a1a04569-20b8-950d-4ac9-d67798b560a5 127.0.0.1 -2021-10-31T23:11:53.733Z [INFO] agent.router: Initializing LAN area manager -11:53.733 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16433 [Follower]" leader= -11:53.734 [INFO] TestAgent.server: Adding LAN server: server="Node-a1a04569-20b8-950d-4ac9-d67798b560a5 (Addr: tcp/127.0.0.1:16433) (DC: dc1)" -11:53.734 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a1a04569-20b8-950d-4ac9-d67798b560a5.dc1 area=wan -11:53.742 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16428 network=udp -11:53.745 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16428 network=tcp -11:53.746 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16429 network=tcp -11:53.746 [INFO] TestAgent: started state syncer -11:53.746 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16434 network=tcp -11:53.775 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:53.775 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16433 [Candidate]" term=2 -11:53.775 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:53.775 [DEBUG] TestAgent.server.raft: vote granted: from=a1a04569-20b8-950d-4ac9-d67798b560a5 term=2 tally=1 -11:53.775 [INFO] TestAgent.server.raft: election won: tally=1 -11:53.775 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16433 [Leader]" -11:53.775 [INFO] TestAgent.server: cluster leadership acquired -11:53.775 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16433 -11:53.776 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:53.777 [INFO] TestAgent.server: New leader elected: payload=Node-a1a04569-20b8-950d-4ac9-d67798b560a5 -11:53.777 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:53.778 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:53.778 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:53.778 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:53.778 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:53.778 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1a04569-20b8-950d-4ac9-d67798b560a5 -11:53.778 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a1a04569-20b8-950d-4ac9-d67798b560a5 -11:53.778 [INFO] TestAgent.server: federation state anti-entropy synced -11:53.818 [INFO] TestAgent: Synced node info -11:53.819 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -11:53.819 [INFO] TestAgent: Synced service: service=web -11:53.819 [DEBUG] TestAgent: removed service: service=web -11:53.819 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -11:53.819 [DEBUG] TestAgent: Node info in sync -11:53.820 [INFO] TestAgent: Deregistered service: service=web -11:53.820 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -11:53.820 [INFO] TestAgent: Requesting shutdown -11:53.820 [INFO] TestAgent.server: shutting down server -11:53.820 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:53.820 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:53.820 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:53.820 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:53.820 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:53.820 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:53.820 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:53.820 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:53.820 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:53.820 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:53.820 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:53.820 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:53.820Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:53.820Z [INFO] agent.router.manager: shutting down -11:53.821 [INFO] TestAgent: consul server down -11:53.821 [INFO] TestAgent: shutdown complete -11:53.821 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16428 network=tcp -11:53.821 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16428 network=udp -11:53.821 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16429 network=tcp -11:54.321 [INFO] TestAgent: Waiting for endpoints to shut down -11:54.321 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:53.928Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:53.975Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:53.977 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:679c1af5-3a9b-6b52-6818-f213223d265d Address:127.0.0.1:16440}]" -11:53.977 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-679c1af5-3a9b-6b52-6818-f213223d265d.dc1 127.0.0.1 -11:53.977 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16440 [Follower]" leader= -11:53.978 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-679c1af5-3a9b-6b52-6818-f213223d265d 127.0.0.1 -2021-10-31T23:11:53.978Z [INFO] agent.router: Initializing LAN area manager -11:53.978 [INFO] TestAgent.server: Adding LAN server: server="Node-679c1af5-3a9b-6b52-6818-f213223d265d (Addr: tcp/127.0.0.1:16440) (DC: dc1)" -11:53.978 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16435 network=udp -11:53.978 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16435 network=tcp -11:53.978 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-679c1af5-3a9b-6b52-6818-f213223d265d.dc1 area=wan -11:53.979 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16436 network=tcp -11:53.979 [INFO] TestAgent: started state syncer -11:53.979 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16441 network=tcp -11:54.022 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:54.022 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16440 [Candidate]" term=2 -11:54.022 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:54.022 [DEBUG] TestAgent.server.raft: vote granted: from=679c1af5-3a9b-6b52-6818-f213223d265d term=2 tally=1 -11:54.022 [INFO] TestAgent.server.raft: election won: tally=1 -11:54.022 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16440 [Leader]" -11:54.022 [INFO] TestAgent.server: cluster leadership acquired -11:54.023 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16440 -11:54.023 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:54.025 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:54.025 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:54.025 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:54.025 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:54.025 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:54.025 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-679c1af5-3a9b-6b52-6818-f213223d265d -11:54.025 [INFO] TestAgent.server: member joined, marking health alive: member=Node-679c1af5-3a9b-6b52-6818-f213223d265d -11:54.026 [INFO] TestAgent.server: New leader elected: payload=Node-679c1af5-3a9b-6b52-6818-f213223d265d -11:54.026 [INFO] TestAgent.server: federation state anti-entropy synced -11:54.158 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy -11:54.159 [INFO] TestAgent: Synced node info -11:54.159 [INFO] TestAgent: Synced service: service=web -11:54.160 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -11:54.160 [DEBUG] TestAgent: removed service: service=web -11:54.160 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -11:54.160 [DEBUG] TestAgent: Node info in sync -11:54.160 [INFO] TestAgent: Deregistered service: service=web -11:54.161 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy -11:54.161 [INFO] TestAgent: Requesting shutdown -11:54.161 [INFO] TestAgent.server: shutting down server -11:54.161 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:54.161 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:54.161 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:54.161 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:54.161 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:54.161 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:54.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:54.161 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:54.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:54.161 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:54.161 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:54.161 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:54.161Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:54.161Z [INFO] agent.router.manager: shutting down -11:54.161 [INFO] TestAgent: consul server down -11:54.161 [INFO] TestAgent: shutdown complete -11:54.161 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16435 network=tcp -11:54.162 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16435 network=udp -11:54.162 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16436 network=tcp -11:54.662 [INFO] TestAgent: Waiting for endpoints to shut down -11:54.662 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:10.607Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:10.616Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:10.618 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:01f0bcc4-f768-363f-d2d9-8a16f20c00e3 Address:127.0.0.1:23580}]" +48:10.618 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23580 [Follower]" leader= +48:10.618 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-01f0bcc4-f768-363f-d2d9-8a16f20c00e3.dc1 127.0.0.1 +48:10.619 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-01f0bcc4-f768-363f-d2d9-8a16f20c00e3 127.0.0.1 +2022-12-04T05:48:10.619Z [INFO] agent.router: Initializing LAN area manager +48:10.619 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23575 network=udp +48:10.619 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-01f0bcc4-f768-363f-d2d9-8a16f20c00e3.dc1 area=wan +48:10.619 [INFO] TestAgent.server: Adding LAN server: server="Node-01f0bcc4-f768-363f-d2d9-8a16f20c00e3 (Addr: tcp/127.0.0.1:23580) (DC: dc1)" +48:10.619 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23575 network=tcp +48:10.619 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23576 network=tcp +48:10.619 [INFO] TestAgent: started state syncer +48:10.620 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23581 network=tcp +48:10.678 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:10.679 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23580 [Candidate]" term=2 +48:10.679 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:10.679 [DEBUG] TestAgent.server.raft: vote granted: from=01f0bcc4-f768-363f-d2d9-8a16f20c00e3 term=2 tally=1 +48:10.679 [INFO] TestAgent.server.raft: election won: tally=1 +48:10.679 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23580 [Leader]" +48:10.679 [INFO] TestAgent.server: cluster leadership acquired +48:10.679 [INFO] TestAgent.server: New leader elected: payload=Node-01f0bcc4-f768-363f-d2d9-8a16f20c00e3 +48:10.679 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23580 +48:10.680 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:10.681 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:10.681 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:10.681 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:10.681 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:10.681 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:10.681 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-01f0bcc4-f768-363f-d2d9-8a16f20c00e3 +48:10.681 [INFO] TestAgent.server: member joined, marking health alive: member=Node-01f0bcc4-f768-363f-d2d9-8a16f20c00e3 +48:10.681 [INFO] TestAgent.server: federation state anti-entropy synced +48:10.689 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:10.689 [INFO] TestAgent: Synced node info +48:10.876 [DEBUG] TestAgent: Node info in sync +48:10.877 [INFO] TestAgent: Synced service: service=web +48:10.877 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +48:10.877 [DEBUG] TestAgent: removed service: service=web +48:10.877 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +48:10.877 [DEBUG] TestAgent: Node info in sync +48:10.878 [INFO] TestAgent: Deregistered service: service=web +48:10.878 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +48:10.878 [INFO] TestAgent: Requesting shutdown +48:10.878 [INFO] TestAgent.server: shutting down server +48:10.878 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:10.878 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:10.878 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:10.878 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:10.878 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:10.878 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:10.878 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:10.878 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:10.878 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:10.878 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:10.878 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:10.879Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:10.879Z [INFO] agent.router.manager: shutting down +48:10.879 [INFO] TestAgent: consul server down +48:10.879 [INFO] TestAgent: shutdown complete +48:10.879 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23575 network=tcp +48:10.879 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23575 network=udp +48:10.879 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23576 network=tcp +48:11.379 [INFO] TestAgent: Waiting for endpoints to shut down +48:11.379 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:10.802Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:10.812Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:10.814 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1afe3129-a5c7-7226-0f3c-34ce81946faa Address:127.0.0.1:23587}]" +48:10.820 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23587 [Follower]" leader= +48:10.821 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1afe3129-a5c7-7226-0f3c-34ce81946faa.dc1 127.0.0.1 +48:10.821 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1afe3129-a5c7-7226-0f3c-34ce81946faa 127.0.0.1 +2022-12-04T05:48:10.821Z [INFO] agent.router: Initializing LAN area manager +48:10.822 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23582 network=udp +48:10.822 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1afe3129-a5c7-7226-0f3c-34ce81946faa.dc1 area=wan +48:10.822 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23582 network=tcp +48:10.822 [INFO] TestAgent.server: Adding LAN server: server="Node-1afe3129-a5c7-7226-0f3c-34ce81946faa (Addr: tcp/127.0.0.1:23587) (DC: dc1)" +48:10.824 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23583 network=tcp +48:10.824 [INFO] TestAgent: started state syncer +48:10.824 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23588 network=tcp +48:10.861 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:10.861 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23587 [Candidate]" term=2 +48:10.861 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:10.861 [DEBUG] TestAgent.server.raft: vote granted: from=1afe3129-a5c7-7226-0f3c-34ce81946faa term=2 tally=1 +48:10.861 [INFO] TestAgent.server.raft: election won: tally=1 +48:10.861 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23587 [Leader]" +48:10.861 [INFO] TestAgent.server: cluster leadership acquired +48:10.861 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23587 +48:10.862 [INFO] TestAgent.server: New leader elected: payload=Node-1afe3129-a5c7-7226-0f3c-34ce81946faa +48:10.862 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:10.864 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:10.864 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:10.864 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:10.864 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:10.864 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:10.864 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1afe3129-a5c7-7226-0f3c-34ce81946faa +48:10.864 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1afe3129-a5c7-7226-0f3c-34ce81946faa +48:10.864 [INFO] TestAgent.server: federation state anti-entropy synced +48:11.146 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:11.147 [INFO] TestAgent: Synced node info +48:11.147 [DEBUG] TestAgent: Node info in sync +48:11.158 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy +48:11.158 [DEBUG] TestAgent: Node info in sync +48:11.158 [INFO] TestAgent: Synced service: service=web +48:11.159 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +48:11.159 [DEBUG] TestAgent: removed service: service=web +48:11.159 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +48:11.159 [DEBUG] TestAgent: Node info in sync +48:11.159 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +48:11.159 [INFO] TestAgent: Deregistered service: service=web +48:11.160 [INFO] TestAgent: Deregistered service: service=web-sidecar-proxy +48:11.160 [INFO] TestAgent: Requesting shutdown +48:11.160 [INFO] TestAgent.server: shutting down server +48:11.160 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +48:11.160 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:11.160 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:11.160 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:11.160 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:11.160 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:11.160 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:11.160 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:11.160 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:11.160 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:11.160 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:11.160 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:11.160Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:11.160Z [INFO] agent.router.manager: shutting down +48:11.160 [INFO] TestAgent: consul server down +48:11.160 [INFO] TestAgent: shutdown complete +48:11.160 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23582 network=tcp +48:11.160 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23582 network=udp +48:11.160 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23583 network=tcp +48:11.660 [INFO] TestAgent: Waiting for endpoints to shut down +48:11.660 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar (0.00s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal (12.58s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/sanity_check_no_sidecar_case (0.74s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/default_sidecar (0.74s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_defaults (0.68s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_denied (0.97s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_not_for_sidecar (0.75s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_and_sidecar_but_not_sidecar's_overridden_destination (1.07s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_not_for_overridden_sidecar (0.80s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_and_overridden_for_sidecar (1.01s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_check_definition_in_sidecar (0.84s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_checks_definitions_in_sidecar (0.77s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_check_status_in_sidecar (0.90s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_checks_status_in_sidecar (0.94s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/another_service_registered_with_same_ID_as_a_sidecar_should_not_be_deregistered (0.90s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/updates_to_sidecar_should_work (0.80s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/update_that_removes_sidecar_should_NOT_deregister_it (0.66s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager (12.92s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/sanity_check_no_sidecar_case (0.99s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/default_sidecar (1.04s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_defaults (0.72s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_denied (1.00s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_not_for_sidecar (0.90s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_and_sidecar_but_not_sidecar's_overridden_destination (0.67s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_not_for_overridden_sidecar (0.92s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_and_overridden_for_sidecar (0.86s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_check_definition_in_sidecar (0.90s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_checks_definitions_in_sidecar (0.62s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_check_status_in_sidecar (0.73s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_checks_status_in_sidecar (0.96s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/another_service_registered_with_same_ID_as_a_sidecar_should_not_be_deregistered (0.90s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/updates_to_sidecar_should_work (0.95s) - --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/update_that_removes_sidecar_should_NOT_deregister_it (0.75s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal (11.90s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/sanity_check_no_sidecar_case (0.77s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/default_sidecar (0.84s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_defaults (0.97s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_denied (0.85s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_not_for_sidecar (0.84s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_and_sidecar_but_not_sidecar's_overridden_destination (0.96s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_not_for_overridden_sidecar (0.62s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/ACL_OK_for_service_but_and_overridden_for_sidecar (0.72s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_check_definition_in_sidecar (0.83s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_checks_definitions_in_sidecar (0.74s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_check_status_in_sidecar (0.66s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/invalid_checks_status_in_sidecar (0.81s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/another_service_registered_with_same_ID_as_a_sidecar_should_not_be_deregistered (0.71s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/updates_to_sidecar_should_work (0.79s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/normal/update_that_removes_sidecar_should_NOT_deregister_it (0.78s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager (12.19s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/sanity_check_no_sidecar_case (0.91s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/default_sidecar (0.86s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_defaults (0.78s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_denied (0.90s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_not_for_sidecar (0.70s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_and_sidecar_but_not_sidecar's_overridden_destination (0.90s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_not_for_overridden_sidecar (0.70s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/ACL_OK_for_service_but_and_overridden_for_sidecar (0.83s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_check_definition_in_sidecar (0.91s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_checks_definitions_in_sidecar (0.75s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_check_status_in_sidecar (0.68s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/invalid_checks_status_in_sidecar (0.71s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/another_service_registered_with_same_ID_as_a_sidecar_should_not_be_deregistered (0.73s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/updates_to_sidecar_should_work (0.96s) + --- PASS: TestAgent_RegisterServiceDeregisterService_Sidecar/service_manager/update_that_removes_sidecar_should_NOT_deregister_it (0.87s) === RUN TestAgent_RegisterService_UnmanagedConnectProxyInvalid === RUN TestAgent_RegisterService_UnmanagedConnectProxyInvalid/normal === PAUSE TestAgent_RegisterService_UnmanagedConnectProxyInvalid/normal @@ -9080,121 +9114,121 @@ === PAUSE TestAgent_RegisterService_UnmanagedConnectProxyInvalid/service_manager === CONT TestAgent_RegisterService_UnmanagedConnectProxyInvalid/normal === CONT TestAgent_RegisterService_UnmanagedConnectProxyInvalid/service_manager -2021-10-31T23:11:54.681Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:54.703Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:54.705 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:830edbba-bb05-0b1d-77c9-3b94525e79da Address:127.0.0.1:16447}]" -11:54.705 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16447 [Follower]" leader= -11:54.706 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-830edbba-bb05-0b1d-77c9-3b94525e79da.dc1 127.0.0.1 -11:54.707 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-830edbba-bb05-0b1d-77c9-3b94525e79da 127.0.0.1 -2021-10-31T23:11:54.707Z [INFO] agent.router: Initializing LAN area manager -11:54.707 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16442 network=udp -11:54.707 [INFO] TestAgent.server: Adding LAN server: server="Node-830edbba-bb05-0b1d-77c9-3b94525e79da (Addr: tcp/127.0.0.1:16447) (DC: dc1)" -11:54.707 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-830edbba-bb05-0b1d-77c9-3b94525e79da.dc1 area=wan -11:54.707 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16442 network=tcp -11:54.708 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16443 network=tcp -11:54.708 [INFO] TestAgent: started state syncer -11:54.708 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16448 network=tcp -11:54.744 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:54.744 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16447 [Candidate]" term=2 -11:54.744 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:54.744 [DEBUG] TestAgent.server.raft: vote granted: from=830edbba-bb05-0b1d-77c9-3b94525e79da term=2 tally=1 -11:54.744 [INFO] TestAgent.server.raft: election won: tally=1 -11:54.744 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16447 [Leader]" -11:54.744 [INFO] TestAgent.server: cluster leadership acquired -11:54.745 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16447 -11:54.745 [INFO] TestAgent.server: New leader elected: payload=Node-830edbba-bb05-0b1d-77c9-3b94525e79da -11:54.746 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:54.747 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:54.747 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:54.747 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:54.747 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:54.747 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:54.747 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-830edbba-bb05-0b1d-77c9-3b94525e79da -11:54.747 [INFO] TestAgent.server: member joined, marking health alive: member=Node-830edbba-bb05-0b1d-77c9-3b94525e79da -11:54.748 [INFO] TestAgent.server: federation state anti-entropy synced -11:54.942 [INFO] TestAgent: Requesting shutdown -11:54.942 [INFO] TestAgent.server: shutting down server -11:54.942 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:54.942 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:54.942 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:54.942 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:54.942 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:54.942 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:54.942 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:54.942 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:54.942 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:54.942 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:54.942 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:54.942 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:54.942Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:54.942Z [INFO] agent.router.manager: shutting down -11:54.943 [INFO] TestAgent: consul server down -11:54.943 [INFO] TestAgent: shutdown complete -11:54.943 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16442 network=tcp -11:54.943 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16442 network=udp -11:54.943 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16443 network=tcp -11:55.444 [INFO] TestAgent: Waiting for endpoints to shut down -11:55.444 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:54.689Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:54.727Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:54.730 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:64213964-a9d9-50bc-d959-176d9870d013 Address:127.0.0.1:16454}]" -11:54.730 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16454 [Follower]" leader= -11:54.731 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-64213964-a9d9-50bc-d959-176d9870d013.dc1 127.0.0.1 -11:54.732 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-64213964-a9d9-50bc-d959-176d9870d013 127.0.0.1 -2021-10-31T23:11:54.732Z [INFO] agent.router: Initializing LAN area manager -11:54.732 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16449 network=udp -11:54.733 [INFO] TestAgent.server: Adding LAN server: server="Node-64213964-a9d9-50bc-d959-176d9870d013 (Addr: tcp/127.0.0.1:16454) (DC: dc1)" -11:54.733 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-64213964-a9d9-50bc-d959-176d9870d013.dc1 area=wan -11:54.733 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16449 network=tcp -11:54.735 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16450 network=tcp -11:54.735 [INFO] TestAgent: started state syncer -11:54.735 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16455 network=tcp -11:54.769 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:54.770 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16454 [Candidate]" term=2 -11:54.770 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:54.770 [DEBUG] TestAgent.server.raft: vote granted: from=64213964-a9d9-50bc-d959-176d9870d013 term=2 tally=1 -11:54.770 [INFO] TestAgent.server.raft: election won: tally=1 -11:54.770 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16454 [Leader]" -11:54.770 [INFO] TestAgent.server: cluster leadership acquired -11:54.770 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16454 -11:54.771 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:54.772 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:54.772 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:54.772 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:54.772 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:54.772 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:54.772 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-64213964-a9d9-50bc-d959-176d9870d013 -11:54.772 [INFO] TestAgent.server: member joined, marking health alive: member=Node-64213964-a9d9-50bc-d959-176d9870d013 -11:54.773 [INFO] TestAgent.server: New leader elected: payload=Node-64213964-a9d9-50bc-d959-176d9870d013 -11:54.773 [INFO] TestAgent.server: federation state anti-entropy synced -11:55.022 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:55.022 [INFO] TestAgent: Synced node info -11:55.022 [DEBUG] TestAgent: Node info in sync -11:55.058 [INFO] TestAgent: Requesting shutdown -11:55.058 [INFO] TestAgent.server: shutting down server -11:55.058 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:55.058 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:55.058 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:55.058 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:55.058 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:55.058 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:55.058 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:55.058 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:55.058 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:55.058 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:55.058 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:55.058Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:55.058Z [INFO] agent.router.manager: shutting down -11:55.058 [INFO] TestAgent: consul server down -11:55.058 [INFO] TestAgent: shutdown complete -11:55.058 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16449 network=tcp -11:55.058 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16449 network=udp -11:55.058 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16450 network=tcp -11:55.561 [INFO] TestAgent: Waiting for endpoints to shut down -11:55.561 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:11.689Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:11.703Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:11.704 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4ddf1c89-c9c0-9eef-651f-ae893cb9c3b5 Address:127.0.0.1:23594}]" +48:11.704 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23594 [Follower]" leader= +48:11.704 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4ddf1c89-c9c0-9eef-651f-ae893cb9c3b5.dc1 127.0.0.1 +48:11.705 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4ddf1c89-c9c0-9eef-651f-ae893cb9c3b5 127.0.0.1 +2022-12-04T05:48:11.705Z [INFO] agent.router: Initializing LAN area manager +48:11.705 [INFO] TestAgent.server: Adding LAN server: server="Node-4ddf1c89-c9c0-9eef-651f-ae893cb9c3b5 (Addr: tcp/127.0.0.1:23594) (DC: dc1)" +48:11.705 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4ddf1c89-c9c0-9eef-651f-ae893cb9c3b5.dc1 area=wan +48:11.705 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23589 network=udp +48:11.705 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23589 network=tcp +48:11.706 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23590 network=tcp +48:11.706 [INFO] TestAgent: started state syncer +48:11.706 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23595 network=tcp +48:11.759 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:11.759 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23594 [Candidate]" term=2 +48:11.759 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:11.759 [DEBUG] TestAgent.server.raft: vote granted: from=4ddf1c89-c9c0-9eef-651f-ae893cb9c3b5 term=2 tally=1 +48:11.759 [INFO] TestAgent.server.raft: election won: tally=1 +48:11.759 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23594 [Leader]" +48:11.759 [INFO] TestAgent.server: cluster leadership acquired +48:11.760 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23594 +48:11.760 [INFO] TestAgent.server: New leader elected: payload=Node-4ddf1c89-c9c0-9eef-651f-ae893cb9c3b5 +48:11.760 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:11.782 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:11.782 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:11.782 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:11.782 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:11.782 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:11.782 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4ddf1c89-c9c0-9eef-651f-ae893cb9c3b5 +48:11.782 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4ddf1c89-c9c0-9eef-651f-ae893cb9c3b5 +48:11.784 [INFO] TestAgent.server: federation state anti-entropy synced +48:11.845 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:11.845 [INFO] TestAgent: Synced node info +48:11.906 [INFO] TestAgent: Requesting shutdown +48:11.906 [INFO] TestAgent.server: shutting down server +48:11.906 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:11.906 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:11.906 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:11.906 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:11.906 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:11.906 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:11.906 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:11.906 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:11.906 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:11.906 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:11.906 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:48:11.907Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:11.907Z [INFO] agent.router.manager: shutting down +48:11.907 [INFO] TestAgent: consul server down +48:11.907 [INFO] TestAgent: shutdown complete +48:11.907 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23589 network=tcp +48:11.907 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23589 network=udp +48:11.907 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23590 network=tcp +48:12.408 [INFO] TestAgent: Waiting for endpoints to shut down +48:12.408 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:11.693Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:11.706Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:11.708 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:712293da-b369-b687-5bdf-b426ab5dacaa Address:127.0.0.1:23601}]" +48:11.708 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23601 [Follower]" leader= +48:11.708 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-712293da-b369-b687-5bdf-b426ab5dacaa.dc1 127.0.0.1 +48:11.709 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-712293da-b369-b687-5bdf-b426ab5dacaa 127.0.0.1 +2022-12-04T05:48:11.709Z [INFO] agent.router: Initializing LAN area manager +48:11.709 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23596 network=udp +48:11.709 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-712293da-b369-b687-5bdf-b426ab5dacaa.dc1 area=wan +48:11.709 [INFO] TestAgent.server: Adding LAN server: server="Node-712293da-b369-b687-5bdf-b426ab5dacaa (Addr: tcp/127.0.0.1:23601) (DC: dc1)" +48:11.709 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23596 network=tcp +48:11.709 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23597 network=tcp +48:11.709 [INFO] TestAgent: started state syncer +48:11.709 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23602 network=tcp +48:11.776 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:11.776 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23601 [Candidate]" term=2 +48:11.777 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:11.777 [DEBUG] TestAgent.server.raft: vote granted: from=712293da-b369-b687-5bdf-b426ab5dacaa term=2 tally=1 +48:11.777 [INFO] TestAgent.server.raft: election won: tally=1 +48:11.777 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23601 [Leader]" +48:11.780 [INFO] TestAgent.server: cluster leadership acquired +48:11.780 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23601 +48:11.781 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:11.784 [INFO] TestAgent.server: New leader elected: payload=Node-712293da-b369-b687-5bdf-b426ab5dacaa +48:11.785 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:11.785 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:11.785 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:11.785 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:11.785 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:11.785 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-712293da-b369-b687-5bdf-b426ab5dacaa +48:11.785 [INFO] TestAgent.server: member joined, marking health alive: member=Node-712293da-b369-b687-5bdf-b426ab5dacaa +48:11.785 [INFO] TestAgent.server: federation state anti-entropy synced +48:11.954 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:11.955 [INFO] TestAgent: Synced node info +48:12.124 [INFO] TestAgent: Requesting shutdown +48:12.124 [INFO] TestAgent.server: shutting down server +48:12.125 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:12.125 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:12.125 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:12.125 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:12.125 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:12.125 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:12.125 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:12.125 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:12.125 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:12.125 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:12.125 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:12.125Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:12.125Z [INFO] agent.router.manager: shutting down +48:12.125 [INFO] TestAgent: consul server down +48:12.125 [INFO] TestAgent: shutdown complete +48:12.125 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23596 network=tcp +48:12.125 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23596 network=udp +48:12.125 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23597 network=tcp +48:12.625 [INFO] TestAgent: Waiting for endpoints to shut down +48:12.625 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_UnmanagedConnectProxyInvalid (0.00s) - --- PASS: TestAgent_RegisterService_UnmanagedConnectProxyInvalid/normal (0.78s) - --- PASS: TestAgent_RegisterService_UnmanagedConnectProxyInvalid/service_manager (0.90s) + --- PASS: TestAgent_RegisterService_UnmanagedConnectProxyInvalid/normal (0.75s) + --- PASS: TestAgent_RegisterService_UnmanagedConnectProxyInvalid/service_manager (0.96s) === RUN TestAgent_RegisterService_ConnectNative === RUN TestAgent_RegisterService_ConnectNative/normal === PAUSE TestAgent_RegisterService_ConnectNative/normal @@ -9202,128 +9236,126 @@ === PAUSE TestAgent_RegisterService_ConnectNative/service_manager === CONT TestAgent_RegisterService_ConnectNative/normal === CONT TestAgent_RegisterService_ConnectNative/service_manager -2021-10-31T23:11:55.585Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:55.611Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:55.612 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:11408094-7243-3edf-fc8f-bd4ac2c20619 Address:127.0.0.1:16461}]" -11:55.613 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-11408094-7243-3edf-fc8f-bd4ac2c20619.dc1 127.0.0.1 -11:55.613 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16461 [Follower]" leader= -11:55.614 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-11408094-7243-3edf-fc8f-bd4ac2c20619 127.0.0.1 -2021-10-31T23:11:55.614Z [INFO] agent.router: Initializing LAN area manager -11:55.614 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16456 network=udp -11:55.614 [INFO] TestAgent.server: Adding LAN server: server="Node-11408094-7243-3edf-fc8f-bd4ac2c20619 (Addr: tcp/127.0.0.1:16461) (DC: dc1)" -11:55.614 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-11408094-7243-3edf-fc8f-bd4ac2c20619.dc1 area=wan -11:55.615 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16456 network=tcp -11:55.615 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16457 network=tcp -11:55.615 [INFO] TestAgent: started state syncer -11:55.615 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16462 network=tcp -11:55.673 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:55.673 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16461 [Candidate]" term=2 -11:55.674 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:55.674 [DEBUG] TestAgent.server.raft: vote granted: from=11408094-7243-3edf-fc8f-bd4ac2c20619 term=2 tally=1 -11:55.674 [INFO] TestAgent.server.raft: election won: tally=1 -11:55.674 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16461 [Leader]" -11:55.674 [INFO] TestAgent.server: cluster leadership acquired -11:55.674 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16461 -11:55.675 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:55.676 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:55.676 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:55.676 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:55.676 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:55.676 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:55.676 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-11408094-7243-3edf-fc8f-bd4ac2c20619 -11:55.676 [INFO] TestAgent.server: member joined, marking health alive: member=Node-11408094-7243-3edf-fc8f-bd4ac2c20619 -11:55.677 [INFO] TestAgent.server: New leader elected: payload=Node-11408094-7243-3edf-fc8f-bd4ac2c20619 -11:55.677 [INFO] TestAgent.server: federation state anti-entropy synced -11:55.681 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:55.682 [INFO] TestAgent: Synced node info -11:55.748 [DEBUG] TestAgent: Node info in sync -11:55.749 [INFO] TestAgent: Synced service: service=web -11:55.749 [DEBUG] TestAgent: Check in sync: check=service:web -11:55.749 [INFO] TestAgent: Requesting shutdown -11:55.749 [INFO] TestAgent.server: shutting down server -11:55.749 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:55.749 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:55.749 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:55.749 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:55.749 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:55.749 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:55.749 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:55.749 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:55.749 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:55.749 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:55.749 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -11:55.749 [INFO] TestAgent: consul server down -11:55.749 [INFO] TestAgent: shutdown complete -11:55.750 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16456 network=tcp -2021-10-31T23:11:55.749Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:55.750Z [INFO] agent.router.manager: shutting down -11:55.750 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16456 network=udp -11:55.750 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16457 network=tcp -11:56.253 [INFO] TestAgent: Waiting for endpoints to shut down -11:56.253 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:55.599Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:55.629Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:55.638 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:19f7a085-9d65-06f3-ce9e-68683cc68b99 Address:127.0.0.1:16468}]" -11:55.639 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16468 [Follower]" leader= -11:55.639 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-19f7a085-9d65-06f3-ce9e-68683cc68b99.dc1 127.0.0.1 -11:55.640 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-19f7a085-9d65-06f3-ce9e-68683cc68b99 127.0.0.1 -2021-10-31T23:11:55.640Z [INFO] agent.router: Initializing LAN area manager -11:55.640 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16463 network=udp -11:55.640 [INFO] TestAgent.server: Adding LAN server: server="Node-19f7a085-9d65-06f3-ce9e-68683cc68b99 (Addr: tcp/127.0.0.1:16468) (DC: dc1)" -11:55.640 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-19f7a085-9d65-06f3-ce9e-68683cc68b99.dc1 area=wan -11:55.640 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16463 network=tcp -11:55.641 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16464 network=tcp -11:55.641 [INFO] TestAgent: started state syncer -11:55.641 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16469 network=tcp -11:55.709 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:55.709 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16468 [Candidate]" term=2 -11:55.710 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:55.710 [DEBUG] TestAgent.server.raft: vote granted: from=19f7a085-9d65-06f3-ce9e-68683cc68b99 term=2 tally=1 -11:55.710 [INFO] TestAgent.server.raft: election won: tally=1 -11:55.710 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16468 [Leader]" -11:55.710 [INFO] TestAgent.server: cluster leadership acquired -11:55.710 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16468 -11:55.711 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:55.712 [INFO] TestAgent.server: New leader elected: payload=Node-19f7a085-9d65-06f3-ce9e-68683cc68b99 -11:55.712 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:55.712 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:55.712 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:55.712 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:55.712 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:55.712 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-19f7a085-9d65-06f3-ce9e-68683cc68b99 -11:55.712 [INFO] TestAgent.server: member joined, marking health alive: member=Node-19f7a085-9d65-06f3-ce9e-68683cc68b99 -11:55.713 [INFO] TestAgent.server: federation state anti-entropy synced -11:55.908 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:55.908 [INFO] TestAgent: Synced node info -11:55.908 [DEBUG] TestAgent: Node info in sync -11:56.002 [DEBUG] TestAgent: Node info in sync -11:56.012 [INFO] TestAgent: Synced service: service=web -11:56.012 [DEBUG] TestAgent: Check in sync: check=service:web -11:56.012 [INFO] TestAgent: Requesting shutdown -11:56.012 [INFO] TestAgent.server: shutting down server -11:56.012 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:56.012 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:56.012 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:56.012 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:56.012 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:56.012 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:56.012 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:56.012 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:56.012 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:56.012 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:56.012 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:56.013Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:56.013Z [INFO] agent.router.manager: shutting down -11:56.013 [INFO] TestAgent: consul server down -11:56.013 [INFO] TestAgent: shutdown complete -11:56.013 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16463 network=tcp -11:56.013 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16463 network=udp -11:56.013 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16464 network=tcp -11:56.513 [INFO] TestAgent: Waiting for endpoints to shut down -11:56.513 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:12.652Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:12.673Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:12.675 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b9e067ba-77cf-114d-7bea-c3c7a88fb776 Address:127.0.0.1:23615}]" +48:12.675 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23615 [Follower]" leader= +48:12.675 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b9e067ba-77cf-114d-7bea-c3c7a88fb776.dc1 127.0.0.1 +48:12.675 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b9e067ba-77cf-114d-7bea-c3c7a88fb776 127.0.0.1 +2022-12-04T05:48:12.675Z [INFO] agent.router: Initializing LAN area manager +48:12.676 [INFO] TestAgent.server: Adding LAN server: server="Node-b9e067ba-77cf-114d-7bea-c3c7a88fb776 (Addr: tcp/127.0.0.1:23615) (DC: dc1)" +48:12.676 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b9e067ba-77cf-114d-7bea-c3c7a88fb776.dc1 area=wan +48:12.676 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23610 network=udp +48:12.676 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23610 network=tcp +48:12.676 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23611 network=tcp +48:12.676 [INFO] TestAgent: started state syncer +48:12.676 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23616 network=tcp +48:12.738 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:12.738 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23615 [Candidate]" term=2 +48:12.738 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:12.738 [DEBUG] TestAgent.server.raft: vote granted: from=b9e067ba-77cf-114d-7bea-c3c7a88fb776 term=2 tally=1 +48:12.738 [INFO] TestAgent.server.raft: election won: tally=1 +48:12.738 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23615 [Leader]" +48:12.738 [INFO] TestAgent.server: cluster leadership acquired +48:12.739 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23615 +48:12.739 [INFO] TestAgent.server: New leader elected: payload=Node-b9e067ba-77cf-114d-7bea-c3c7a88fb776 +48:12.739 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:12.740 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:12.740 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:12.740 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:12.740 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:12.740 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:12.740 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b9e067ba-77cf-114d-7bea-c3c7a88fb776 +48:12.740 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b9e067ba-77cf-114d-7bea-c3c7a88fb776 +48:12.741 [INFO] TestAgent.server: federation state anti-entropy synced +48:12.797 [INFO] TestAgent: Synced node info +48:12.797 [INFO] TestAgent: Synced service: service=web +48:12.797 [DEBUG] TestAgent: Check in sync: check=service:web +48:12.797 [INFO] TestAgent: Requesting shutdown +48:12.798 [INFO] TestAgent.server: shutting down server +48:12.798 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:12.798 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:12.798 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:12.798 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:12.798 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:12.798 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:12.798 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:12.798 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:12.798 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:12.798 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:12.798 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:12.798 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:12.798Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:12.798Z [INFO] agent.router.manager: shutting down +48:12.798 [INFO] TestAgent: consul server down +48:12.798 [INFO] TestAgent: shutdown complete +48:12.798 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23610 network=tcp +48:12.798 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23610 network=udp +48:12.798 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23611 network=tcp +48:13.300 [INFO] TestAgent: Waiting for endpoints to shut down +48:13.300 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:12.650Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:12.674Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:12.675 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0c4c3154-0a86-059b-86e2-35afe09d6e12 Address:127.0.0.1:23608}]" +48:12.675 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23608 [Follower]" leader= +48:12.676 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0c4c3154-0a86-059b-86e2-35afe09d6e12.dc1 127.0.0.1 +48:12.676 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0c4c3154-0a86-059b-86e2-35afe09d6e12 127.0.0.1 +2022-12-04T05:48:12.676Z [INFO] agent.router: Initializing LAN area manager +48:12.677 [INFO] TestAgent.server: Adding LAN server: server="Node-0c4c3154-0a86-059b-86e2-35afe09d6e12 (Addr: tcp/127.0.0.1:23608) (DC: dc1)" +48:12.677 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0c4c3154-0a86-059b-86e2-35afe09d6e12.dc1 area=wan +48:12.677 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23603 network=udp +48:12.677 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23603 network=tcp +48:12.677 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23604 network=tcp +48:12.677 [INFO] TestAgent: started state syncer +48:12.677 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23609 network=tcp +48:12.734 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:12.734 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23608 [Candidate]" term=2 +48:12.735 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:12.735 [DEBUG] TestAgent.server.raft: vote granted: from=0c4c3154-0a86-059b-86e2-35afe09d6e12 term=2 tally=1 +48:12.735 [INFO] TestAgent.server.raft: election won: tally=1 +48:12.735 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23608 [Leader]" +48:12.735 [INFO] TestAgent.server: cluster leadership acquired +48:12.735 [INFO] TestAgent.server: New leader elected: payload=Node-0c4c3154-0a86-059b-86e2-35afe09d6e12 +48:12.735 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23608 +48:12.735 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:12.736 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:12.736 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:12.736 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:12.736 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:12.736 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:12.737 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0c4c3154-0a86-059b-86e2-35afe09d6e12 +48:12.737 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0c4c3154-0a86-059b-86e2-35afe09d6e12 +48:12.737 [INFO] TestAgent.server: federation state anti-entropy synced +48:13.074 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:13.075 [INFO] TestAgent: Synced node info +48:13.083 [DEBUG] TestAgent: Node info in sync +48:13.084 [INFO] TestAgent: Synced service: service=web +48:13.084 [DEBUG] TestAgent: Check in sync: check=service:web +48:13.084 [INFO] TestAgent: Requesting shutdown +48:13.084 [INFO] TestAgent.server: shutting down server +48:13.084 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:13.084 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:13.084 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:13.084 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:13.084 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:13.084 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:13.084 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:13.084 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:13.084 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:13.084 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:13.084 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:13.084Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:13.084Z [INFO] agent.router.manager: shutting down +48:13.085 [INFO] TestAgent: consul server down +48:13.085 [INFO] TestAgent: shutdown complete +48:13.085 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23603 network=tcp +48:13.085 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23603 network=udp +48:13.085 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23604 network=tcp +48:13.588 [INFO] TestAgent: Waiting for endpoints to shut down +48:13.588 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_ConnectNative (0.00s) - --- PASS: TestAgent_RegisterService_ConnectNative/normal (0.69s) - --- PASS: TestAgent_RegisterService_ConnectNative/service_manager (0.95s) + --- PASS: TestAgent_RegisterService_ConnectNative/service_manager (0.67s) + --- PASS: TestAgent_RegisterService_ConnectNative/normal (0.96s) === RUN TestAgent_RegisterService_ScriptCheck_ExecDisable === RUN TestAgent_RegisterService_ScriptCheck_ExecDisable/normal === PAUSE TestAgent_RegisterService_ScriptCheck_ExecDisable/normal @@ -9331,120 +9363,119 @@ === PAUSE TestAgent_RegisterService_ScriptCheck_ExecDisable/service_manager === CONT TestAgent_RegisterService_ScriptCheck_ExecDisable/normal === CONT TestAgent_RegisterService_ScriptCheck_ExecDisable/service_manager -2021-10-31T23:11:56.531Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:56.561Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:56.567 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:07c5919f-f1e6-9cc5-860a-434e2521e6a1 Address:127.0.0.1:16475}]" -11:56.568 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16475 [Follower]" leader= -11:56.568 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-07c5919f-f1e6-9cc5-860a-434e2521e6a1.dc1 127.0.0.1 -11:56.569 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-07c5919f-f1e6-9cc5-860a-434e2521e6a1 127.0.0.1 -2021-10-31T23:11:56.569Z [INFO] agent.router: Initializing LAN area manager -11:56.569 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16470 network=udp -11:56.569 [INFO] TestAgent.server: Adding LAN server: server="Node-07c5919f-f1e6-9cc5-860a-434e2521e6a1 (Addr: tcp/127.0.0.1:16475) (DC: dc1)" -11:56.569 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-07c5919f-f1e6-9cc5-860a-434e2521e6a1.dc1 area=wan -11:56.569 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16470 network=tcp -11:56.570 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16471 network=tcp -11:56.570 [INFO] TestAgent: started state syncer -11:56.570 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16476 network=tcp -11:56.609 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:56.609 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16475 [Candidate]" term=2 -11:56.610 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:56.610 [DEBUG] TestAgent.server.raft: vote granted: from=07c5919f-f1e6-9cc5-860a-434e2521e6a1 term=2 tally=1 -11:56.610 [INFO] TestAgent.server.raft: election won: tally=1 -11:56.610 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16475 [Leader]" -11:56.610 [INFO] TestAgent.server: cluster leadership acquired -11:56.610 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16475 -11:56.611 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:56.612 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:56.612 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:56.612 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:56.612 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:56.612 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:56.612 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-07c5919f-f1e6-9cc5-860a-434e2521e6a1 -11:56.612 [INFO] TestAgent.server: member joined, marking health alive: member=Node-07c5919f-f1e6-9cc5-860a-434e2521e6a1 -11:56.613 [INFO] TestAgent.server: New leader elected: payload=Node-07c5919f-f1e6-9cc5-860a-434e2521e6a1 -11:56.613 [INFO] TestAgent.server: federation state anti-entropy synced -11:56.706 [INFO] TestAgent: Requesting shutdown -11:56.706 [INFO] TestAgent.server: shutting down server -11:56.706 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:56.706 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:56.706 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:56.706 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:56.706 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:56.706 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:56.706 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:56.706 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:56.706 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:56.706 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:56.707 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:56.707 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:56.707Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:56.707Z [INFO] agent.router.manager: shutting down -11:56.707 [INFO] TestAgent: consul server down -11:56.707 [INFO] TestAgent: shutdown complete -11:56.707 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16470 network=tcp -11:56.707 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16470 network=udp -11:56.707 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16471 network=tcp -11:57.209 [INFO] TestAgent: Waiting for endpoints to shut down -11:57.210 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:56.532Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:56.551Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:56.552 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3ba7f1bf-5a98-c27c-de7b-4bdb29161f3e Address:127.0.0.1:16482}]" -11:56.553 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3ba7f1bf-5a98-c27c-de7b-4bdb29161f3e.dc1 127.0.0.1 -11:56.553 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3ba7f1bf-5a98-c27c-de7b-4bdb29161f3e 127.0.0.1 -2021-10-31T23:11:56.553Z [INFO] agent.router: Initializing LAN area manager -11:56.553 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16477 network=udp -11:56.553 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16482 [Follower]" leader= -11:56.554 [INFO] TestAgent.server: Adding LAN server: server="Node-3ba7f1bf-5a98-c27c-de7b-4bdb29161f3e (Addr: tcp/127.0.0.1:16482) (DC: dc1)" -11:56.554 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3ba7f1bf-5a98-c27c-de7b-4bdb29161f3e.dc1 area=wan -11:56.554 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16477 network=tcp -11:56.554 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16478 network=tcp -11:56.555 [INFO] TestAgent: started state syncer -11:56.555 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16483 network=tcp -11:56.621 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:56.621 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16482 [Candidate]" term=2 -11:56.622 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:56.622 [DEBUG] TestAgent.server.raft: vote granted: from=3ba7f1bf-5a98-c27c-de7b-4bdb29161f3e term=2 tally=1 -11:56.622 [INFO] TestAgent.server.raft: election won: tally=1 -11:56.622 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16482 [Leader]" -11:56.622 [INFO] TestAgent.server: cluster leadership acquired -11:56.622 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16482 -11:56.623 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:56.624 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:56.624 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:56.624 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:56.624 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:56.624 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:56.624 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3ba7f1bf-5a98-c27c-de7b-4bdb29161f3e -11:56.624 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3ba7f1bf-5a98-c27c-de7b-4bdb29161f3e -11:56.625 [INFO] TestAgent.server: New leader elected: payload=Node-3ba7f1bf-5a98-c27c-de7b-4bdb29161f3e -11:56.625 [INFO] TestAgent.server: federation state anti-entropy synced -11:56.702 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:56.702 [INFO] TestAgent: Synced node info -11:56.945 [INFO] TestAgent: Requesting shutdown -11:56.945 [INFO] TestAgent.server: shutting down server -11:56.945 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:56.945 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:56.945 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:56.945 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:56.945 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:56.945 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:56.945 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:56.945 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:56.945 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:56.945 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:56.945 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:56.945Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:56.945Z [INFO] agent.router.manager: shutting down -11:56.945 [INFO] TestAgent: consul server down -11:56.945 [INFO] TestAgent: shutdown complete -11:56.945 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16477 network=tcp -11:56.945 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16477 network=udp -11:56.945 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16478 network=tcp -11:57.450 [INFO] TestAgent: Waiting for endpoints to shut down -11:57.450 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:13.646Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:13.683Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:13.685 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2c2cbcec-c427-a4dc-06da-ce0c92ecc1e8 Address:127.0.0.1:23629}]" +48:13.685 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2c2cbcec-c427-a4dc-06da-ce0c92ecc1e8.dc1 127.0.0.1 +48:13.686 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2c2cbcec-c427-a4dc-06da-ce0c92ecc1e8 127.0.0.1 +2022-12-04T05:48:13.686Z [INFO] agent.router: Initializing LAN area manager +48:13.686 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23624 network=udp +48:13.686 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23629 [Follower]" leader= +48:13.687 [INFO] TestAgent.server: Adding LAN server: server="Node-2c2cbcec-c427-a4dc-06da-ce0c92ecc1e8 (Addr: tcp/127.0.0.1:23629) (DC: dc1)" +48:13.687 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2c2cbcec-c427-a4dc-06da-ce0c92ecc1e8.dc1 area=wan +48:13.687 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23624 network=tcp +48:13.687 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23625 network=tcp +48:13.687 [INFO] TestAgent: started state syncer +48:13.687 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23630 network=tcp +48:13.750 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:13.750 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23629 [Candidate]" term=2 +48:13.750 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:13.750 [DEBUG] TestAgent.server.raft: vote granted: from=2c2cbcec-c427-a4dc-06da-ce0c92ecc1e8 term=2 tally=1 +48:13.750 [INFO] TestAgent.server.raft: election won: tally=1 +48:13.750 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23629 [Leader]" +48:13.751 [INFO] TestAgent.server: cluster leadership acquired +48:13.751 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23629 +48:13.751 [INFO] TestAgent.server: New leader elected: payload=Node-2c2cbcec-c427-a4dc-06da-ce0c92ecc1e8 +48:13.752 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:13.753 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:13.753 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:13.753 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:13.753 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:13.753 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:13.753 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2c2cbcec-c427-a4dc-06da-ce0c92ecc1e8 +48:13.753 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2c2cbcec-c427-a4dc-06da-ce0c92ecc1e8 +48:13.754 [INFO] TestAgent.server: federation state anti-entropy synced +48:13.947 [INFO] TestAgent: Requesting shutdown +48:13.948 [INFO] TestAgent.server: shutting down server +48:13.948 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:13.948 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:13.948 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:13.948 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:13.948 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:13.948 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:13.948 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:13.948 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:13.948 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:13.948 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:13.948 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:13.948 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:13.948Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:13.948Z [INFO] agent.router.manager: shutting down +48:13.948 [INFO] TestAgent: consul server down +48:13.948 [INFO] TestAgent: shutdown complete +48:13.948 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23624 network=tcp +48:13.948 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23624 network=udp +48:13.948 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23625 network=tcp +48:14.448 [INFO] TestAgent: Waiting for endpoints to shut down +48:14.448 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:13.674Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:13.707Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:13.708 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:20f0d248-2300-e5bb-fbc9-3e21ded055e4 Address:127.0.0.1:23622}]" +48:13.709 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23622 [Follower]" leader= +48:13.709 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-20f0d248-2300-e5bb-fbc9-3e21ded055e4.dc1 127.0.0.1 +48:13.709 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-20f0d248-2300-e5bb-fbc9-3e21ded055e4 127.0.0.1 +2022-12-04T05:48:13.709Z [INFO] agent.router: Initializing LAN area manager +48:13.710 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23617 network=udp +48:13.710 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-20f0d248-2300-e5bb-fbc9-3e21ded055e4.dc1 area=wan +48:13.710 [INFO] TestAgent.server: Adding LAN server: server="Node-20f0d248-2300-e5bb-fbc9-3e21ded055e4 (Addr: tcp/127.0.0.1:23622) (DC: dc1)" +48:13.710 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23617 network=tcp +48:13.710 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23618 network=tcp +48:13.710 [INFO] TestAgent: started state syncer +48:13.710 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23623 network=tcp +48:13.778 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:13.778 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23622 [Candidate]" term=2 +48:13.778 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:13.778 [DEBUG] TestAgent.server.raft: vote granted: from=20f0d248-2300-e5bb-fbc9-3e21ded055e4 term=2 tally=1 +48:13.778 [INFO] TestAgent.server.raft: election won: tally=1 +48:13.778 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23622 [Leader]" +48:13.778 [INFO] TestAgent.server: cluster leadership acquired +48:13.778 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23622 +48:13.779 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:13.780 [INFO] TestAgent.server: New leader elected: payload=Node-20f0d248-2300-e5bb-fbc9-3e21ded055e4 +48:13.781 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:13.781 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:13.781 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:13.781 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:13.781 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:13.781 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-20f0d248-2300-e5bb-fbc9-3e21ded055e4 +48:13.781 [INFO] TestAgent.server: member joined, marking health alive: member=Node-20f0d248-2300-e5bb-fbc9-3e21ded055e4 +48:13.782 [INFO] TestAgent.server: federation state anti-entropy synced +48:14.025 [INFO] TestAgent: Requesting shutdown +48:14.025 [INFO] TestAgent.server: shutting down server +48:14.025 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:14.025 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:14.025 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:14.025 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:14.025 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:14.025 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:14.025 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:14.025 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:14.025 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:14.025 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:14.025 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:14.025 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:14.025Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:14.025Z [INFO] agent.router.manager: shutting down +48:14.026 [INFO] TestAgent: consul server down +48:14.026 [INFO] TestAgent: shutdown complete +48:14.026 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23617 network=tcp +48:14.026 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23617 network=udp +48:14.026 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23618 network=tcp +48:14.526 [INFO] TestAgent: Waiting for endpoints to shut down +48:14.526 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_ScriptCheck_ExecDisable (0.00s) - --- PASS: TestAgent_RegisterService_ScriptCheck_ExecDisable/normal (0.70s) - --- PASS: TestAgent_RegisterService_ScriptCheck_ExecDisable/service_manager (0.93s) + --- PASS: TestAgent_RegisterService_ScriptCheck_ExecDisable/service_manager (0.86s) + --- PASS: TestAgent_RegisterService_ScriptCheck_ExecDisable/normal (0.94s) === RUN TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable === RUN TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/normal === PAUSE TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/normal @@ -9452,121 +9483,123 @@ === PAUSE TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/service_manager === CONT TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/normal === CONT TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/service_manager -2021-10-31T23:11:57.525Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:57.560Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:57.561 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fdee52e2-5e69-64f7-ba6c-e9a025abada9 Address:127.0.0.1:16496}]" -11:57.561 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16496 [Follower]" leader= -11:57.566 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fdee52e2-5e69-64f7-ba6c-e9a025abada9.dc1 127.0.0.1 -11:57.567 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fdee52e2-5e69-64f7-ba6c-e9a025abada9 127.0.0.1 -2021-10-31T23:11:57.567Z [INFO] agent.router: Initializing LAN area manager -11:57.567 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16491 network=udp -11:57.567 [INFO] TestAgent.server: Adding LAN server: server="Node-fdee52e2-5e69-64f7-ba6c-e9a025abada9 (Addr: tcp/127.0.0.1:16496) (DC: dc1)" -11:57.567 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fdee52e2-5e69-64f7-ba6c-e9a025abada9.dc1 area=wan -11:57.567 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16491 network=tcp -11:57.568 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16492 network=tcp -11:57.568 [INFO] TestAgent: started state syncer -11:57.568 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16497 network=tcp -11:57.602 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:57.602 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16496 [Candidate]" term=2 -11:57.602 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:57.602 [DEBUG] TestAgent.server.raft: vote granted: from=fdee52e2-5e69-64f7-ba6c-e9a025abada9 term=2 tally=1 -11:57.602 [INFO] TestAgent.server.raft: election won: tally=1 -11:57.602 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16496 [Leader]" -11:57.602 [INFO] TestAgent.server: cluster leadership acquired -11:57.602 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16496 -11:57.603 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:57.604 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:57.604 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:57.604 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:57.604 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:57.604 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:57.604 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fdee52e2-5e69-64f7-ba6c-e9a025abada9 -11:57.604 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fdee52e2-5e69-64f7-ba6c-e9a025abada9 -11:57.605 [INFO] TestAgent.server: New leader elected: payload=Node-fdee52e2-5e69-64f7-ba6c-e9a025abada9 -11:57.605 [INFO] TestAgent.server: federation state anti-entropy synced -11:57.665 [INFO] TestAgent: Requesting shutdown -11:57.665 [INFO] TestAgent.server: shutting down server -11:57.665 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:57.665 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:57.665 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:57.665 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:57.665 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:57.665 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:57.665 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:57.665 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:57.665 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:57.665 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:57.665 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:57.665 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:57.666Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:57.666Z [INFO] agent.router.manager: shutting down -11:57.666 [INFO] TestAgent: consul server down -11:57.666 [INFO] TestAgent: shutdown complete -11:57.666 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16491 network=tcp -11:57.666 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16491 network=udp -11:57.666 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16492 network=tcp -11:58.168 [INFO] TestAgent: Waiting for endpoints to shut down -11:58.168 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:57.526Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:57.553Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:57.567 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0819e3c5-585b-a5f3-2c4f-547e987c5001 Address:127.0.0.1:16489}]" -11:57.568 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0819e3c5-585b-a5f3-2c4f-547e987c5001.dc1 127.0.0.1 -11:57.568 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0819e3c5-585b-a5f3-2c4f-547e987c5001 127.0.0.1 -2021-10-31T23:11:57.568Z [INFO] agent.router: Initializing LAN area manager -11:57.568 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16484 network=udp -11:57.568 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16489 [Follower]" leader= -11:57.569 [INFO] TestAgent.server: Adding LAN server: server="Node-0819e3c5-585b-a5f3-2c4f-547e987c5001 (Addr: tcp/127.0.0.1:16489) (DC: dc1)" -11:57.569 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0819e3c5-585b-a5f3-2c4f-547e987c5001.dc1 area=wan -11:57.569 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16484 network=tcp -11:57.572 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16485 network=tcp -11:57.572 [INFO] TestAgent: started state syncer -11:57.572 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16490 network=tcp -11:57.618 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:57.618 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16489 [Candidate]" term=2 -11:57.618 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:57.618 [DEBUG] TestAgent.server.raft: vote granted: from=0819e3c5-585b-a5f3-2c4f-547e987c5001 term=2 tally=1 -11:57.618 [INFO] TestAgent.server.raft: election won: tally=1 -11:57.618 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16489 [Leader]" -11:57.618 [INFO] TestAgent.server: cluster leadership acquired -11:57.618 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16489 -11:57.619 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:57.620 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:57.620 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:57.620 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:57.620 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:57.620 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:57.620 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0819e3c5-585b-a5f3-2c4f-547e987c5001 -11:57.620 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0819e3c5-585b-a5f3-2c4f-547e987c5001 -11:57.621 [INFO] TestAgent.server: New leader elected: payload=Node-0819e3c5-585b-a5f3-2c4f-547e987c5001 -11:57.621 [INFO] TestAgent.server: federation state anti-entropy synced -11:57.646 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:57.646 [INFO] TestAgent: Synced node info -11:57.646 [DEBUG] TestAgent: Node info in sync -11:57.718 [INFO] TestAgent: Requesting shutdown -11:57.718 [INFO] TestAgent.server: shutting down server -11:57.718 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:57.718 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:57.718 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:57.718 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:57.718 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:57.718 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:57.718 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:57.718 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:57.718 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:57.718 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:57.718 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:57.718Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:57.718Z [INFO] agent.router.manager: shutting down -11:57.719 [INFO] TestAgent: consul server down -11:57.719 [INFO] TestAgent: shutdown complete -11:57.719 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16484 network=tcp -11:57.719 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16484 network=udp -11:57.719 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16485 network=tcp -11:58.221 [INFO] TestAgent: Waiting for endpoints to shut down -11:58.221 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:14.549Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:14.570Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:14.572 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9b8cf92a-e5fb-4f44-0a2b-899e5ee4103c Address:127.0.0.1:23643}]" +48:14.572 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23643 [Follower]" leader= +48:14.572 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9b8cf92a-e5fb-4f44-0a2b-899e5ee4103c.dc1 127.0.0.1 +48:14.572 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9b8cf92a-e5fb-4f44-0a2b-899e5ee4103c 127.0.0.1 +2022-12-04T05:48:14.573Z [INFO] agent.router: Initializing LAN area manager +48:14.573 [INFO] TestAgent.server: Adding LAN server: server="Node-9b8cf92a-e5fb-4f44-0a2b-899e5ee4103c (Addr: tcp/127.0.0.1:23643) (DC: dc1)" +48:14.573 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9b8cf92a-e5fb-4f44-0a2b-899e5ee4103c.dc1 area=wan +48:14.573 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23638 network=udp +48:14.573 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23638 network=tcp +48:14.573 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23639 network=tcp +48:14.573 [INFO] TestAgent: started state syncer +48:14.573 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23644 network=tcp +48:14.637 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:14.638 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23643 [Candidate]" term=2 +48:14.638 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:14.638 [DEBUG] TestAgent.server.raft: vote granted: from=9b8cf92a-e5fb-4f44-0a2b-899e5ee4103c term=2 tally=1 +48:14.638 [INFO] TestAgent.server.raft: election won: tally=1 +48:14.638 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23643 [Leader]" +48:14.638 [INFO] TestAgent.server: cluster leadership acquired +48:14.638 [INFO] TestAgent.server: New leader elected: payload=Node-9b8cf92a-e5fb-4f44-0a2b-899e5ee4103c +48:14.638 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23643 +48:14.639 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:14.640 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:14.640 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:14.640 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:14.640 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:14.640 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:14.640 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9b8cf92a-e5fb-4f44-0a2b-899e5ee4103c +48:14.640 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9b8cf92a-e5fb-4f44-0a2b-899e5ee4103c +48:14.640 [INFO] TestAgent.server: federation state anti-entropy synced +48:14.659 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:14.659 [INFO] TestAgent: Synced node info +48:14.659 [DEBUG] TestAgent: Node info in sync +48:14.794 [INFO] TestAgent: Requesting shutdown +48:14.794 [INFO] TestAgent.server: shutting down server +48:14.794 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:14.794 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:14.794 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:14.794 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:14.794 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:14.794 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:14.794 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:14.794 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:14.794 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:48:14.794Z [INFO] agent.router.manager: shutting down +48:14.794 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:48:14.794Z [INFO] agent.router.manager: shutting down +48:14.794 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:14.795 [INFO] TestAgent: consul server down +48:14.795 [INFO] TestAgent: shutdown complete +48:14.795 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23638 network=tcp +48:14.795 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23638 network=udp +48:14.795 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23639 network=tcp +48:15.295 [INFO] TestAgent: Waiting for endpoints to shut down +48:15.295 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:14.539Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:14.552Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:14.570 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6517c76c-ab3e-d485-c150-69232641787a Address:127.0.0.1:23636}]" +48:14.570 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23636 [Follower]" leader= +48:14.571 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6517c76c-ab3e-d485-c150-69232641787a.dc1 127.0.0.1 +48:14.571 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6517c76c-ab3e-d485-c150-69232641787a 127.0.0.1 +2022-12-04T05:48:14.572Z [INFO] agent.router: Initializing LAN area manager +48:14.572 [INFO] TestAgent.server: Adding LAN server: server="Node-6517c76c-ab3e-d485-c150-69232641787a (Addr: tcp/127.0.0.1:23636) (DC: dc1)" +48:14.572 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6517c76c-ab3e-d485-c150-69232641787a.dc1 area=wan +48:14.572 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23631 network=udp +48:14.572 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23631 network=tcp +48:14.572 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23632 network=tcp +48:14.572 [INFO] TestAgent: started state syncer +48:14.572 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23637 network=tcp +48:14.606 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:14.606 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23636 [Candidate]" term=2 +48:14.606 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:14.606 [DEBUG] TestAgent.server.raft: vote granted: from=6517c76c-ab3e-d485-c150-69232641787a term=2 tally=1 +48:14.606 [INFO] TestAgent.server.raft: election won: tally=1 +48:14.606 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23636 [Leader]" +48:14.606 [INFO] TestAgent.server: cluster leadership acquired +48:14.606 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23636 +48:14.607 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:14.606 [INFO] TestAgent.server: New leader elected: payload=Node-6517c76c-ab3e-d485-c150-69232641787a +48:14.608 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:14.608 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:14.608 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:14.608 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:14.608 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:14.608 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6517c76c-ab3e-d485-c150-69232641787a +48:14.608 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6517c76c-ab3e-d485-c150-69232641787a +48:14.609 [INFO] TestAgent.server: federation state anti-entropy synced +48:14.803 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:14.803 [INFO] TestAgent: Synced node info +48:14.803 [DEBUG] TestAgent: Node info in sync +48:14.862 [INFO] TestAgent: Requesting shutdown +48:14.862 [INFO] TestAgent.server: shutting down server +48:14.862 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:14.862 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:14.862 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:14.862 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:14.862 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:14.862 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:14.862 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:14.862 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:14.862 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:14.862 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:14.862 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:14.863Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:14.863Z [INFO] agent.router.manager: shutting down +48:14.863 [INFO] TestAgent: consul server down +48:14.863 [INFO] TestAgent: shutdown complete +48:14.863 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23631 network=tcp +48:14.863 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23631 network=udp +48:14.863 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23632 network=tcp +48:15.363 [INFO] TestAgent: Waiting for endpoints to shut down +48:15.363 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable (0.00s) - --- PASS: TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/service_manager (0.71s) - --- PASS: TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/normal (0.77s) + --- PASS: TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/service_manager (0.77s) + --- PASS: TestAgent_RegisterService_ScriptCheck_ExecRemoteDisable/normal (0.84s) === RUN TestAgent_DeregisterService === PAUSE TestAgent_DeregisterService === RUN TestAgent_DeregisterService_ACLDeny @@ -9666,716 +9699,722 @@ === PAUSE TestAgent_MultiStartStop/#09 === CONT TestAgent_MultiStartStop/#00 === CONT TestAgent_MultiStartStop/#05 -=== CONT TestAgent_MultiStartStop/#03 -=== CONT TestAgent_MultiStartStop/#04 === CONT TestAgent_MultiStartStop/#08 === CONT TestAgent_MultiStartStop/#07 +=== CONT TestAgent_MultiStartStop/#03 +=== CONT TestAgent_MultiStartStop/#04 === CONT TestAgent_MultiStartStop/#06 === CONT TestAgent_MultiStartStop/#09 === CONT TestAgent_MultiStartStop/#02 === CONT TestAgent_MultiStartStop/#01 -2021-10-31T23:11:58.308Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:58.387Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:58.389 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6e351382-e39b-c26a-caa2-157817ef6faf Address:127.0.0.1:16524}]" -11:58.389 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16524 [Follower]" leader= -11:58.410 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6e351382-e39b-c26a-caa2-157817ef6faf.dc1 127.0.0.1 -11:58.411 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6e351382-e39b-c26a-caa2-157817ef6faf 127.0.0.1 -2021-10-31T23:11:58.411Z [INFO] agent.router: Initializing LAN area manager -11:58.411 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16519 network=udp -11:58.411 [INFO] TestAgent.server: Adding LAN server: server="Node-6e351382-e39b-c26a-caa2-157817ef6faf (Addr: tcp/127.0.0.1:16524) (DC: dc1)" -11:58.411 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6e351382-e39b-c26a-caa2-157817ef6faf.dc1 area=wan -11:58.413 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16519 network=tcp -11:58.426 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:58.426 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16524 [Candidate]" term=2 -11:58.426 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:58.426 [DEBUG] TestAgent.server.raft: vote granted: from=6e351382-e39b-c26a-caa2-157817ef6faf term=2 tally=1 -11:58.426 [INFO] TestAgent.server.raft: election won: tally=1 -11:58.426 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16524 [Leader]" -11:58.426 [INFO] TestAgent.server: cluster leadership acquired -11:58.427 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16524 -11:58.427 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:58.429 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:58.429 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:58.429 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:58.429 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:58.429 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:58.429 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6e351382-e39b-c26a-caa2-157817ef6faf -11:58.429 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6e351382-e39b-c26a-caa2-157817ef6faf -11:58.429 [INFO] TestAgent.server: New leader elected: payload=Node-6e351382-e39b-c26a-caa2-157817ef6faf -11:58.430 [INFO] TestAgent.server: federation state anti-entropy synced -11:58.430 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16520 network=tcp -11:58.430 [INFO] TestAgent: started state syncer -11:58.430 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16525 network=tcp -11:58.430 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:58.430 [INFO] TestAgent: Synced node info -11:58.680 [INFO] TestAgent: Requesting shutdown -11:58.680 [INFO] TestAgent.server: shutting down server -11:58.680 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:58.680 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:58.680 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:58.680 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:58.680 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:58.680 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:58.681 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:58.681 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:58.681 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:58.681 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:58.681 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:58.681Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:58.681Z [INFO] agent.router.manager: shutting down -11:58.681 [INFO] TestAgent: consul server down -11:58.681 [INFO] TestAgent: shutdown complete -11:58.681 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16519 network=tcp -11:58.681 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16519 network=udp -11:58.681 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16520 network=tcp -11:59.181 [INFO] TestAgent: Waiting for endpoints to shut down -11:59.181 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:58.316Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:58.385Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:58.391 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f65b03e4-86f5-05d0-9715-9a07ef765fdb Address:127.0.0.1:16545}]" -11:58.392 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f65b03e4-86f5-05d0-9715-9a07ef765fdb.dc1 127.0.0.1 -11:58.393 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16545 [Follower]" leader= -11:58.412 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f65b03e4-86f5-05d0-9715-9a07ef765fdb 127.0.0.1 -2021-10-31T23:11:58.412Z [INFO] agent.router: Initializing LAN area manager -11:58.412 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16540 network=udp -11:58.414 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f65b03e4-86f5-05d0-9715-9a07ef765fdb.dc1 area=wan -11:58.414 [INFO] TestAgent.server: Adding LAN server: server="Node-f65b03e4-86f5-05d0-9715-9a07ef765fdb (Addr: tcp/127.0.0.1:16545) (DC: dc1)" -11:58.415 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16540 network=tcp -11:58.430 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:58.430 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16545 [Candidate]" term=2 -11:58.430 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:58.430 [DEBUG] TestAgent.server.raft: vote granted: from=f65b03e4-86f5-05d0-9715-9a07ef765fdb term=2 tally=1 -11:58.430 [INFO] TestAgent.server.raft: election won: tally=1 -11:58.430 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16545 [Leader]" -11:58.430 [INFO] TestAgent.server: cluster leadership acquired -11:58.430 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16541 network=tcp -11:58.431 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16545 -11:58.431 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16546 network=tcp -11:58.431 [INFO] TestAgent.server: New leader elected: payload=Node-f65b03e4-86f5-05d0-9715-9a07ef765fdb -11:58.431 [INFO] TestAgent: started state syncer -11:58.431 [INFO] TestAgent: Synced node info -11:58.431 [DEBUG] TestAgent: Node info in sync -11:58.432 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:58.433 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:58.433 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:58.433 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:58.433 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:58.433 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:58.433 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f65b03e4-86f5-05d0-9715-9a07ef765fdb -11:58.434 [INFO] TestAgent.server: federation state anti-entropy synced -11:58.436 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f65b03e4-86f5-05d0-9715-9a07ef765fdb -11:58.681 [INFO] TestAgent: Requesting shutdown -11:58.681 [INFO] TestAgent.server: shutting down server -11:58.681 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:58.681 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:58.681 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:58.681 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:58.681 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:58.681 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:58.681 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:58.681 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:58.681 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:58.681 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:58.681 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:58.681Z [INFO] agent.router.manager: shutting down -11:58.682 [INFO] TestAgent: consul server down -11:58.682 [INFO] TestAgent: shutdown complete -11:58.682 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16540 network=tcp -2021-10-31T23:11:58.682Z [INFO] agent.router.manager: shutting down -11:58.682 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16540 network=udp -11:58.682 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16541 network=tcp -11:59.182 [INFO] TestAgent: Waiting for endpoints to shut down -11:59.182 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:58.384Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:58.412Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:58.483 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:105ae364-ab09-e9c4-e259-657fc40c2c9b Address:127.0.0.1:16566}]" -11:58.483 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-105ae364-ab09-e9c4-e259-657fc40c2c9b.dc1 127.0.0.1 -11:58.484 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-105ae364-ab09-e9c4-e259-657fc40c2c9b 127.0.0.1 -2021-10-31T23:11:58.484Z [INFO] agent.router: Initializing LAN area manager -11:58.484 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16561 network=udp -11:58.484 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16566 [Follower]" leader= -11:58.484 [INFO] TestAgent.server: Adding LAN server: server="Node-105ae364-ab09-e9c4-e259-657fc40c2c9b (Addr: tcp/127.0.0.1:16566) (DC: dc1)" -11:58.484 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-105ae364-ab09-e9c4-e259-657fc40c2c9b.dc1 area=wan -11:58.484 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16561 network=tcp -11:58.485 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16562 network=tcp -11:58.485 [INFO] TestAgent: started state syncer -11:58.485 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16567 network=tcp -11:58.538 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:58.538 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16566 [Candidate]" term=2 -11:58.538 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:58.538 [DEBUG] TestAgent.server.raft: vote granted: from=105ae364-ab09-e9c4-e259-657fc40c2c9b term=2 tally=1 -11:58.538 [INFO] TestAgent.server.raft: election won: tally=1 -11:58.538 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16566 [Leader]" -11:58.538 [INFO] TestAgent.server: cluster leadership acquired -11:58.538 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16566 -11:58.539 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:58.539 [INFO] TestAgent.server: New leader elected: payload=Node-105ae364-ab09-e9c4-e259-657fc40c2c9b -11:58.540 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:58.540 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:58.540 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:58.540 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:58.540 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:58.540 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-105ae364-ab09-e9c4-e259-657fc40c2c9b -11:58.540 [INFO] TestAgent.server: member joined, marking health alive: member=Node-105ae364-ab09-e9c4-e259-657fc40c2c9b -11:58.541 [INFO] TestAgent.server: federation state anti-entropy synced -11:58.824 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:58.825 [INFO] TestAgent: Synced node info -11:58.825 [DEBUG] TestAgent: Node info in sync -11:58.837 [INFO] TestAgent: Requesting shutdown -11:58.837 [INFO] TestAgent.server: shutting down server -11:58.837 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:58.837 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:58.837 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:58.837 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:58.837 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:58.837 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:58.837 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:58.837 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:58.838 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:58.838 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:58.838 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:58.838Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:58.838Z [INFO] agent.router.manager: shutting down -11:58.838 [INFO] TestAgent: consul server down -11:58.838 [INFO] TestAgent: shutdown complete -11:58.838 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16561 network=tcp -11:58.838 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16561 network=udp -11:58.838 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16562 network=tcp -11:59.340 [INFO] TestAgent: Waiting for endpoints to shut down -11:59.340 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:58.351Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:58.393Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:58.414 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e642cf40-f1d8-003e-c7d4-124ae6867bae Address:127.0.0.1:16503}]" -11:58.426 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16503 [Follower]" leader= -11:58.426 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e642cf40-f1d8-003e-c7d4-124ae6867bae.dc1 127.0.0.1 -11:58.427 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e642cf40-f1d8-003e-c7d4-124ae6867bae 127.0.0.1 -2021-10-31T23:11:58.427Z [INFO] agent.router: Initializing LAN area manager -11:58.427 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16498 network=udp -11:58.429 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e642cf40-f1d8-003e-c7d4-124ae6867bae.dc1 area=wan -11:58.429 [INFO] TestAgent.server: Adding LAN server: server="Node-e642cf40-f1d8-003e-c7d4-124ae6867bae (Addr: tcp/127.0.0.1:16503) (DC: dc1)" -11:58.430 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16498 network=tcp -11:58.430 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16499 network=tcp -11:58.430 [INFO] TestAgent: started state syncer -11:58.430 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16504 network=tcp -11:58.477 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:58.477 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16503 [Candidate]" term=2 -11:58.478 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:58.478 [DEBUG] TestAgent.server.raft: vote granted: from=e642cf40-f1d8-003e-c7d4-124ae6867bae term=2 tally=1 -11:58.478 [INFO] TestAgent.server.raft: election won: tally=1 -11:58.478 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16503 [Leader]" -11:58.478 [INFO] TestAgent.server: cluster leadership acquired -11:58.478 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16503 -11:58.479 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:58.480 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:58.480 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:58.480 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:58.480 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:58.480 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:58.480 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e642cf40-f1d8-003e-c7d4-124ae6867bae -11:58.480 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e642cf40-f1d8-003e-c7d4-124ae6867bae -11:58.481 [INFO] TestAgent.server: New leader elected: payload=Node-e642cf40-f1d8-003e-c7d4-124ae6867bae -11:58.481 [INFO] TestAgent.server: federation state anti-entropy synced -11:58.486 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:58.486 [INFO] TestAgent: Synced node info -11:58.872 [INFO] TestAgent: Requesting shutdown -11:58.872 [INFO] TestAgent.server: shutting down server -11:58.872 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:58.872 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:58.872 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:58.872 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:58.872 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:58.872 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:58.872 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:58.872 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:58.873 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:58.873 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:58.873 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:58.873Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:58.873Z [INFO] agent.router.manager: shutting down -11:58.873 [INFO] TestAgent: consul server down -11:58.873 [INFO] TestAgent: shutdown complete -11:58.873 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16498 network=tcp -11:58.873 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16498 network=udp -11:58.873 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16499 network=tcp -11:59.373 [INFO] TestAgent: Waiting for endpoints to shut down -11:59.373 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:58.320Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:58.378Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:58.380 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5cfe9eb8-3cb2-979e-b396-02e7227e87ec Address:127.0.0.1:16552}]" -11:58.381 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5cfe9eb8-3cb2-979e-b396-02e7227e87ec.dc1 127.0.0.1 -11:58.381 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16552 [Follower]" leader= -11:58.382 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5cfe9eb8-3cb2-979e-b396-02e7227e87ec 127.0.0.1 -2021-10-31T23:11:58.382Z [INFO] agent.router: Initializing LAN area manager -11:58.382 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16547 network=udp -11:58.382 [INFO] TestAgent.server: Adding LAN server: server="Node-5cfe9eb8-3cb2-979e-b396-02e7227e87ec (Addr: tcp/127.0.0.1:16552) (DC: dc1)" -11:58.382 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5cfe9eb8-3cb2-979e-b396-02e7227e87ec.dc1 area=wan -11:58.382 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16547 network=tcp -11:58.383 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16548 network=tcp -11:58.383 [INFO] TestAgent: started state syncer -11:58.383 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16553 network=tcp -11:58.452 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:58.452 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16552 [Candidate]" term=2 -11:58.452 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:58.452 [DEBUG] TestAgent.server.raft: vote granted: from=5cfe9eb8-3cb2-979e-b396-02e7227e87ec term=2 tally=1 -11:58.452 [INFO] TestAgent.server.raft: election won: tally=1 -11:58.452 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16552 [Leader]" -11:58.452 [INFO] TestAgent.server: cluster leadership acquired -11:58.452 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16552 -11:58.452 [INFO] TestAgent.server: New leader elected: payload=Node-5cfe9eb8-3cb2-979e-b396-02e7227e87ec -11:58.453 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:58.455 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:58.455 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:58.455 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:58.455 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:58.455 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:58.455 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5cfe9eb8-3cb2-979e-b396-02e7227e87ec -11:58.455 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5cfe9eb8-3cb2-979e-b396-02e7227e87ec -11:58.455 [INFO] TestAgent.server: federation state anti-entropy synced -11:58.752 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:58.753 [INFO] TestAgent: Synced node info -11:58.753 [DEBUG] TestAgent: Node info in sync -11:58.914 [INFO] TestAgent: Requesting shutdown -11:58.914 [INFO] TestAgent.server: shutting down server -11:58.914 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:58.914 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:58.914 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:58.914 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:58.914 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:58.914 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:58.914 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:58.914 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:58.914 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:58.914 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:58.914 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:58.914Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:58.915Z [INFO] agent.router.manager: shutting down -11:58.915 [INFO] TestAgent: consul server down -11:58.915 [INFO] TestAgent: shutdown complete -11:58.915 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16547 network=tcp -11:58.915 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16547 network=udp -11:58.915 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16548 network=tcp -11:59.415 [INFO] TestAgent: Waiting for endpoints to shut down -11:59.415 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:58.384Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:58.486Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:58.498 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c02c7468-7a92-6406-cae3-d9a8a2bdf911 Address:127.0.0.1:16559}]" -11:58.500 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c02c7468-7a92-6406-cae3-d9a8a2bdf911.dc1 127.0.0.1 -11:58.506 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c02c7468-7a92-6406-cae3-d9a8a2bdf911 127.0.0.1 -2021-10-31T23:11:58.506Z [INFO] agent.router: Initializing LAN area manager -11:58.507 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16554 network=udp -11:58.507 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16554 network=tcp -11:58.508 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16555 network=tcp -11:58.508 [INFO] TestAgent: started state syncer -11:58.508 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16560 network=tcp -11:58.508 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c02c7468-7a92-6406-cae3-d9a8a2bdf911.dc1 area=wan -11:58.510 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16559 [Follower]" leader= -11:58.510 [INFO] TestAgent.server: Adding LAN server: server="Node-c02c7468-7a92-6406-cae3-d9a8a2bdf911 (Addr: tcp/127.0.0.1:16559) (DC: dc1)" -11:58.578 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:58.578 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16559 [Candidate]" term=2 -11:58.578 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:58.578 [DEBUG] TestAgent.server.raft: vote granted: from=c02c7468-7a92-6406-cae3-d9a8a2bdf911 term=2 tally=1 -11:58.578 [INFO] TestAgent.server.raft: election won: tally=1 -11:58.578 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16559 [Leader]" -11:58.578 [INFO] TestAgent.server: cluster leadership acquired -11:58.578 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16559 -11:58.578 [INFO] TestAgent.server: New leader elected: payload=Node-c02c7468-7a92-6406-cae3-d9a8a2bdf911 -11:58.579 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:58.580 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:58.580 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:58.580 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:58.580 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:58.580 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:58.580 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c02c7468-7a92-6406-cae3-d9a8a2bdf911 -11:58.580 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c02c7468-7a92-6406-cae3-d9a8a2bdf911 -11:58.581 [INFO] TestAgent.server: federation state anti-entropy synced -11:58.913 [INFO] TestAgent: Requesting shutdown -11:58.913 [INFO] TestAgent.server: shutting down server -11:58.914 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:58.914 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:58.914 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:58.914 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:58.914 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:58.914 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:58.914 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:58.914 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:58.914 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:58.914 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:58.914 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:58.914 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:58.914Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:58.914Z [INFO] agent.router.manager: shutting down -11:58.914 [INFO] TestAgent: consul server down -11:58.914 [INFO] TestAgent: shutdown complete -11:58.914 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16554 network=tcp -11:58.914 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16554 network=udp -11:58.914 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16555 network=tcp -11:59.416 [INFO] TestAgent: Waiting for endpoints to shut down -11:59.416 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:58.323Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:58.483Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:58.485 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fbb163ac-33b6-6bb8-bd62-90c7e50e1c7a Address:127.0.0.1:16510}]" -11:58.485 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fbb163ac-33b6-6bb8-bd62-90c7e50e1c7a.dc1 127.0.0.1 -11:58.485 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16510 [Follower]" leader= -11:58.486 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fbb163ac-33b6-6bb8-bd62-90c7e50e1c7a 127.0.0.1 -2021-10-31T23:11:58.486Z [INFO] agent.router: Initializing LAN area manager -11:58.486 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16505 network=udp -11:58.486 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fbb163ac-33b6-6bb8-bd62-90c7e50e1c7a.dc1 area=wan -11:58.486 [INFO] TestAgent.server: Adding LAN server: server="Node-fbb163ac-33b6-6bb8-bd62-90c7e50e1c7a (Addr: tcp/127.0.0.1:16510) (DC: dc1)" -11:58.486 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16505 network=tcp -11:58.487 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16506 network=tcp -11:58.487 [INFO] TestAgent: started state syncer -11:58.487 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16511 network=tcp -11:58.535 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:58.535 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16510 [Candidate]" term=2 -11:58.536 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:58.536 [DEBUG] TestAgent.server.raft: vote granted: from=fbb163ac-33b6-6bb8-bd62-90c7e50e1c7a term=2 tally=1 -11:58.536 [INFO] TestAgent.server.raft: election won: tally=1 -11:58.536 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16510 [Leader]" -11:58.536 [INFO] TestAgent.server: cluster leadership acquired -11:58.536 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16510 -11:58.537 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:58.538 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:58.538 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:58.538 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:58.538 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:58.538 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:58.538 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fbb163ac-33b6-6bb8-bd62-90c7e50e1c7a -11:58.538 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fbb163ac-33b6-6bb8-bd62-90c7e50e1c7a -11:58.539 [INFO] TestAgent.server: New leader elected: payload=Node-fbb163ac-33b6-6bb8-bd62-90c7e50e1c7a -11:58.539 [INFO] TestAgent.server: federation state anti-entropy synced -11:58.546 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:58.546 [INFO] TestAgent: Synced node info -11:58.546 [DEBUG] TestAgent: Node info in sync -11:58.762 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:58.762 [DEBUG] TestAgent: Node info in sync -11:59.017 [INFO] TestAgent: Requesting shutdown -11:59.017 [INFO] TestAgent.server: shutting down server -11:59.017 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:59.017 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:59.017 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:59.017 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:59.017 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:59.017 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:59.017 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:59.017 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:59.017 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:59.017 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -11:59.017 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:11:59.017Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:59.017Z [INFO] agent.router.manager: shutting down -11:59.017 [INFO] TestAgent: consul server down -11:59.017 [INFO] TestAgent: shutdown complete -11:59.017 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16505 network=tcp -11:59.017 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16505 network=udp -11:59.017 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16506 network=tcp -11:59.518 [INFO] TestAgent: Waiting for endpoints to shut down -11:59.518 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:58.328Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:58.428Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:58.434 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cb7c046b-90e1-8e09-f22e-e984b84295b6 Address:127.0.0.1:16517}]" -11:58.435 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cb7c046b-90e1-8e09-f22e-e984b84295b6.dc1 127.0.0.1 -11:58.436 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cb7c046b-90e1-8e09-f22e-e984b84295b6 127.0.0.1 -2021-10-31T23:11:58.436Z [INFO] agent.router: Initializing LAN area manager -11:58.436 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16512 network=udp -11:58.436 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16517 [Follower]" leader= -11:58.437 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cb7c046b-90e1-8e09-f22e-e984b84295b6.dc1 area=wan -11:58.437 [INFO] TestAgent.server: Adding LAN server: server="Node-cb7c046b-90e1-8e09-f22e-e984b84295b6 (Addr: tcp/127.0.0.1:16517) (DC: dc1)" -11:58.481 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16512 network=tcp -11:58.482 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16513 network=tcp -11:58.482 [INFO] TestAgent: started state syncer -11:58.482 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16518 network=tcp -11:58.509 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:58.510 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16517 [Candidate]" term=2 -11:58.511 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:58.511 [DEBUG] TestAgent.server.raft: vote granted: from=cb7c046b-90e1-8e09-f22e-e984b84295b6 term=2 tally=1 -11:58.511 [INFO] TestAgent.server.raft: election won: tally=1 -11:58.511 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16517 [Leader]" -11:58.511 [INFO] TestAgent.server: cluster leadership acquired -11:58.511 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16517 -11:58.512 [INFO] TestAgent.server: New leader elected: payload=Node-cb7c046b-90e1-8e09-f22e-e984b84295b6 -11:58.515 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:58.517 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:58.517 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:58.517 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:58.517 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:58.517 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:58.517 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cb7c046b-90e1-8e09-f22e-e984b84295b6 -11:58.517 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cb7c046b-90e1-8e09-f22e-e984b84295b6 -11:58.526 [INFO] TestAgent.server: federation state anti-entropy synced -11:58.898 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:58.898 [INFO] TestAgent: Synced node info -11:58.898 [DEBUG] TestAgent: Node info in sync -11:59.088 [INFO] TestAgent: Requesting shutdown -11:59.088 [INFO] TestAgent.server: shutting down server -11:59.088 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:59.088 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:59.088 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:59.088 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:59.088 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:59.088 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:59.088 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:59.088 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:59.088 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:59.088 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:59.088 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:59.088Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:59.088Z [INFO] agent.router.manager: shutting down -11:59.088 [INFO] TestAgent: consul server down -11:59.088 [INFO] TestAgent: shutdown complete -11:59.088 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16512 network=tcp -11:59.088 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16512 network=udp -11:59.088 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16513 network=tcp -11:59.589 [INFO] TestAgent: Waiting for endpoints to shut down -11:59.589 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:58.391Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:58.506Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:58.509 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:68741ad7-c579-4d09-0c8f-605ef67e17ae Address:127.0.0.1:16538}]" -11:58.510 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16538 [Follower]" leader= -11:58.516 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-68741ad7-c579-4d09-0c8f-605ef67e17ae.dc1 127.0.0.1 -11:58.517 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-68741ad7-c579-4d09-0c8f-605ef67e17ae 127.0.0.1 -2021-10-31T23:11:58.517Z [INFO] agent.router: Initializing LAN area manager -11:58.518 [INFO] TestAgent.server: Adding LAN server: server="Node-68741ad7-c579-4d09-0c8f-605ef67e17ae (Addr: tcp/127.0.0.1:16538) (DC: dc1)" -11:58.518 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-68741ad7-c579-4d09-0c8f-605ef67e17ae.dc1 area=wan -11:58.518 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16533 network=tcp -11:58.519 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16533 network=udp -11:58.520 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16534 network=tcp -11:58.520 [INFO] TestAgent: started state syncer -11:58.520 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16539 network=tcp -11:58.582 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:58.582 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16538 [Candidate]" term=2 -11:58.582 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:58.582 [DEBUG] TestAgent.server.raft: vote granted: from=68741ad7-c579-4d09-0c8f-605ef67e17ae term=2 tally=1 -11:58.582 [INFO] TestAgent.server.raft: election won: tally=1 -11:58.582 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16538 [Leader]" -11:58.582 [INFO] TestAgent.server: cluster leadership acquired -11:58.582 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16538 -11:58.583 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:58.584 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:58.584 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:58.584 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:58.584 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:58.584 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:58.584 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-68741ad7-c579-4d09-0c8f-605ef67e17ae -11:58.584 [INFO] TestAgent.server: member joined, marking health alive: member=Node-68741ad7-c579-4d09-0c8f-605ef67e17ae -11:58.585 [INFO] TestAgent.server: New leader elected: payload=Node-68741ad7-c579-4d09-0c8f-605ef67e17ae -11:58.585 [INFO] TestAgent.server: federation state anti-entropy synced -11:58.739 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:58.739 [INFO] TestAgent: Synced node info -11:58.739 [DEBUG] TestAgent: Node info in sync -11:59.088 [INFO] TestAgent: Requesting shutdown -11:59.089 [INFO] TestAgent.server: shutting down server -11:59.089 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:59.089 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:59.089 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:59.089 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:59.089 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:59.089 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:59.089 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:59.089 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:59.089 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:59.089 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:59.089 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:59.089Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:59.089Z [INFO] agent.router.manager: shutting down -11:59.089 [INFO] TestAgent: consul server down -11:59.089 [INFO] TestAgent: shutdown complete -11:59.089 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16533 network=tcp -11:59.089 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16533 network=udp -11:59.089 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16534 network=tcp -11:59.590 [INFO] TestAgent: Waiting for endpoints to shut down -11:59.590 [INFO] TestAgent: Endpoints down -2021-10-31T23:11:58.305Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:58.483Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:58.484 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fbd51f30-33b1-b199-8fb4-5fbf59f62794 Address:127.0.0.1:16531}]" -11:58.485 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fbd51f30-33b1-b199-8fb4-5fbf59f62794.dc1 127.0.0.1 -11:58.485 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16531 [Follower]" leader= -11:58.487 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fbd51f30-33b1-b199-8fb4-5fbf59f62794 127.0.0.1 -2021-10-31T23:11:58.487Z [INFO] agent.router: Initializing LAN area manager -11:58.487 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16526 network=udp -11:58.487 [INFO] TestAgent.server: Adding LAN server: server="Node-fbd51f30-33b1-b199-8fb4-5fbf59f62794 (Addr: tcp/127.0.0.1:16531) (DC: dc1)" -11:58.487 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fbd51f30-33b1-b199-8fb4-5fbf59f62794.dc1 area=wan -11:58.487 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16526 network=tcp -11:58.488 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16527 network=tcp -11:58.488 [INFO] TestAgent: started state syncer -11:58.488 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16532 network=tcp -11:58.548 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:58.548 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16531 [Candidate]" term=2 -11:58.548 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:58.548 [DEBUG] TestAgent.server.raft: vote granted: from=fbd51f30-33b1-b199-8fb4-5fbf59f62794 term=2 tally=1 -11:58.548 [INFO] TestAgent.server.raft: election won: tally=1 -11:58.548 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16531 [Leader]" -11:58.548 [INFO] TestAgent.server: cluster leadership acquired -11:58.549 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16531 -11:58.549 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:58.549 [INFO] TestAgent.server: New leader elected: payload=Node-fbd51f30-33b1-b199-8fb4-5fbf59f62794 -11:58.551 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:58.551 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:58.551 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:58.551 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:58.551 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:58.551 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fbd51f30-33b1-b199-8fb4-5fbf59f62794 -11:58.551 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fbd51f30-33b1-b199-8fb4-5fbf59f62794 -11:58.552 [INFO] TestAgent.server: federation state anti-entropy synced -11:58.746 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:58.746 [INFO] TestAgent: Synced node info -11:58.746 [DEBUG] TestAgent: Node info in sync -11:59.158 [INFO] TestAgent: Requesting shutdown -11:59.158 [INFO] TestAgent.server: shutting down server -11:59.158 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:59.158 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:59.158 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:59.158 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:59.158 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:59.158 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:59.158 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:59.158 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:59.158 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:59.158 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:59.158 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:59.158Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:59.158Z [INFO] agent.router.manager: shutting down -11:59.158 [INFO] TestAgent: consul server down -11:59.158 [INFO] TestAgent: shutdown complete -11:59.158 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16526 network=tcp -11:59.158 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16526 network=udp -11:59.158 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16527 network=tcp -11:59.659 [INFO] TestAgent: Waiting for endpoints to shut down -11:59.659 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:15.433Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:15.462Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:15.464 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:302a6d83-ca45-d086-57ac-a5c94ca4862b Address:127.0.0.1:23657}]" +48:15.465 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-302a6d83-ca45-d086-57ac-a5c94ca4862b.dc1 127.0.0.1 +48:15.465 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23657 [Follower]" leader= +48:15.508 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:15.508 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23657 [Candidate]" term=2 +48:15.508 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=302a6d83-ca45-d086-57ac-a5c94ca4862b fallback=127.0.0.1:23657 error="Could not find address for server id 302a6d83-ca45-d086-57ac-a5c94ca4862b" +48:15.508 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:15.508 [DEBUG] TestAgent.server.raft: vote granted: from=302a6d83-ca45-d086-57ac-a5c94ca4862b term=2 tally=1 +48:15.508 [INFO] TestAgent.server.raft: election won: tally=1 +48:15.508 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23657 [Leader]" +48:15.509 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-302a6d83-ca45-d086-57ac-a5c94ca4862b 127.0.0.1 +2022-12-04T05:48:15.509Z [INFO] agent.router: Initializing LAN area manager +48:15.510 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23652 network=udp +48:15.510 [INFO] TestAgent.server: Adding LAN server: server="Node-302a6d83-ca45-d086-57ac-a5c94ca4862b (Addr: tcp/127.0.0.1:23657) (DC: dc1)" +48:15.510 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-302a6d83-ca45-d086-57ac-a5c94ca4862b.dc1 area=wan +48:15.510 [INFO] TestAgent.server: cluster leadership acquired +48:15.510 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23657 +48:15.511 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:15.511 [INFO] TestAgent.server: New leader elected: payload=Node-302a6d83-ca45-d086-57ac-a5c94ca4862b +48:15.511 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23652 network=tcp +48:15.512 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23653 network=tcp +48:15.512 [INFO] TestAgent: started state syncer +48:15.512 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23658 network=tcp +48:15.512 [INFO] TestAgent: Synced node info +48:15.512 [DEBUG] TestAgent: Node info in sync +48:15.513 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:15.513 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:15.513 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:15.513 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:15.513 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:15.513 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-302a6d83-ca45-d086-57ac-a5c94ca4862b +48:15.513 [INFO] TestAgent.server: member joined, marking health alive: member=Node-302a6d83-ca45-d086-57ac-a5c94ca4862b +48:15.513 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-302a6d83-ca45-d086-57ac-a5c94ca4862b +48:15.762 [INFO] TestAgent: Requesting shutdown +48:15.762 [INFO] TestAgent.server: shutting down server +48:15.762 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:15.762 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:15.762 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:15.762 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:15.762 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:15.762 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" +48:15.762 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:15.762 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:15.762 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:15.762 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:15.762 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:15.762Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:15.762Z [INFO] agent.router.manager: shutting down +48:15.762 [INFO] TestAgent: consul server down +48:15.762 [INFO] TestAgent: shutdown complete +48:15.762 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23652 network=tcp +48:15.762 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23652 network=udp +48:15.762 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23653 network=tcp +48:16.263 [INFO] TestAgent: Waiting for endpoints to shut down +48:16.263 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:15.403Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:15.435Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:15.437 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:613bc2fc-4d78-c409-a4f9-834f391fd840 Address:127.0.0.1:23671}]" +48:15.437 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23671 [Follower]" leader= +48:15.438 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-613bc2fc-4d78-c409-a4f9-834f391fd840.dc1 127.0.0.1 +48:15.438 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-613bc2fc-4d78-c409-a4f9-834f391fd840 127.0.0.1 +2022-12-04T05:48:15.438Z [INFO] agent.router: Initializing LAN area manager +48:15.439 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23666 network=udp +48:15.439 [INFO] TestAgent.server: Adding LAN server: server="Node-613bc2fc-4d78-c409-a4f9-834f391fd840 (Addr: tcp/127.0.0.1:23671) (DC: dc1)" +48:15.439 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-613bc2fc-4d78-c409-a4f9-834f391fd840.dc1 area=wan +48:15.439 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23666 network=tcp +48:15.440 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23667 network=tcp +48:15.440 [INFO] TestAgent: started state syncer +48:15.440 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23672 network=tcp +48:15.491 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:15.491 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23671 [Candidate]" term=2 +48:15.492 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:15.492 [DEBUG] TestAgent.server.raft: vote granted: from=613bc2fc-4d78-c409-a4f9-834f391fd840 term=2 tally=1 +48:15.492 [INFO] TestAgent.server.raft: election won: tally=1 +48:15.492 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23671 [Leader]" +48:15.492 [INFO] TestAgent.server: cluster leadership acquired +48:15.492 [INFO] TestAgent.server: New leader elected: payload=Node-613bc2fc-4d78-c409-a4f9-834f391fd840 +48:15.492 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23671 +48:15.493 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:15.494 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:15.494 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:15.494 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:15.494 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:15.494 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:15.494 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-613bc2fc-4d78-c409-a4f9-834f391fd840 +48:15.494 [INFO] TestAgent.server: member joined, marking health alive: member=Node-613bc2fc-4d78-c409-a4f9-834f391fd840 +48:15.495 [INFO] TestAgent.server: federation state anti-entropy synced +48:15.807 [INFO] TestAgent: Requesting shutdown +48:15.807 [INFO] TestAgent.server: shutting down server +48:15.807 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:15.807 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:15.807 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:15.807 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:15.807 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:15.808 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:15.808 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:15.808 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:15.808 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:15.808 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:15.808 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:15.808 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:15.808Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:15.808Z [INFO] agent.router.manager: shutting down +48:15.808 [INFO] TestAgent: consul server down +48:15.808 [INFO] TestAgent: shutdown complete +48:15.808 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23666 network=tcp +48:15.808 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23666 network=udp +48:15.808 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23667 network=tcp +48:16.308 [INFO] TestAgent: Waiting for endpoints to shut down +48:16.308 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:15.404Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:15.463Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:15.509 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aeef0162-23ec-877c-bcfe-26ceda2f0101 Address:127.0.0.1:23678}]" +48:15.509 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aeef0162-23ec-877c-bcfe-26ceda2f0101.dc1 127.0.0.1 +48:15.510 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aeef0162-23ec-877c-bcfe-26ceda2f0101 127.0.0.1 +2022-12-04T05:48:15.510Z [INFO] agent.router: Initializing LAN area manager +48:15.510 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23673 network=udp +48:15.510 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23678 [Follower]" leader= +48:15.511 [INFO] TestAgent.server: Adding LAN server: server="Node-aeef0162-23ec-877c-bcfe-26ceda2f0101 (Addr: tcp/127.0.0.1:23678) (DC: dc1)" +48:15.511 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aeef0162-23ec-877c-bcfe-26ceda2f0101.dc1 area=wan +48:15.511 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23673 network=tcp +48:15.511 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23674 network=tcp +48:15.511 [INFO] TestAgent: started state syncer +48:15.511 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23679 network=tcp +48:15.558 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:15.558 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23678 [Candidate]" term=2 +48:15.559 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:15.559 [DEBUG] TestAgent.server.raft: vote granted: from=aeef0162-23ec-877c-bcfe-26ceda2f0101 term=2 tally=1 +48:15.559 [INFO] TestAgent.server.raft: election won: tally=1 +48:15.559 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23678 [Leader]" +48:15.559 [INFO] TestAgent.server: cluster leadership acquired +48:15.559 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23678 +48:15.559 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:15.560 [INFO] TestAgent.server: New leader elected: payload=Node-aeef0162-23ec-877c-bcfe-26ceda2f0101 +48:15.560 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:15.560 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:15.560 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:15.560 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:15.560 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:15.560 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aeef0162-23ec-877c-bcfe-26ceda2f0101 +48:15.561 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aeef0162-23ec-877c-bcfe-26ceda2f0101 +48:15.561 [INFO] TestAgent.server: federation state anti-entropy synced +48:15.621 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:15.622 [INFO] TestAgent: Synced node info +48:15.673 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:15.673 [DEBUG] TestAgent: Node info in sync +48:15.673 [DEBUG] TestAgent: Node info in sync +48:15.812 [INFO] TestAgent: Requesting shutdown +48:15.812 [INFO] TestAgent.server: shutting down server +48:15.812 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:15.812 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:15.812 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:15.812 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:15.812 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:15.812 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:15.812 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:15.812 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:15.812 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:15.812 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:15.813 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:15.813Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:15.813Z [INFO] agent.router.manager: shutting down +48:15.813 [INFO] TestAgent: consul server down +48:15.813 [INFO] TestAgent: shutdown complete +48:15.813 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23673 network=tcp +48:15.813 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23673 network=udp +48:15.813 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23674 network=tcp +48:16.313 [INFO] TestAgent: Waiting for endpoints to shut down +48:16.313 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:15.401Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:15.421Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:15.440 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:93c51732-badf-d360-a24f-e1d419c55ed0 Address:127.0.0.1:23664}]" +48:15.441 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23664 [Follower]" leader= +48:15.441 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-93c51732-badf-d360-a24f-e1d419c55ed0.dc1 127.0.0.1 +48:15.441 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-93c51732-badf-d360-a24f-e1d419c55ed0 127.0.0.1 +2022-12-04T05:48:15.441Z [INFO] agent.router: Initializing LAN area manager +48:15.442 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23659 network=udp +48:15.442 [INFO] TestAgent.server: Adding LAN server: server="Node-93c51732-badf-d360-a24f-e1d419c55ed0 (Addr: tcp/127.0.0.1:23664) (DC: dc1)" +48:15.442 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-93c51732-badf-d360-a24f-e1d419c55ed0.dc1 area=wan +48:15.442 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23659 network=tcp +48:15.442 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23660 network=tcp +48:15.442 [INFO] TestAgent: started state syncer +48:15.443 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23665 network=tcp +48:15.480 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:15.480 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23664 [Candidate]" term=2 +48:15.480 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:15.480 [DEBUG] TestAgent.server.raft: vote granted: from=93c51732-badf-d360-a24f-e1d419c55ed0 term=2 tally=1 +48:15.480 [INFO] TestAgent.server.raft: election won: tally=1 +48:15.480 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23664 [Leader]" +48:15.480 [INFO] TestAgent.server: cluster leadership acquired +48:15.481 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23664 +48:15.481 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:15.482 [INFO] TestAgent.server: New leader elected: payload=Node-93c51732-badf-d360-a24f-e1d419c55ed0 +48:15.483 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:15.483 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:15.483 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:15.483 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:15.483 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:15.483 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-93c51732-badf-d360-a24f-e1d419c55ed0 +48:15.483 [INFO] TestAgent.server: member joined, marking health alive: member=Node-93c51732-badf-d360-a24f-e1d419c55ed0 +48:15.484 [INFO] TestAgent.server: federation state anti-entropy synced +48:15.512 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:15.512 [INFO] TestAgent: Synced node info +48:15.667 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:15.668 [DEBUG] TestAgent: Node info in sync +48:15.668 [DEBUG] TestAgent: Node info in sync +48:15.827 [INFO] TestAgent: Requesting shutdown +48:15.827 [INFO] TestAgent.server: shutting down server +48:15.827 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:15.827 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:15.827 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:15.827 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:15.827 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:15.827 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:15.828 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:15.827 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:15.828 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:15.827 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:15.827 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:48:15.828Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:15.828Z [INFO] agent.router.manager: shutting down +48:15.828 [INFO] TestAgent: consul server down +48:15.828 [INFO] TestAgent: shutdown complete +48:15.828 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23659 network=tcp +48:15.828 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23659 network=udp +48:15.828 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23660 network=tcp +48:16.328 [INFO] TestAgent: Waiting for endpoints to shut down +48:16.328 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:15.406Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:15.436Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:15.438 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:242755db-a85a-23a3-b402-e76230c7da3a Address:127.0.0.1:23685}]" +48:15.439 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-242755db-a85a-23a3-b402-e76230c7da3a.dc1 127.0.0.1 +48:15.439 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23685 [Follower]" leader= +48:15.439 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-242755db-a85a-23a3-b402-e76230c7da3a 127.0.0.1 +2022-12-04T05:48:15.439Z [INFO] agent.router: Initializing LAN area manager +48:15.440 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23680 network=udp +48:15.440 [INFO] TestAgent.server: Adding LAN server: server="Node-242755db-a85a-23a3-b402-e76230c7da3a (Addr: tcp/127.0.0.1:23685) (DC: dc1)" +48:15.440 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-242755db-a85a-23a3-b402-e76230c7da3a.dc1 area=wan +48:15.440 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23680 network=tcp +48:15.440 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23681 network=tcp +48:15.441 [INFO] TestAgent: started state syncer +48:15.441 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23686 network=tcp +48:15.486 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:15.486 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23685 [Candidate]" term=2 +48:15.486 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:15.486 [DEBUG] TestAgent.server.raft: vote granted: from=242755db-a85a-23a3-b402-e76230c7da3a term=2 tally=1 +48:15.486 [INFO] TestAgent.server.raft: election won: tally=1 +48:15.486 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23685 [Leader]" +48:15.487 [INFO] TestAgent.server: cluster leadership acquired +48:15.487 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23685 +48:15.487 [INFO] TestAgent.server: New leader elected: payload=Node-242755db-a85a-23a3-b402-e76230c7da3a +48:15.488 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:15.489 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:15.489 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:15.489 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:15.489 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:15.489 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:15.489 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-242755db-a85a-23a3-b402-e76230c7da3a +48:15.489 [INFO] TestAgent.server: member joined, marking health alive: member=Node-242755db-a85a-23a3-b402-e76230c7da3a +48:15.490 [INFO] TestAgent.server: federation state anti-entropy synced +48:15.652 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:15.653 [INFO] TestAgent: Synced node info +48:15.653 [DEBUG] TestAgent: Node info in sync +48:15.658 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:15.658 [DEBUG] TestAgent: Node info in sync +48:15.847 [INFO] TestAgent: Requesting shutdown +48:15.847 [INFO] TestAgent.server: shutting down server +48:15.847 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:15.847 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:15.847 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:15.847 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:15.847 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:15.847 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:15.847 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:15.847 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:15.847 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:15.847 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:15.847 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:15.848Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:15.848Z [INFO] agent.router.manager: shutting down +48:15.848 [INFO] TestAgent: consul server down +48:15.848 [INFO] TestAgent: shutdown complete +48:15.848 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23680 network=tcp +48:15.848 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23680 network=udp +48:15.848 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23681 network=tcp +48:16.350 [INFO] TestAgent: Waiting for endpoints to shut down +48:16.350 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:15.381Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:15.407Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:15.409 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3cfc2355-1aca-56f6-e699-5cce15d1a26e Address:127.0.0.1:23650}]" +48:15.410 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3cfc2355-1aca-56f6-e699-5cce15d1a26e.dc1 127.0.0.1 +48:15.410 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3cfc2355-1aca-56f6-e699-5cce15d1a26e 127.0.0.1 +2022-12-04T05:48:15.410Z [INFO] agent.router: Initializing LAN area manager +48:15.410 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23645 network=udp +48:15.410 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23650 [Follower]" leader= +48:15.411 [INFO] TestAgent.server: Adding LAN server: server="Node-3cfc2355-1aca-56f6-e699-5cce15d1a26e (Addr: tcp/127.0.0.1:23650) (DC: dc1)" +48:15.411 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3cfc2355-1aca-56f6-e699-5cce15d1a26e.dc1 area=wan +48:15.412 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23645 network=tcp +48:15.413 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23646 network=tcp +48:15.417 [INFO] TestAgent: started state syncer +48:15.424 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23651 network=tcp +48:15.465 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:15.465 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23650 [Candidate]" term=2 +48:15.466 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:15.466 [DEBUG] TestAgent.server.raft: vote granted: from=3cfc2355-1aca-56f6-e699-5cce15d1a26e term=2 tally=1 +48:15.466 [INFO] TestAgent.server.raft: election won: tally=1 +48:15.466 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23650 [Leader]" +48:15.466 [INFO] TestAgent.server: cluster leadership acquired +48:15.466 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23650 +48:15.466 [INFO] TestAgent.server: New leader elected: payload=Node-3cfc2355-1aca-56f6-e699-5cce15d1a26e +48:15.467 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:15.468 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:15.468 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:15.468 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:15.468 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:15.468 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:15.468 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3cfc2355-1aca-56f6-e699-5cce15d1a26e +48:15.468 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3cfc2355-1aca-56f6-e699-5cce15d1a26e +48:15.476 [INFO] TestAgent.server: federation state anti-entropy synced +48:15.704 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:15.704 [INFO] TestAgent: Synced node info +48:15.704 [DEBUG] TestAgent: Node info in sync +48:15.827 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:15.827 [DEBUG] TestAgent: Node info in sync +48:15.915 [INFO] TestAgent: Requesting shutdown +48:15.915 [INFO] TestAgent.server: shutting down server +48:15.915 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:15.915 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:15.915 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:15.915 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:15.915 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:15.915 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:15.915 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:15.915 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:15.915 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:15.915 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:15.915 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:48:15.915Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:15.915Z [INFO] agent.router.manager: shutting down +48:15.916 [INFO] TestAgent: consul server down +48:15.916 [INFO] TestAgent: shutdown complete +48:15.916 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23645 network=tcp +48:15.916 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23645 network=udp +48:15.916 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23646 network=tcp +48:16.416 [INFO] TestAgent: Waiting for endpoints to shut down +48:16.416 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:15.439Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:15.462Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:15.464 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8c174bf5-5bd9-4f56-2cd6-a997a659d36a Address:127.0.0.1:23699}]" +48:15.465 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23699 [Follower]" leader= +48:15.509 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8c174bf5-5bd9-4f56-2cd6-a997a659d36a.dc1 127.0.0.1 +48:15.509 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8c174bf5-5bd9-4f56-2cd6-a997a659d36a 127.0.0.1 +2022-12-04T05:48:15.509Z [INFO] agent.router: Initializing LAN area manager +48:15.510 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23694 network=udp +48:15.510 [INFO] TestAgent.server: Adding LAN server: server="Node-8c174bf5-5bd9-4f56-2cd6-a997a659d36a (Addr: tcp/127.0.0.1:23699) (DC: dc1)" +48:15.510 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8c174bf5-5bd9-4f56-2cd6-a997a659d36a.dc1 area=wan +48:15.510 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23694 network=tcp +48:15.511 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23695 network=tcp +48:15.511 [INFO] TestAgent: started state syncer +48:15.511 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23700 network=tcp +48:15.515 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:15.515 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23699 [Candidate]" term=2 +48:15.515 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:15.515 [DEBUG] TestAgent.server.raft: vote granted: from=8c174bf5-5bd9-4f56-2cd6-a997a659d36a term=2 tally=1 +48:15.515 [INFO] TestAgent.server.raft: election won: tally=1 +48:15.515 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23699 [Leader]" +48:15.515 [INFO] TestAgent.server: cluster leadership acquired +48:15.515 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23699 +48:15.516 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:15.516 [INFO] TestAgent.server: New leader elected: payload=Node-8c174bf5-5bd9-4f56-2cd6-a997a659d36a +48:15.517 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:15.517 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:15.517 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:15.517 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:15.517 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:15.517 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8c174bf5-5bd9-4f56-2cd6-a997a659d36a +48:15.517 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8c174bf5-5bd9-4f56-2cd6-a997a659d36a +48:15.518 [INFO] TestAgent.server: federation state anti-entropy synced +48:15.807 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:15.808 [INFO] TestAgent: Synced node info +48:15.808 [DEBUG] TestAgent: Node info in sync +48:15.935 [INFO] TestAgent: Requesting shutdown +48:15.935 [INFO] TestAgent.server: shutting down server +48:15.935 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:15.935 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:15.935 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:15.935 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:15.935 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:15.935 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:15.935 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:15.935 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:15.935 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:15.935 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:15.935 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:15.935Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:15.935Z [INFO] agent.router.manager: shutting down +48:15.935 [INFO] TestAgent: consul server down +48:15.935 [INFO] TestAgent: shutdown complete +48:15.935 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23694 network=tcp +48:15.935 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23694 network=udp +48:15.935 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23695 network=tcp +48:16.436 [INFO] TestAgent: Waiting for endpoints to shut down +48:16.436 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:15.437Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:15.509Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:15.511 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f6796b13-2b7a-8761-9416-e72b3ea32808 Address:127.0.0.1:23692}]" +48:15.512 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23692 [Follower]" leader= +48:15.512 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f6796b13-2b7a-8761-9416-e72b3ea32808.dc1 127.0.0.1 +48:15.512 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f6796b13-2b7a-8761-9416-e72b3ea32808 127.0.0.1 +2022-12-04T05:48:15.512Z [INFO] agent.router: Initializing LAN area manager +48:15.512 [INFO] TestAgent.server: Adding LAN server: server="Node-f6796b13-2b7a-8761-9416-e72b3ea32808 (Addr: tcp/127.0.0.1:23692) (DC: dc1)" +48:15.512 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f6796b13-2b7a-8761-9416-e72b3ea32808.dc1 area=wan +48:15.513 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23687 network=udp +48:15.513 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23687 network=tcp +48:15.513 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23688 network=tcp +48:15.513 [INFO] TestAgent: started state syncer +48:15.513 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23693 network=tcp +48:15.563 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:15.563 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23692 [Candidate]" term=2 +48:15.563 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:15.563 [DEBUG] TestAgent.server.raft: vote granted: from=f6796b13-2b7a-8761-9416-e72b3ea32808 term=2 tally=1 +48:15.563 [INFO] TestAgent.server.raft: election won: tally=1 +48:15.563 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23692 [Leader]" +48:15.563 [INFO] TestAgent.server: cluster leadership acquired +48:15.563 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23692 +48:15.564 [INFO] TestAgent.server: New leader elected: payload=Node-f6796b13-2b7a-8761-9416-e72b3ea32808 +48:15.564 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:15.565 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:15.565 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:15.565 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:15.565 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:15.565 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:15.565 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f6796b13-2b7a-8761-9416-e72b3ea32808 +48:15.565 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f6796b13-2b7a-8761-9416-e72b3ea32808 +48:15.566 [INFO] TestAgent.server: federation state anti-entropy synced +48:15.788 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:15.789 [INFO] TestAgent: Synced node info +48:15.940 [INFO] TestAgent: Requesting shutdown +48:15.940 [INFO] TestAgent.server: shutting down server +48:15.940 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:15.940 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:15.940 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:15.940 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:15.940 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:15.940 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:15.940 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:15.940 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:15.940 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:15.940 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:15.940 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:15.940Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:15.940Z [INFO] agent.router.manager: shutting down +48:15.941 [INFO] TestAgent: consul server down +48:15.941 [INFO] TestAgent: shutdown complete +48:15.941 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23687 network=tcp +48:15.941 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23687 network=udp +48:15.941 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23688 network=tcp +48:16.441 [INFO] TestAgent: Waiting for endpoints to shut down +48:16.441 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:15.444Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:15.513Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:15.515 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0eab5e07-1cbb-4b0e-3b1e-df4c2f365212 Address:127.0.0.1:23706}]" +48:15.515 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23706 [Follower]" leader= +48:15.515 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0eab5e07-1cbb-4b0e-3b1e-df4c2f365212.dc1 127.0.0.1 +48:15.515 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0eab5e07-1cbb-4b0e-3b1e-df4c2f365212 127.0.0.1 +2022-12-04T05:48:15.515Z [INFO] agent.router: Initializing LAN area manager +48:15.516 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23701 network=udp +48:15.516 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0eab5e07-1cbb-4b0e-3b1e-df4c2f365212.dc1 area=wan +48:15.516 [INFO] TestAgent.server: Adding LAN server: server="Node-0eab5e07-1cbb-4b0e-3b1e-df4c2f365212 (Addr: tcp/127.0.0.1:23706) (DC: dc1)" +48:15.516 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23701 network=tcp +48:15.517 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23702 network=tcp +48:15.517 [INFO] TestAgent: started state syncer +48:15.517 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23707 network=tcp +48:15.580 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:15.580 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23706 [Candidate]" term=2 +48:15.580 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:15.580 [DEBUG] TestAgent.server.raft: vote granted: from=0eab5e07-1cbb-4b0e-3b1e-df4c2f365212 term=2 tally=1 +48:15.580 [INFO] TestAgent.server.raft: election won: tally=1 +48:15.580 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23706 [Leader]" +48:15.580 [INFO] TestAgent.server: cluster leadership acquired +48:15.580 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23706 +48:15.580 [INFO] TestAgent.server: New leader elected: payload=Node-0eab5e07-1cbb-4b0e-3b1e-df4c2f365212 +48:15.581 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:15.589 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:15.589 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:15.589 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:15.589 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:15.589 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:15.589 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0eab5e07-1cbb-4b0e-3b1e-df4c2f365212 +48:15.589 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0eab5e07-1cbb-4b0e-3b1e-df4c2f365212 +48:15.590 [INFO] TestAgent.server: federation state anti-entropy synced +48:15.592 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:15.592 [INFO] TestAgent: Synced node info +48:16.126 [INFO] TestAgent: Requesting shutdown +48:16.126 [INFO] TestAgent.server: shutting down server +48:16.126 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:16.126 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:16.126 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:16.126 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:16.126 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:16.127 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:16.127 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:16.127 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:16.127 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:16.127 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:16.127 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:16.127Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:16.127Z [INFO] agent.router.manager: shutting down +48:16.127 [INFO] TestAgent: consul server down +48:16.127 [INFO] TestAgent: shutdown complete +48:16.127 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23701 network=tcp +48:16.127 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23701 network=udp +48:16.127 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23702 network=tcp +48:16.627 [INFO] TestAgent: Waiting for endpoints to shut down +48:16.627 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:15.443Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:15.464Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:15.509 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7c9a959f-724b-9b34-6a27-8a60cc406265 Address:127.0.0.1:23713}]" +48:15.510 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7c9a959f-724b-9b34-6a27-8a60cc406265.dc1 127.0.0.1 +48:15.510 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7c9a959f-724b-9b34-6a27-8a60cc406265 127.0.0.1 +2022-12-04T05:48:15.510Z [INFO] agent.router: Initializing LAN area manager +48:15.510 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23708 network=udp +48:15.510 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23713 [Follower]" leader= +48:15.511 [INFO] TestAgent.server: Adding LAN server: server="Node-7c9a959f-724b-9b34-6a27-8a60cc406265 (Addr: tcp/127.0.0.1:23713) (DC: dc1)" +48:15.511 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7c9a959f-724b-9b34-6a27-8a60cc406265.dc1 area=wan +48:15.511 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23708 network=tcp +48:15.512 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23709 network=tcp +48:15.512 [INFO] TestAgent: started state syncer +48:15.512 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23714 network=tcp +48:15.554 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:15.554 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23713 [Candidate]" term=2 +48:15.554 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:15.554 [DEBUG] TestAgent.server.raft: vote granted: from=7c9a959f-724b-9b34-6a27-8a60cc406265 term=2 tally=1 +48:15.554 [INFO] TestAgent.server.raft: election won: tally=1 +48:15.554 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23713 [Leader]" +48:15.554 [INFO] TestAgent.server: cluster leadership acquired +48:15.555 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23713 +48:15.555 [INFO] TestAgent.server: New leader elected: payload=Node-7c9a959f-724b-9b34-6a27-8a60cc406265 +48:15.556 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:15.557 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:15.557 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:15.557 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:15.557 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:15.557 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:15.557 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7c9a959f-724b-9b34-6a27-8a60cc406265 +48:15.557 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7c9a959f-724b-9b34-6a27-8a60cc406265 +48:15.557 [INFO] TestAgent.server: federation state anti-entropy synced +48:15.876 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:15.877 [INFO] TestAgent: Synced node info +48:15.877 [DEBUG] TestAgent: Node info in sync +48:16.155 [INFO] TestAgent: Requesting shutdown +48:16.155 [INFO] TestAgent.server: shutting down server +48:16.155 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:16.155 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:16.155 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:16.155 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:16.155 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:16.155 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:16.155 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:16.155 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:16.155 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:16.155 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:16.155 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:16.155Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:16.155Z [INFO] agent.router.manager: shutting down +48:16.155 [INFO] TestAgent: consul server down +48:16.155 [INFO] TestAgent: shutdown complete +48:16.155 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23708 network=tcp +48:16.155 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23708 network=udp +48:16.155 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23709 network=tcp +48:16.656 [INFO] TestAgent: Waiting for endpoints to shut down +48:16.656 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_MultiStartStop (0.00s) - --- PASS: TestAgent_MultiStartStop/#07 (0.95s) - --- PASS: TestAgent_MultiStartStop/#02 (0.94s) - --- PASS: TestAgent_MultiStartStop/#04 (1.11s) - --- PASS: TestAgent_MultiStartStop/#00 (1.15s) - --- PASS: TestAgent_MultiStartStop/#01 (1.17s) - --- PASS: TestAgent_MultiStartStop/#03 (1.19s) - --- PASS: TestAgent_MultiStartStop/#05 (1.29s) - --- PASS: TestAgent_MultiStartStop/#08 (1.36s) - --- PASS: TestAgent_MultiStartStop/#09 (1.35s) - --- PASS: TestAgent_MultiStartStop/#06 (1.43s) + --- PASS: TestAgent_MultiStartStop/#05 (0.90s) + --- PASS: TestAgent_MultiStartStop/#04 (0.94s) + --- PASS: TestAgent_MultiStartStop/#06 (0.94s) + --- PASS: TestAgent_MultiStartStop/#08 (0.96s) + --- PASS: TestAgent_MultiStartStop/#09 (0.97s) + --- PASS: TestAgent_MultiStartStop/#00 (1.05s) + --- PASS: TestAgent_MultiStartStop/#07 (1.06s) + --- PASS: TestAgent_MultiStartStop/#02 (1.06s) + --- PASS: TestAgent_MultiStartStop/#03 (1.26s) + --- PASS: TestAgent_MultiStartStop/#01 (1.27s) === RUN TestAgent_ConnectClusterIDConfig === RUN TestAgent_ConnectClusterIDConfig/default_TestAgent_has_fixed_cluster_id -2021-10-31T23:11:59.675Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:59.688Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:59.690 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ed3df04b-5bda-aa26-d123-3dd014c8fd90 Address:127.0.0.1:16573}]" -11:59.691 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ed3df04b-5bda-aa26-d123-3dd014c8fd90.dc1 127.0.0.1 -11:59.691 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ed3df04b-5bda-aa26-d123-3dd014c8fd90 127.0.0.1 -2021-10-31T23:11:59.691Z [INFO] agent.router: Initializing LAN area manager -11:59.692 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16568 network=udp -11:59.692 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16573 [Follower]" leader= -11:59.692 [INFO] TestAgent.server: Adding LAN server: server="Node-ed3df04b-5bda-aa26-d123-3dd014c8fd90 (Addr: tcp/127.0.0.1:16573) (DC: dc1)" -11:59.692 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ed3df04b-5bda-aa26-d123-3dd014c8fd90.dc1 area=wan -11:59.692 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16568 network=tcp -11:59.692 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16569 network=tcp -11:59.692 [INFO] TestAgent: started state syncer -11:59.693 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16574 network=tcp -11:59.752 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:59.752 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16573 [Candidate]" term=2 -11:59.753 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:59.753 [DEBUG] TestAgent.server.raft: vote granted: from=ed3df04b-5bda-aa26-d123-3dd014c8fd90 term=2 tally=1 -11:59.753 [INFO] TestAgent.server.raft: election won: tally=1 -11:59.753 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16573 [Leader]" -11:59.753 [INFO] TestAgent.server: cluster leadership acquired -11:59.753 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16573 -11:59.753 [INFO] TestAgent.server: New leader elected: payload=Node-ed3df04b-5bda-aa26-d123-3dd014c8fd90 -11:59.754 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:59.755 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:59.755 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:59.755 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:59.755 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:59.755 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:59.755 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ed3df04b-5bda-aa26-d123-3dd014c8fd90 -11:59.755 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ed3df04b-5bda-aa26-d123-3dd014c8fd90 -11:59.756 [INFO] TestAgent.server: federation state anti-entropy synced -11:59.818 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:59.818 [INFO] TestAgent: Synced node info -11:59.818 [DEBUG] TestAgent: Node info in sync -12:00.073 [INFO] TestAgent: Requesting shutdown -12:00.073 [INFO] TestAgent.server: shutting down server -12:00.073 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:00.073 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:00.073 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:00.073 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:00.073 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:00.074 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:00.074 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:00.074 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:00.074 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:00.074 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:00.074 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:00.074Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:00.074Z [INFO] agent.router.manager: shutting down -12:00.074 [INFO] TestAgent: consul server down -12:00.074 [INFO] TestAgent: shutdown complete -12:00.074 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16568 network=tcp -12:00.074 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16568 network=udp -12:00.074 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16569 network=tcp -12:00.577 [INFO] TestAgent: Waiting for endpoints to shut down -12:00.577 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:16.671Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:16.683Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:16.685 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fa080e20-5a6e-52ef-b146-0c8affd2e402 Address:127.0.0.1:23720}]" +48:16.686 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23720 [Follower]" leader= +48:16.686 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fa080e20-5a6e-52ef-b146-0c8affd2e402.dc1 127.0.0.1 +48:16.686 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fa080e20-5a6e-52ef-b146-0c8affd2e402 127.0.0.1 +2022-12-04T05:48:16.686Z [INFO] agent.router: Initializing LAN area manager +48:16.687 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23715 network=udp +48:16.687 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23715 network=tcp +48:16.687 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23716 network=tcp +48:16.688 [INFO] TestAgent: started state syncer +48:16.688 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23721 network=tcp +48:16.688 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fa080e20-5a6e-52ef-b146-0c8affd2e402.dc1 area=wan +48:16.688 [INFO] TestAgent.server: Adding LAN server: server="Node-fa080e20-5a6e-52ef-b146-0c8affd2e402 (Addr: tcp/127.0.0.1:23720) (DC: dc1)" +48:16.751 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:16.751 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23720 [Candidate]" term=2 +48:16.751 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:16.751 [DEBUG] TestAgent.server.raft: vote granted: from=fa080e20-5a6e-52ef-b146-0c8affd2e402 term=2 tally=1 +48:16.751 [INFO] TestAgent.server.raft: election won: tally=1 +48:16.751 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23720 [Leader]" +48:16.752 [INFO] TestAgent.server: cluster leadership acquired +48:16.752 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23720 +48:16.752 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:16.754 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:16.754 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:16.754 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:16.754 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:16.754 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:16.754 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fa080e20-5a6e-52ef-b146-0c8affd2e402 +48:16.754 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fa080e20-5a6e-52ef-b146-0c8affd2e402 +48:16.754 [INFO] TestAgent.server: New leader elected: payload=Node-fa080e20-5a6e-52ef-b146-0c8affd2e402 +48:16.755 [INFO] TestAgent.server: federation state anti-entropy synced +48:16.785 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:16.786 [INFO] TestAgent: Synced node info +48:16.786 [DEBUG] TestAgent: Node info in sync +48:16.857 [INFO] TestAgent: Requesting shutdown +48:16.857 [INFO] TestAgent.server: shutting down server +48:16.857 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:16.857 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:16.857 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:16.857 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:16.857 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:16.857 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:16.857 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:16.857 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:16.857 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:16.857 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:16.857 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:16.858Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:16.858Z [INFO] agent.router.manager: shutting down +48:16.858 [INFO] TestAgent: consul server down +48:16.858 [INFO] TestAgent: shutdown complete +48:16.858 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23715 network=tcp +48:16.858 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23715 network=udp +48:16.858 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23716 network=tcp +48:17.360 [INFO] TestAgent: Waiting for endpoints to shut down +48:17.360 [INFO] TestAgent: Endpoints down === RUN TestAgent_ConnectClusterIDConfig/no_cluster_ID_specified_sets_to_test_ID -2021-10-31T23:12:00.608Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:00.642Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:00.643 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ac062089-3bb3-b434-0442-bacaea95d79b Address:127.0.0.1:16580}]" -12:00.644 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ac062089-3bb3-b434-0442-bacaea95d79b.dc1 127.0.0.1 -12:00.645 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ac062089-3bb3-b434-0442-bacaea95d79b 127.0.0.1 -2021-10-31T23:12:00.645Z [INFO] agent.router: Initializing LAN area manager -12:00.645 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16575 network=udp -12:00.645 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16580 [Follower]" leader= -12:00.645 [INFO] TestAgent.server: Adding LAN server: server="Node-ac062089-3bb3-b434-0442-bacaea95d79b (Addr: tcp/127.0.0.1:16580) (DC: dc1)" -12:00.645 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ac062089-3bb3-b434-0442-bacaea95d79b.dc1 area=wan -12:00.650 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16575 network=tcp -12:00.650 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16576 network=tcp -12:00.650 [INFO] TestAgent: started state syncer -12:00.651 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16581 network=tcp -12:00.696 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:00.696 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16580 [Candidate]" term=2 -12:00.696 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:00.696 [DEBUG] TestAgent.server.raft: vote granted: from=ac062089-3bb3-b434-0442-bacaea95d79b term=2 tally=1 -12:00.696 [INFO] TestAgent.server.raft: election won: tally=1 -12:00.696 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16580 [Leader]" -12:00.696 [INFO] TestAgent.server: cluster leadership acquired -12:00.697 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16580 -12:00.697 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:00.699 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:00.699 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:00.699 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:00.699 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:00.699 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:00.699 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ac062089-3bb3-b434-0442-bacaea95d79b -12:00.699 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ac062089-3bb3-b434-0442-bacaea95d79b -12:00.699 [INFO] TestAgent.server: New leader elected: payload=Node-ac062089-3bb3-b434-0442-bacaea95d79b -12:00.700 [INFO] TestAgent.server: federation state anti-entropy synced -12:00.712 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:00.712 [INFO] TestAgent: Synced node info -12:01.066 [INFO] TestAgent: Requesting shutdown -12:01.066 [INFO] TestAgent.server: shutting down server -12:01.066 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:01.066 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:01.066 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:01.066 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:01.066 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:01.066 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:01.066 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:01.066 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:01.066 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:01.066 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:01.066 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:01.066Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:01.066Z [INFO] agent.router.manager: shutting down -12:01.066 [INFO] TestAgent: consul server down -12:01.066 [INFO] TestAgent: shutdown complete -12:01.066 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16575 network=tcp -12:01.066 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16575 network=udp -12:01.066 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16576 network=tcp -12:01.569 [INFO] TestAgent: Waiting for endpoints to shut down -12:01.569 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:17.386Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:17.411Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:17.417 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c1084737-db27-7255-c470-2e5b5f3e9d10 Address:127.0.0.1:23727}]" +48:17.418 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c1084737-db27-7255-c470-2e5b5f3e9d10.dc1 127.0.0.1 +48:17.418 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c1084737-db27-7255-c470-2e5b5f3e9d10 127.0.0.1 +2022-12-04T05:48:17.418Z [INFO] agent.router: Initializing LAN area manager +48:17.419 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23722 network=udp +48:17.419 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23727 [Follower]" leader= +48:17.419 [INFO] TestAgent.server: Adding LAN server: server="Node-c1084737-db27-7255-c470-2e5b5f3e9d10 (Addr: tcp/127.0.0.1:23727) (DC: dc1)" +48:17.419 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c1084737-db27-7255-c470-2e5b5f3e9d10.dc1 area=wan +48:17.419 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23722 network=tcp +48:17.424 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23723 network=tcp +48:17.424 [INFO] TestAgent: started state syncer +48:17.424 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23728 network=tcp +48:17.464 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:17.464 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23727 [Candidate]" term=2 +48:17.464 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:17.464 [DEBUG] TestAgent.server.raft: vote granted: from=c1084737-db27-7255-c470-2e5b5f3e9d10 term=2 tally=1 +48:17.464 [INFO] TestAgent.server.raft: election won: tally=1 +48:17.464 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23727 [Leader]" +48:17.464 [INFO] TestAgent.server: cluster leadership acquired +48:17.464 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23727 +48:17.465 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:17.467 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:17.467 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:17.467 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:17.467 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:17.467 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:17.467 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c1084737-db27-7255-c470-2e5b5f3e9d10 +48:17.467 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c1084737-db27-7255-c470-2e5b5f3e9d10 +48:17.467 [INFO] TestAgent.server: New leader elected: payload=Node-c1084737-db27-7255-c470-2e5b5f3e9d10 +48:17.467 [INFO] TestAgent.server: federation state anti-entropy synced +48:17.800 [INFO] TestAgent: Requesting shutdown +48:17.800 [INFO] TestAgent.server: shutting down server +48:17.800 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:17.800 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:17.800 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:17.800 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:17.800 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:17.800 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:17.800 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:17.800 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:17.801 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:17.801 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:17.801 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:17.801 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:17.801Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:17.801Z [INFO] agent.router.manager: shutting down +48:17.801 [INFO] TestAgent: consul server down +48:17.801 [INFO] TestAgent: shutdown complete +48:17.801 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23722 network=tcp +48:17.801 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23722 network=udp +48:17.801 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23723 network=tcp +48:18.304 [INFO] TestAgent: Waiting for endpoints to shut down +48:18.304 [INFO] TestAgent: Endpoints down === RUN TestAgent_ConnectClusterIDConfig/non-UUID_cluster_id_is_fatal -2021-10-31T23:12:01.584Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:01.643Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:01.643 [INFO] TestAgent: Requesting shutdown -12:01.643 [INFO] TestAgent: shutdown complete ---- PASS: TestAgent_ConnectClusterIDConfig (1.98s) - --- PASS: TestAgent_ConnectClusterIDConfig/default_TestAgent_has_fixed_cluster_id (0.92s) - --- PASS: TestAgent_ConnectClusterIDConfig/no_cluster_ID_specified_sets_to_test_ID (0.99s) +2022-12-04T05:48:18.338Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:18.368Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:18.372 [INFO] TestAgent: Requesting shutdown +48:18.372 [INFO] TestAgent: shutdown complete +--- PASS: TestAgent_ConnectClusterIDConfig (1.72s) + --- PASS: TestAgent_ConnectClusterIDConfig/default_TestAgent_has_fixed_cluster_id (0.70s) + --- PASS: TestAgent_ConnectClusterIDConfig/no_cluster_ID_specified_sets_to_test_ID (0.94s) --- PASS: TestAgent_ConnectClusterIDConfig/non-UUID_cluster_id_is_fatal (0.07s) === RUN TestAgent_StartStop === PAUSE TestAgent_StartStop @@ -10394,194 +10433,176 @@ === PAUSE TestAgent_AddService/service_manager === CONT TestAgent_AddService/normal === CONT TestAgent_AddService/service_manager +=== RUN TestAgent_AddService/service_manager/one_check +=== RUN TestAgent_AddService/service_manager/one_check/svcid1 +=== RUN TestAgent_AddService/service_manager/one_check/check1 +=== RUN TestAgent_AddService/service_manager/one_check/check1_ttl +=== RUN TestAgent_AddService/service_manager/multiple_checks +=== RUN TestAgent_AddService/service_manager/multiple_checks/svcid2 +=== RUN TestAgent_AddService/service_manager/multiple_checks/service:svcid2:4 +=== RUN TestAgent_AddService/service_manager/multiple_checks/check1 +=== RUN TestAgent_AddService/service_manager/multiple_checks/check-noname +=== RUN TestAgent_AddService/service_manager/multiple_checks/service:svcid2:3 +=== RUN TestAgent_AddService/service_manager/multiple_checks/check-noname_ttl +=== RUN TestAgent_AddService/service_manager/multiple_checks/service:svcid2:3_ttl +=== RUN TestAgent_AddService/service_manager/multiple_checks/service:svcid2:4_ttl +=== RUN TestAgent_AddService/service_manager/multiple_checks/check1_ttl === RUN TestAgent_AddService/normal/one_check === RUN TestAgent_AddService/normal/one_check/svcid1 === RUN TestAgent_AddService/normal/one_check/check1 === RUN TestAgent_AddService/normal/one_check/check1_ttl === RUN TestAgent_AddService/normal/multiple_checks === RUN TestAgent_AddService/normal/multiple_checks/svcid2 -=== RUN TestAgent_AddService/normal/multiple_checks/check-noname -=== RUN TestAgent_AddService/normal/multiple_checks/service:svcid2:3 === RUN TestAgent_AddService/normal/multiple_checks/service:svcid2:4 === RUN TestAgent_AddService/normal/multiple_checks/check1 +=== RUN TestAgent_AddService/normal/multiple_checks/check-noname +=== RUN TestAgent_AddService/normal/multiple_checks/service:svcid2:3 === RUN TestAgent_AddService/normal/multiple_checks/check1_ttl === RUN TestAgent_AddService/normal/multiple_checks/check-noname_ttl === RUN TestAgent_AddService/normal/multiple_checks/service:svcid2:3_ttl === RUN TestAgent_AddService/normal/multiple_checks/service:svcid2:4_ttl -=== RUN TestAgent_AddService/service_manager/one_check -=== RUN TestAgent_AddService/service_manager/one_check/svcid1 -=== RUN TestAgent_AddService/service_manager/one_check/check1 -=== RUN TestAgent_AddService/service_manager/one_check/check1_ttl -=== RUN TestAgent_AddService/service_manager/multiple_checks -=== RUN TestAgent_AddService/service_manager/multiple_checks/svcid2 -=== RUN TestAgent_AddService/service_manager/multiple_checks/check-noname -=== RUN TestAgent_AddService/service_manager/multiple_checks/service:svcid2:3 -=== RUN TestAgent_AddService/service_manager/multiple_checks/service:svcid2:4 -=== RUN TestAgent_AddService/service_manager/multiple_checks/check1 -=== RUN TestAgent_AddService/service_manager/multiple_checks/check1_ttl -=== RUN TestAgent_AddService/service_manager/multiple_checks/check-noname_ttl -=== RUN TestAgent_AddService/service_manager/multiple_checks/service:svcid2:3_ttl -=== RUN TestAgent_AddService/service_manager/multiple_checks/service:svcid2:4_ttl -2021-10-31T23:12:01.659Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:01.673Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:01.675 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:20a2d94f-22fc-da4a-16e8-d66fb07b558e Address:127.0.0.1:16594}]" -12:01.675 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16594 [Follower]" leader= -12:01.676 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -12:01.676 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2021-10-31T23:12:01.676Z [INFO] agent.router: Initializing LAN area manager -12:01.677 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16589 network=udp -12:01.677 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -12:01.677 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:16594) (DC: dc1)" -12:01.677 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16589 network=tcp -12:01.677 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16590 network=tcp -12:01.677 [INFO] TestAgent: started state syncer -12:01.677 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16595 network=tcp -12:01.745 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:01.745 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16594 [Candidate]" term=2 -12:01.746 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:01.746 [DEBUG] TestAgent.server.raft: vote granted: from=20a2d94f-22fc-da4a-16e8-d66fb07b558e term=2 tally=1 -12:01.746 [INFO] TestAgent.server.raft: election won: tally=1 -12:01.746 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16594 [Leader]" -12:01.746 [INFO] TestAgent.server: cluster leadership acquired -12:01.746 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16594 -12:01.747 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:01.748 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:01.748 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:01.748 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:01.748 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:01.748 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:01.748 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -12:01.748 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -12:01.749 [INFO] TestAgent.server: New leader elected: payload=node1 -12:01.749 [INFO] TestAgent.server: federation state anti-entropy synced -12:01.878 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:01.878 [INFO] TestAgent: Synced node info -12:02.035 [INFO] TestAgent: Requesting shutdown -12:02.035 [INFO] TestAgent.server: shutting down server -12:02.035 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:02.035 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:02.035 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:02.035 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:02.035 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:02.035 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:02.035 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:02.035 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:02.035 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:02.035 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:02.035 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:02.035Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:02.035Z [INFO] agent.router.manager: shutting down -12:02.035 [INFO] TestAgent: consul server down -12:02.035 [INFO] TestAgent: shutdown complete -12:02.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16589 network=tcp -12:02.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16589 network=udp -12:02.035 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16590 network=tcp -12:02.537 [INFO] TestAgent: Waiting for endpoints to shut down -12:02.537 [INFO] TestAgent: Endpoints down -2021-10-31T23:12:01.665Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:01.683Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:01.685 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:faf627e0-18ef-483a-dc12-70edde4da160 Address:127.0.0.1:16601}]" -12:01.685 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16601 [Follower]" leader= -12:01.690 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -12:01.691 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2021-10-31T23:12:01.691Z [INFO] agent.router: Initializing LAN area manager -12:01.691 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16596 network=udp -12:01.691 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:16601) (DC: dc1)" -12:01.691 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -12:01.691 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16596 network=tcp -12:01.692 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16597 network=tcp -12:01.692 [INFO] TestAgent: started state syncer -12:01.692 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16602 network=tcp -12:01.753 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:01.753 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16601 [Candidate]" term=2 -12:01.754 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:01.754 [DEBUG] TestAgent.server.raft: vote granted: from=faf627e0-18ef-483a-dc12-70edde4da160 term=2 tally=1 -12:01.754 [INFO] TestAgent.server.raft: election won: tally=1 -12:01.754 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16601 [Leader]" -12:01.754 [INFO] TestAgent.server: cluster leadership acquired -12:01.754 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16601 -12:01.755 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:01.756 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:01.756 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:01.756 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:01.756 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:01.756 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:01.756 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -12:01.756 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -12:01.757 [INFO] TestAgent.server: New leader elected: payload=node1 -12:01.757 [INFO] TestAgent.server: federation state anti-entropy synced -12:01.761 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:01.762 [INFO] TestAgent: Synced node info -12:01.982 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:01.982 [DEBUG] TestAgent: Node info in sync -12:01.982 [DEBUG] TestAgent: Node info in sync -12:02.078 [INFO] TestAgent: Requesting shutdown -12:02.078 [INFO] TestAgent.server: shutting down server -12:02.078 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:02.078 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:02.078 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:02.078 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:02.079 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:02.079 [DEBUG] TestAgent: Node info in sync -12:02.079 [INFO] TestAgent: Synced service: service=svcid1 -12:02.080 [INFO] TestAgent: Synced service: service=svcid2 -12:02.080 [DEBUG] TestAgent: Check in sync: check=check1 -12:02.080 [DEBUG] TestAgent: Check in sync: check=check-noname -12:02.080 [DEBUG] TestAgent: Check in sync: check=service:svcid2:3 -12:02.080 [DEBUG] TestAgent: Check in sync: check=service:svcid2:4 -12:02.080 [DEBUG] TestAgent: Node info in sync -12:02.080 [DEBUG] TestAgent: Service in sync: service=svcid1 -12:02.080 [DEBUG] TestAgent: Service in sync: service=svcid2 -12:02.080 [DEBUG] TestAgent: Check in sync: check=check1 -12:02.080 [DEBUG] TestAgent: Check in sync: check=check-noname -12:02.080 [DEBUG] TestAgent: Check in sync: check=service:svcid2:3 -12:02.080 [DEBUG] TestAgent: Check in sync: check=service:svcid2:4 -12:02.080 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:02.080 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:02.080 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:02.080 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -12:02.080 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:02.080 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -2021-10-31T23:12:02.080Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:02.080Z [INFO] agent.router.manager: shutting down -12:02.080 [INFO] TestAgent: consul server down -12:02.080 [INFO] TestAgent: shutdown complete -12:02.080 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16596 network=tcp -12:02.080 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16596 network=udp -12:02.080 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16597 network=tcp -12:02.580 [INFO] TestAgent: Waiting for endpoints to shut down -12:02.580 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:18.438Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:18.452Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:18.454 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:551555e4-5fbb-937a-5fbd-2d395353d6ba Address:127.0.0.1:23748}]" +48:18.454 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +48:18.455 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2022-12-04T05:48:18.455Z [INFO] agent.router: Initializing LAN area manager +48:18.455 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23743 network=udp +48:18.455 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23748 [Follower]" leader= +48:18.455 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:23748) (DC: dc1)" +48:18.455 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +48:18.456 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23743 network=tcp +48:18.456 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23744 network=tcp +48:18.456 [INFO] TestAgent: started state syncer +48:18.456 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23749 network=tcp +48:18.500 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:18.500 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23748 [Candidate]" term=2 +48:18.500 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:18.500 [DEBUG] TestAgent.server.raft: vote granted: from=551555e4-5fbb-937a-5fbd-2d395353d6ba term=2 tally=1 +48:18.500 [INFO] TestAgent.server.raft: election won: tally=1 +48:18.500 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23748 [Leader]" +48:18.500 [INFO] TestAgent.server: cluster leadership acquired +48:18.500 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23748 +48:18.501 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:18.503 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:18.503 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:18.503 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:18.503 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:18.503 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:18.503 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +48:18.503 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +48:18.504 [INFO] TestAgent.server: New leader elected: payload=node1 +48:18.511 [INFO] TestAgent.server: federation state anti-entropy synced +48:18.617 [INFO] TestAgent: Requesting shutdown +48:18.617 [INFO] TestAgent.server: shutting down server +48:18.617 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:18.617 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:18.617 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:18.617 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:18.617 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:18.617 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:18.617 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:18.617 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:18.617 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:18.617 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:18.617 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:18.617 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:18.617Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:18.617Z [INFO] agent.router.manager: shutting down +48:18.617 [INFO] TestAgent: consul server down +48:18.617 [INFO] TestAgent: shutdown complete +48:18.617 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23743 network=tcp +48:18.617 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23743 network=udp +48:18.617 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23744 network=tcp +48:19.120 [INFO] TestAgent: Waiting for endpoints to shut down +48:19.120 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:18.410Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:18.443Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:18.447 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6d5eb7e1-06db-d1cb-8df6-bfacb01f09a2 Address:127.0.0.1:23741}]" +48:18.448 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23741 [Follower]" leader= +48:18.456 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +48:18.457 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2022-12-04T05:48:18.457Z [INFO] agent.router: Initializing LAN area manager +48:18.457 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23736 network=udp +48:18.457 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:23741) (DC: dc1)" +48:18.457 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +48:18.457 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23736 network=tcp +48:18.458 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23737 network=tcp +48:18.458 [INFO] TestAgent: started state syncer +48:18.458 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23742 network=tcp +48:18.500 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:18.500 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23741 [Candidate]" term=2 +48:18.500 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:18.500 [DEBUG] TestAgent.server.raft: vote granted: from=6d5eb7e1-06db-d1cb-8df6-bfacb01f09a2 term=2 tally=1 +48:18.500 [INFO] TestAgent.server.raft: election won: tally=1 +48:18.500 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23741 [Leader]" +48:18.500 [INFO] TestAgent.server: cluster leadership acquired +48:18.500 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23741 +48:18.501 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:18.503 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:18.503 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:18.503 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:18.503 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:18.503 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:18.503 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +48:18.503 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +48:18.503 [INFO] TestAgent.server: New leader elected: payload=node1 +48:18.504 [INFO] TestAgent.server: federation state anti-entropy synced +48:18.592 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:18.593 [INFO] TestAgent: Synced node info +48:18.649 [INFO] TestAgent: Requesting shutdown +48:18.649 [INFO] TestAgent.server: shutting down server +48:18.649 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:18.649 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:18.649 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:18.649 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:18.649 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:18.649 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:18.649 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:18.649 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:18.649 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:18.649 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:18.649 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:18.649Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:18.649Z [INFO] agent.router.manager: shutting down +48:18.650 [INFO] TestAgent: consul server down +48:18.650 [INFO] TestAgent: shutdown complete +48:18.650 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23736 network=tcp +48:18.650 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23736 network=udp +48:18.650 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23737 network=tcp +48:19.152 [INFO] TestAgent: Waiting for endpoints to shut down +48:19.152 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_AddService (0.00s) - --- PASS: TestAgent_AddService/normal (0.89s) + --- PASS: TestAgent_AddService/service_manager (0.74s) + --- PASS: TestAgent_AddService/service_manager/one_check (0.00s) + --- PASS: TestAgent_AddService/service_manager/one_check/svcid1 (0.00s) + --- PASS: TestAgent_AddService/service_manager/one_check/check1 (0.00s) + --- PASS: TestAgent_AddService/service_manager/one_check/check1_ttl (0.00s) + --- PASS: TestAgent_AddService/service_manager/multiple_checks (0.00s) + --- PASS: TestAgent_AddService/service_manager/multiple_checks/svcid2 (0.00s) + --- PASS: TestAgent_AddService/service_manager/multiple_checks/service:svcid2:4 (0.00s) + --- PASS: TestAgent_AddService/service_manager/multiple_checks/check1 (0.00s) + --- PASS: TestAgent_AddService/service_manager/multiple_checks/check-noname (0.00s) + --- PASS: TestAgent_AddService/service_manager/multiple_checks/service:svcid2:3 (0.00s) + --- PASS: TestAgent_AddService/service_manager/multiple_checks/check-noname_ttl (0.00s) + --- PASS: TestAgent_AddService/service_manager/multiple_checks/service:svcid2:3_ttl (0.00s) + --- PASS: TestAgent_AddService/service_manager/multiple_checks/service:svcid2:4_ttl (0.00s) + --- PASS: TestAgent_AddService/service_manager/multiple_checks/check1_ttl (0.00s) + --- PASS: TestAgent_AddService/normal (0.78s) --- PASS: TestAgent_AddService/normal/one_check (0.00s) --- PASS: TestAgent_AddService/normal/one_check/svcid1 (0.00s) --- PASS: TestAgent_AddService/normal/one_check/check1 (0.00s) --- PASS: TestAgent_AddService/normal/one_check/check1_ttl (0.00s) --- PASS: TestAgent_AddService/normal/multiple_checks (0.00s) --- PASS: TestAgent_AddService/normal/multiple_checks/svcid2 (0.00s) - --- PASS: TestAgent_AddService/normal/multiple_checks/check-noname (0.00s) - --- PASS: TestAgent_AddService/normal/multiple_checks/service:svcid2:3 (0.00s) --- PASS: TestAgent_AddService/normal/multiple_checks/service:svcid2:4 (0.00s) --- PASS: TestAgent_AddService/normal/multiple_checks/check1 (0.00s) + --- PASS: TestAgent_AddService/normal/multiple_checks/check-noname (0.00s) + --- PASS: TestAgent_AddService/normal/multiple_checks/service:svcid2:3 (0.00s) --- PASS: TestAgent_AddService/normal/multiple_checks/check1_ttl (0.00s) --- PASS: TestAgent_AddService/normal/multiple_checks/check-noname_ttl (0.00s) --- PASS: TestAgent_AddService/normal/multiple_checks/service:svcid2:3_ttl (0.00s) --- PASS: TestAgent_AddService/normal/multiple_checks/service:svcid2:4_ttl (0.00s) - --- PASS: TestAgent_AddService/service_manager (0.94s) - --- PASS: TestAgent_AddService/service_manager/one_check (0.00s) - --- PASS: TestAgent_AddService/service_manager/one_check/svcid1 (0.00s) - --- PASS: TestAgent_AddService/service_manager/one_check/check1 (0.00s) - --- PASS: TestAgent_AddService/service_manager/one_check/check1_ttl (0.00s) - --- PASS: TestAgent_AddService/service_manager/multiple_checks (0.00s) - --- PASS: TestAgent_AddService/service_manager/multiple_checks/svcid2 (0.00s) - --- PASS: TestAgent_AddService/service_manager/multiple_checks/check-noname (0.00s) - --- PASS: TestAgent_AddService/service_manager/multiple_checks/service:svcid2:3 (0.00s) - --- PASS: TestAgent_AddService/service_manager/multiple_checks/service:svcid2:4 (0.00s) - --- PASS: TestAgent_AddService/service_manager/multiple_checks/check1 (0.00s) - --- PASS: TestAgent_AddService/service_manager/multiple_checks/check1_ttl (0.00s) - --- PASS: TestAgent_AddService/service_manager/multiple_checks/check-noname_ttl (0.00s) - --- PASS: TestAgent_AddService/service_manager/multiple_checks/service:svcid2:3_ttl (0.00s) - --- PASS: TestAgent_AddService/service_manager/multiple_checks/service:svcid2:4_ttl (0.00s) === RUN TestAgent_AddServices_AliasUpdateCheckNotReverted === RUN TestAgent_AddServices_AliasUpdateCheckNotReverted/normal === PAUSE TestAgent_AddServices_AliasUpdateCheckNotReverted/normal @@ -10589,174 +10610,174 @@ === PAUSE TestAgent_AddServices_AliasUpdateCheckNotReverted/service_manager === CONT TestAgent_AddServices_AliasUpdateCheckNotReverted/normal === CONT TestAgent_AddServices_AliasUpdateCheckNotReverted/service_manager -2021-10-31T23:12:02.604Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:02.632Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:02.638 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d3090322-ce93-1203-e231-acae52aee9d9 Address:127.0.0.1:16608}]" -12:02.652 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -12:02.654 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16608 [Follower]" leader= -12:02.658 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2021-10-31T23:12:02.658Z [INFO] agent.router: Initializing LAN area manager -12:02.659 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16603 network=udp -12:02.659 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16603 network=tcp -12:02.660 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:16608) (DC: dc1)" -12:02.660 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -12:02.661 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16604 network=tcp -12:02.661 [INFO] TestAgent: started state syncer -12:02.661 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16609 network=tcp -12:02.693 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:02.693 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16608 [Candidate]" term=2 -12:02.694 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:02.694 [DEBUG] TestAgent.server.raft: vote granted: from=d3090322-ce93-1203-e231-acae52aee9d9 term=2 tally=1 -12:02.694 [INFO] TestAgent.server.raft: election won: tally=1 -12:02.694 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16608 [Leader]" -12:02.694 [INFO] TestAgent.server: cluster leadership acquired -12:02.694 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16608 -12:02.695 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:02.696 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:02.696 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:02.696 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:02.696 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:02.696 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:02.697 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -12:02.697 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -12:02.697 [INFO] TestAgent.server: New leader elected: payload=node1 -12:02.704 [INFO] TestAgent.server: federation state anti-entropy synced -12:03.042 [INFO] TestAgent: Requesting shutdown -12:03.042 [INFO] TestAgent.server: shutting down server -12:03.042 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:03.042 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:03.042 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:03.042 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:03.042 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:03.042 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:03.042 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:03.042 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:03.042 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:03.042 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:03.042 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:03.042 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:03.042Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:03.042Z [INFO] agent.router.manager: shutting down -12:03.042 [INFO] TestAgent: consul server down -12:03.042 [INFO] TestAgent: shutdown complete -12:03.042 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16603 network=tcp -12:03.042 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16603 network=udp -12:03.042 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16604 network=tcp -12:03.545 [INFO] TestAgent: Waiting for endpoints to shut down -12:03.545 [INFO] TestAgent: Endpoints down -2021-10-31T23:12:02.616Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:02.693Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:02.695 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:67b2f63f-b052-9b1d-a9e7-47d072498d94 Address:127.0.0.1:16615}]" -12:02.696 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -12:02.696 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2021-10-31T23:12:02.696Z [INFO] agent.router: Initializing LAN area manager -12:02.696 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16610 network=udp -12:02.696 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16615 [Follower]" leader= -12:02.697 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:16615) (DC: dc1)" -12:02.697 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -12:02.697 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16610 network=tcp -12:02.698 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16611 network=tcp -12:02.698 [INFO] TestAgent: started state syncer -12:02.698 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16616 network=tcp -12:02.733 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:02.733 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16615 [Candidate]" term=2 -12:02.733 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:02.733 [DEBUG] TestAgent.server.raft: vote granted: from=67b2f63f-b052-9b1d-a9e7-47d072498d94 term=2 tally=1 -12:02.733 [INFO] TestAgent.server.raft: election won: tally=1 -12:02.733 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16615 [Leader]" -12:02.734 [INFO] TestAgent.server: cluster leadership acquired -12:02.734 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16615 -12:02.734 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:02.736 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:02.736 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:02.736 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:02.736 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:02.736 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:02.736 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -12:02.736 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -12:02.736 [INFO] TestAgent.server: New leader elected: payload=node1 -12:02.737 [INFO] TestAgent.server: federation state anti-entropy synced -12:02.974 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:02.974 [INFO] TestAgent: Synced node info -12:02.974 [DEBUG] TestAgent: Node info in sync -12:03.118 [INFO] TestAgent: Requesting shutdown -12:03.118 [INFO] TestAgent.server: shutting down server -12:03.118 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:03.118 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:03.118 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:03.118 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:03.118 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:03.118 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:03.118 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:03.118 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:03.118 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:03.118 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:03.118 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:03.118Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:03.118Z [INFO] agent.router.manager: shutting down -12:03.118 [INFO] TestAgent: consul server down -12:03.118 [INFO] TestAgent: shutdown complete -12:03.118 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16610 network=tcp -12:03.118 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16610 network=udp -12:03.118 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16611 network=tcp -12:03.621 [INFO] TestAgent: Waiting for endpoints to shut down -12:03.621 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:19.179Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:19.213Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:19.215 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c8444b43-fb23-07f4-2b1a-a6ebdd43f82d Address:127.0.0.1:23755}]" +48:19.215 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +48:19.216 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23755 [Follower]" leader= +48:19.224 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2022-12-04T05:48:19.224Z [INFO] agent.router: Initializing LAN area manager +48:19.224 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23750 network=udp +48:19.224 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:23755) (DC: dc1)" +48:19.224 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +48:19.225 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23750 network=tcp +48:19.225 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23751 network=tcp +48:19.225 [INFO] TestAgent: started state syncer +48:19.225 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23756 network=tcp +48:19.272 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:19.272 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23755 [Candidate]" term=2 +48:19.272 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:19.272 [DEBUG] TestAgent.server.raft: vote granted: from=c8444b43-fb23-07f4-2b1a-a6ebdd43f82d term=2 tally=1 +48:19.272 [INFO] TestAgent.server.raft: election won: tally=1 +48:19.272 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23755 [Leader]" +48:19.272 [INFO] TestAgent.server: cluster leadership acquired +48:19.272 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23755 +48:19.273 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:19.275 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:19.275 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:19.275 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:19.275 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:19.275 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:19.275 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +48:19.275 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +48:19.275 [INFO] TestAgent.server: New leader elected: payload=node1 +48:19.276 [INFO] TestAgent.server: federation state anti-entropy synced +48:19.304 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:19.304 [INFO] TestAgent: Synced node info +48:19.476 [INFO] TestAgent: Requesting shutdown +48:19.476 [INFO] TestAgent.server: shutting down server +48:19.476 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:19.476 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:19.476 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:19.476 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:19.476 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:19.476 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:19.476 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:19.476 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:19.476 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:19.476 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:19.476 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:19.477Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:19.477Z [INFO] agent.router.manager: shutting down +48:19.477 [INFO] TestAgent: consul server down +48:19.477 [INFO] TestAgent: shutdown complete +48:19.477 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23750 network=tcp +48:19.477 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23750 network=udp +48:19.477 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23751 network=tcp +48:19.980 [INFO] TestAgent: Waiting for endpoints to shut down +48:19.980 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:19.190Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:19.219Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:19.220 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:02eb41c4-3e8e-652a-9800-4e5dfbde09a4 Address:127.0.0.1:23762}]" +48:19.221 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +48:19.221 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2022-12-04T05:48:19.221Z [INFO] agent.router: Initializing LAN area manager +48:19.222 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23757 network=udp +48:19.222 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23762 [Follower]" leader= +48:19.222 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:23762) (DC: dc1)" +48:19.222 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +48:19.222 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23757 network=tcp +48:19.223 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23758 network=tcp +48:19.223 [INFO] TestAgent: started state syncer +48:19.223 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23763 network=tcp +48:19.292 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:19.292 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23762 [Candidate]" term=2 +48:19.292 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:19.292 [DEBUG] TestAgent.server.raft: vote granted: from=02eb41c4-3e8e-652a-9800-4e5dfbde09a4 term=2 tally=1 +48:19.292 [INFO] TestAgent.server.raft: election won: tally=1 +48:19.292 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23762 [Leader]" +48:19.292 [INFO] TestAgent.server: cluster leadership acquired +48:19.292 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23762 +48:19.293 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:19.294 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:19.294 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:19.294 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:19.294 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:19.294 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:19.294 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +48:19.294 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +48:19.295 [INFO] TestAgent.server: New leader elected: payload=node1 +48:19.295 [INFO] TestAgent.server: federation state anti-entropy synced +48:19.492 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:19.492 [INFO] TestAgent: Synced node info +48:19.548 [INFO] TestAgent: Requesting shutdown +48:19.548 [INFO] TestAgent.server: shutting down server +48:19.548 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:19.548 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:19.548 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:19.548 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:19.548 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:19.548 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:19.548 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:19.548 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:19.548 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:19.548 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:19.548 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:19.549Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:19.549Z [INFO] agent.router.manager: shutting down +48:19.549 [INFO] TestAgent: consul server down +48:19.549 [INFO] TestAgent: shutdown complete +48:19.549 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23757 network=tcp +48:19.549 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23757 network=udp +48:19.549 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23758 network=tcp +48:20.052 [INFO] TestAgent: Waiting for endpoints to shut down +48:20.052 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_AddServices_AliasUpdateCheckNotReverted (0.00s) - --- PASS: TestAgent_AddServices_AliasUpdateCheckNotReverted/normal (0.96s) - --- PASS: TestAgent_AddServices_AliasUpdateCheckNotReverted/service_manager (1.04s) + --- PASS: TestAgent_AddServices_AliasUpdateCheckNotReverted/normal (0.83s) + --- PASS: TestAgent_AddServices_AliasUpdateCheckNotReverted/service_manager (0.90s) === RUN TestAgent_CheckAliasRPC -[DEBUG] Service Removed {serviceAlias-8055685710231091158 {}} , err:= -[DEBUG] Service Removed {serviceAlias-6853658586754706953 {}} , err:= -[DEBUG] Service Removed {serviceAlias-2306269138787977234 {}} , err:= -[DEBUG] Service Removed {serviceAlias-7732141209168097257 {}} , err:= -[DEBUG] STARTING shutdown for TestAgent_CheckAliasRPC 2021-10-31 23:12:04.13858719 +0000 UTC m=+31.960491763 -[DEBUG] DONE shutdown for TestAgent_CheckAliasRPC 2021-10-31 23:12:04.138629251 +0000 UTC m=+31.960533836 -2021-10-31T23:12:03.657Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:03.684Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:03.691 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:27cad3ad-efc4-9a56-0fbe-e924262749f0 Address:127.0.0.1:16622}]" -12:03.692 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -12:03.692 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2021-10-31T23:12:03.692Z [INFO] agent.router: Initializing LAN area manager -12:03.692 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16617 network=udp -12:03.692 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16622 [Follower]" leader= -12:03.693 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:16622) (DC: dc1)" -12:03.693 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -12:03.693 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16617 network=tcp -12:03.698 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16618 network=tcp -12:03.699 [INFO] TestAgent: started state syncer -12:03.699 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16623 network=tcp -12:03.761 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:03.761 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16622 [Candidate]" term=2 -12:03.761 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:03.761 [DEBUG] TestAgent.server.raft: vote granted: from=27cad3ad-efc4-9a56-0fbe-e924262749f0 term=2 tally=1 -12:03.761 [INFO] TestAgent.server.raft: election won: tally=1 -12:03.761 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16622 [Leader]" -12:03.762 [INFO] TestAgent.server: cluster leadership acquired -12:03.762 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16622 -12:03.762 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:03.764 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:03.764 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:03.764 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:03.764 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:03.764 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:03.764 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -12:03.764 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -12:03.765 [INFO] TestAgent.server: New leader elected: payload=node1 -12:03.765 [INFO] TestAgent.server: federation state anti-entropy synced -12:04.013 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:04.013 [INFO] TestAgent: Synced node info -12:04.014 [INFO] TestAgent: Synced service: service=svcid1 -12:04.014 [DEBUG] TestAgent: Node info in sync -12:04.014 [DEBUG] TestAgent: Service in sync: service=svcid1 -12:04.036 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2021-10-31 23:12:04.036468929 +0000 UTC m=+31.858373510" -12:04.138 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2021-10-31 23:12:04.137866891 +0000 UTC m=+31.959771665" -12:04.138 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2021-10-31 23:12:04.13815905 +0000 UTC m=+31.960063644" -12:04.138 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2021-10-31 23:12:04.138332204 +0000 UTC m=+31.960236768" -12:04.138 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2021-10-31 23:12:04.138416075 +0000 UTC m=+31.960320664" -12:04.138 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2021-10-31 23:12:04.138478889 +0000 UTC m=+31.960383459" -12:04.138 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2021-10-31 23:12:04.13855381 +0000 UTC m=+31.960458390" -12:04.138 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2021-10-31 23:12:04.13859646 +0000 UTC m=+31.960501038" ---- PASS: TestAgent_CheckAliasRPC (0.52s) +[DEBUG] Service Removed {serviceAlias-5856107764183129028 {}} , err:= +[DEBUG] Service Removed {serviceAlias-7356640799344228282 {}} , err:= +[DEBUG] Service Removed {serviceAlias-4201956325198226881 {}} , err:= +[DEBUG] Service Removed {serviceAlias-3637745180291504015 {}} , err:= +[DEBUG] STARTING shutdown for TestAgent_CheckAliasRPC 2022-12-04 05:48:20.741217511 +0000 UTC m=+29.262801424 +[DEBUG] DONE shutdown for TestAgent_CheckAliasRPC 2022-12-04 05:48:20.741253578 +0000 UTC m=+29.262837497 +2022-12-04T05:48:20.117Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:20.163Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:20.169 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9ccc14c0-94e0-2b72-9ac1-911f53928ef5 Address:127.0.0.1:23769}]" +48:20.169 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +48:20.170 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2022-12-04T05:48:20.170Z [INFO] agent.router: Initializing LAN area manager +48:20.170 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23764 network=udp +48:20.170 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23769 [Follower]" leader= +48:20.170 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:23769) (DC: dc1)" +48:20.171 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +48:20.171 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23764 network=tcp +48:20.171 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23765 network=tcp +48:20.171 [INFO] TestAgent: started state syncer +48:20.171 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23770 network=tcp +48:20.220 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:20.220 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23769 [Candidate]" term=2 +48:20.220 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:20.220 [DEBUG] TestAgent.server.raft: vote granted: from=9ccc14c0-94e0-2b72-9ac1-911f53928ef5 term=2 tally=1 +48:20.220 [INFO] TestAgent.server.raft: election won: tally=1 +48:20.220 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23769 [Leader]" +48:20.220 [INFO] TestAgent.server: cluster leadership acquired +48:20.220 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23769 +48:20.221 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:20.223 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:20.223 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:20.223 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:20.223 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:20.223 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:20.223 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +48:20.223 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +48:20.223 [INFO] TestAgent.server: New leader elected: payload=node1 +48:20.224 [INFO] TestAgent.server: federation state anti-entropy synced +48:20.624 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:20.624 [INFO] TestAgent: Synced node info +48:20.625 [INFO] TestAgent: Synced service: service=svcid1 +48:20.625 [DEBUG] TestAgent: Node info in sync +48:20.625 [DEBUG] TestAgent: Service in sync: service=svcid1 +48:20.636 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2022-12-04 05:48:20.636859787 +0000 UTC m=+29.158443737" +48:20.740 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2022-12-04 05:48:20.740371042 +0000 UTC m=+29.261955036" +48:20.740 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2022-12-04 05:48:20.740571189 +0000 UTC m=+29.262155093" +48:20.740 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2022-12-04 05:48:20.7407171 +0000 UTC m=+29.262301008" +48:20.741 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2022-12-04 05:48:20.740855923 +0000 UTC m=+29.262439844" +48:20.741 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2022-12-04 05:48:20.741126838 +0000 UTC m=+29.262710739" +48:20.741 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2022-12-04 05:48:20.741186366 +0000 UTC m=+29.262770262" +48:20.741 [DEBUG] TestAgent: Service tag will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: tag="tag-2022-12-04 05:48:20.741224716 +0000 UTC m=+29.262808623" +--- PASS: TestAgent_CheckAliasRPC (0.69s) === RUN TestAgent_AddServiceNoExec === RUN TestAgent_AddServiceNoExec/normal === PAUSE TestAgent_AddServiceNoExec/normal @@ -10767,122 +10788,119 @@ [WARN] freeport: 6 out of 7 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:04.211Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:04.260Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:04.266 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:74a8ab67-dbfc-fb9b-7d55-535af633dc46 Address:127.0.0.1:16636}]" -12:04.267 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -12:04.267 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2021-10-31T23:12:04.267Z [INFO] agent.router: Initializing LAN area manager -12:04.268 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16631 network=udp -12:04.268 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16636 [Follower]" leader= -12:04.268 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:16636) (DC: dc1)" -12:04.268 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -12:04.268 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16631 network=tcp -12:04.269 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16632 network=tcp -12:04.269 [INFO] TestAgent: started state syncer -12:04.269 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16637 network=tcp -12:04.334 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:04.334 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16636 [Candidate]" term=2 -12:04.334 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:04.334 [DEBUG] TestAgent.server.raft: vote granted: from=74a8ab67-dbfc-fb9b-7d55-535af633dc46 term=2 tally=1 -12:04.334 [INFO] TestAgent.server.raft: election won: tally=1 -12:04.334 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16636 [Leader]" -12:04.334 [INFO] TestAgent.server: cluster leadership acquired -12:04.334 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16636 -12:04.335 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:04.336 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:04.336 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:04.336 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:04.336 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:04.336 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:04.336 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -12:04.336 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -12:04.337 [INFO] TestAgent.server: New leader elected: payload=node1 -12:04.337 [INFO] TestAgent.server: federation state anti-entropy synced -12:04.418 [INFO] TestAgent: Requesting shutdown -12:04.418 [INFO] TestAgent.server: shutting down server -12:04.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:04.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:04.418 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:04.418 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:04.418 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:04.418 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:04.418 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:04.418 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:04.418 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:04.418 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:04.418 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:04.418 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:04.418Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:04.418Z [INFO] agent.router.manager: shutting down -12:04.419 [INFO] TestAgent: consul server down -12:04.419 [INFO] TestAgent: shutdown complete -12:04.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16631 network=tcp -12:04.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16631 network=udp -12:04.419 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16632 network=tcp -12:04.921 [INFO] TestAgent: Waiting for endpoints to shut down -12:04.921 [INFO] TestAgent: Endpoints down -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:04.170Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:04.252Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:04.259 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d7a13ee7-3d2e-bc14-965f-a0170b736064 Address:127.0.0.1:16629}]" -12:04.259 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -12:04.260 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2021-10-31T23:12:04.260Z [INFO] agent.router: Initializing LAN area manager -12:04.260 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16624 network=udp -12:04.260 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16629 [Follower]" leader= -12:04.260 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:16629) (DC: dc1)" -12:04.260 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -12:04.260 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16624 network=tcp -12:04.261 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16625 network=tcp -12:04.261 [INFO] TestAgent: started state syncer -12:04.261 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16630 network=tcp -12:04.297 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:04.297 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16629 [Candidate]" term=2 -12:04.298 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:04.298 [DEBUG] TestAgent.server.raft: vote granted: from=d7a13ee7-3d2e-bc14-965f-a0170b736064 term=2 tally=1 -12:04.298 [INFO] TestAgent.server.raft: election won: tally=1 -12:04.298 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16629 [Leader]" -12:04.298 [INFO] TestAgent.server: cluster leadership acquired -12:04.298 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16629 -12:04.299 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:04.301 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:04.301 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:04.301 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:04.301 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:04.301 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:04.301 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -12:04.301 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -12:04.302 [INFO] TestAgent.server: federation state anti-entropy synced -12:04.307 [INFO] TestAgent.server: New leader elected: payload=node1 -12:04.562 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:04.562 [INFO] TestAgent: Synced node info -12:04.562 [DEBUG] TestAgent: Node info in sync -12:04.666 [INFO] TestAgent: Requesting shutdown -12:04.666 [INFO] TestAgent.server: shutting down server -12:04.666 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:04.666 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:04.666 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:04.666 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:04.666 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:04.666 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:04.666 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:04.666 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:04.666 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:04.666 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:04.666 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:04.666Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:04.666Z [INFO] agent.router.manager: shutting down -12:04.666 [INFO] TestAgent: consul server down -12:04.666 [INFO] TestAgent: shutdown complete -12:04.666 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16624 network=tcp -12:04.666 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16624 network=udp -12:04.666 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16625 network=tcp -12:05.169 [INFO] TestAgent: Waiting for endpoints to shut down -12:05.169 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:20.759Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:20.774Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:20.776 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:38fb0064-7c47-c114-66d9-fbde6974320e Address:127.0.0.1:23783}]" +48:20.776 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23783 [Follower]" leader= +48:20.777 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +48:20.777 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2022-12-04T05:48:20.777Z [INFO] agent.router: Initializing LAN area manager +48:20.778 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23778 network=udp +48:20.778 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:23783) (DC: dc1)" +48:20.778 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +48:20.778 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23778 network=tcp +48:20.779 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23779 network=tcp +48:20.779 [INFO] TestAgent: started state syncer +48:20.779 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23784 network=tcp +48:20.828 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:20.828 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23783 [Candidate]" term=2 +48:20.828 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:20.828 [DEBUG] TestAgent.server.raft: vote granted: from=38fb0064-7c47-c114-66d9-fbde6974320e term=2 tally=1 +48:20.828 [INFO] TestAgent.server.raft: election won: tally=1 +48:20.828 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23783 [Leader]" +48:20.828 [INFO] TestAgent.server: cluster leadership acquired +48:20.829 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23783 +48:20.829 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:20.831 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:20.831 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:20.831 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:20.831 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:20.831 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:20.831 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +48:20.831 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +48:20.831 [INFO] TestAgent.server: New leader elected: payload=node1 +48:20.832 [INFO] TestAgent.server: federation state anti-entropy synced +48:20.877 [INFO] TestAgent: Requesting shutdown +48:20.877 [INFO] TestAgent.server: shutting down server +48:20.877 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:20.877 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:20.877 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:20.877 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:20.877 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:20.877 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:20.877 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:20.877 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:20.877 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:20.877 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:20.877 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:20.878 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:20.878Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:20.878Z [INFO] agent.router.manager: shutting down +48:20.878 [INFO] TestAgent: consul server down +48:20.878 [INFO] TestAgent: shutdown complete +48:20.878 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23778 network=tcp +48:20.878 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23778 network=udp +48:20.878 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23779 network=tcp +48:21.380 [INFO] TestAgent: Waiting for endpoints to shut down +48:21.380 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:20.768Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:20.795Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:20.801 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:964d2167-185a-9fca-6ca7-6881525f87b7 Address:127.0.0.1:23776}]" +48:20.802 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +48:20.802 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2022-12-04T05:48:20.802Z [INFO] agent.router: Initializing LAN area manager +48:20.803 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23771 network=udp +48:20.803 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23776 [Follower]" leader= +48:20.803 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:23776) (DC: dc1)" +48:20.803 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +48:20.803 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23771 network=tcp +48:20.804 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23772 network=tcp +48:20.804 [INFO] TestAgent: started state syncer +48:20.804 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23777 network=tcp +48:20.860 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:20.860 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23776 [Candidate]" term=2 +48:20.860 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:20.860 [DEBUG] TestAgent.server.raft: vote granted: from=964d2167-185a-9fca-6ca7-6881525f87b7 term=2 tally=1 +48:20.860 [INFO] TestAgent.server.raft: election won: tally=1 +48:20.860 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23776 [Leader]" +48:20.860 [INFO] TestAgent.server: cluster leadership acquired +48:20.860 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23776 +48:20.881 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:20.883 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:20.883 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:20.883 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:20.883 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:20.883 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:20.883 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +48:20.883 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +48:20.883 [INFO] TestAgent.server: New leader elected: payload=node1 +48:20.888 [INFO] TestAgent.server: federation state anti-entropy synced +48:20.892 [INFO] TestAgent: Requesting shutdown +48:20.892 [INFO] TestAgent.server: shutting down server +48:20.892 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:20.893 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:20.893 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:20.893 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:20.893 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:20.893 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:20.893 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:20.893 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:20.893 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:20.893 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:20.893 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:20.893 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:20.893Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:20.893Z [INFO] agent.router.manager: shutting down +48:20.893 [INFO] TestAgent: consul server down +48:20.893 [INFO] TestAgent: shutdown complete +48:20.893 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23771 network=tcp +48:20.893 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23771 network=udp +48:20.893 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23772 network=tcp +48:21.396 [INFO] TestAgent: Waiting for endpoints to shut down +48:21.396 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_AddServiceNoExec (0.00s) - --- PASS: TestAgent_AddServiceNoExec/service_manager (0.78s) - --- PASS: TestAgent_AddServiceNoExec/normal (1.03s) + --- PASS: TestAgent_AddServiceNoExec/service_manager (0.64s) + --- PASS: TestAgent_AddServiceNoExec/normal (0.65s) === RUN TestAgent_AddServiceNoRemoteExec === RUN TestAgent_AddServiceNoRemoteExec/normal === PAUSE TestAgent_AddServiceNoRemoteExec/normal @@ -10890,374 +10908,375 @@ === PAUSE TestAgent_AddServiceNoRemoteExec/service_manager === CONT TestAgent_AddServiceNoRemoteExec/normal === CONT TestAgent_AddServiceNoRemoteExec/service_manager -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:05.197Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:05.246Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:05.248 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:724a5841-1a34-65e5-9005-c01383c97221 Address:127.0.0.1:16650}]" -12:05.248 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -12:05.282 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16650 [Follower]" leader= -12:05.287 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2021-10-31T23:12:05.287Z [INFO] agent.router: Initializing LAN area manager -12:05.287 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16645 network=udp -12:05.288 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:16650) (DC: dc1)" -12:05.288 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -12:05.288 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16645 network=tcp -12:05.289 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16646 network=tcp -12:05.289 [INFO] TestAgent: started state syncer -12:05.289 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16651 network=tcp -12:05.325 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:05.325 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16650 [Candidate]" term=2 -12:05.326 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:05.326 [DEBUG] TestAgent.server.raft: vote granted: from=724a5841-1a34-65e5-9005-c01383c97221 term=2 tally=1 -12:05.326 [INFO] TestAgent.server.raft: election won: tally=1 -12:05.326 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16650 [Leader]" -12:05.326 [INFO] TestAgent.server: cluster leadership acquired -12:05.326 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16650 -12:05.327 [INFO] TestAgent.server: New leader elected: payload=node1 -12:05.327 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:05.328 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:05.328 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:05.328 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:05.328 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:05.328 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:05.328 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -12:05.328 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -12:05.329 [INFO] TestAgent.server: federation state anti-entropy synced -12:05.410 [INFO] TestAgent: Requesting shutdown -12:05.410 [INFO] TestAgent.server: shutting down server -12:05.410 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:05.410 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:05.410 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:05.410 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:05.410 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:05.410 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:05.410 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:05.410 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:05.410 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:05.410 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:05.410 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:05.410 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:05.410Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:05.410Z [INFO] agent.router.manager: shutting down -12:05.410 [INFO] TestAgent: consul server down -12:05.411 [INFO] TestAgent: shutdown complete -12:05.411 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16645 network=tcp -12:05.411 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16645 network=udp -12:05.411 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16646 network=tcp -12:05.913 [INFO] TestAgent: Waiting for endpoints to shut down -12:05.913 [INFO] TestAgent: Endpoints down -2021-10-31T23:12:05.212Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:05.291Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:05.303 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:72f9b44f-b309-a358-50d8-869b9caab305 Address:127.0.0.1:16643}]" -12:05.304 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -12:05.305 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2021-10-31T23:12:05.305Z [INFO] agent.router: Initializing LAN area manager -12:05.305 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16643 [Follower]" leader= -12:05.306 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:16643) (DC: dc1)" -12:05.306 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -12:05.306 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16638 network=tcp -12:05.306 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16638 network=udp -12:05.322 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16639 network=tcp -12:05.322 [INFO] TestAgent: started state syncer -12:05.322 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16644 network=tcp -12:05.349 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:05.350 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16643 [Candidate]" term=2 -12:05.350 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:05.350 [DEBUG] TestAgent.server.raft: vote granted: from=72f9b44f-b309-a358-50d8-869b9caab305 term=2 tally=1 -12:05.350 [INFO] TestAgent.server.raft: election won: tally=1 -12:05.350 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16643 [Leader]" -12:05.350 [INFO] TestAgent.server: cluster leadership acquired -12:05.350 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16643 -12:05.351 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:05.352 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:05.352 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:05.352 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:05.352 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:05.352 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:05.353 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -12:05.353 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -12:05.353 [INFO] TestAgent.server: New leader elected: payload=node1 -12:05.358 [INFO] TestAgent.server: federation state anti-entropy synced -12:05.414 [INFO] TestAgent: Requesting shutdown -12:05.414 [INFO] TestAgent.server: shutting down server -12:05.414 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:05.414 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:05.414 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:05.414 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:05.414 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:05.414 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:05.414 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:05.414 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:05.414 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:05.414 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:05.414 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:05.414 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:05.414Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:05.414Z [INFO] agent.router.manager: shutting down -12:05.415 [INFO] TestAgent: consul server down -12:05.415 [INFO] TestAgent: shutdown complete -12:05.415 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16638 network=tcp -12:05.415 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16638 network=udp -12:05.415 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16639 network=tcp -12:05.917 [INFO] TestAgent: Waiting for endpoints to shut down -12:05.917 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:21.426Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:21.451Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:21.457 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b39458ac-e00a-d3b8-72fc-086c5821683e Address:127.0.0.1:23790}]" +48:21.457 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23790 [Follower]" leader= +48:21.458 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +48:21.458 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2022-12-04T05:48:21.458Z [INFO] agent.router: Initializing LAN area manager +48:21.459 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23785 network=udp +48:21.459 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:23790) (DC: dc1)" +48:21.459 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +48:21.459 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23785 network=tcp +48:21.460 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23786 network=tcp +48:21.460 [INFO] TestAgent: started state syncer +48:21.460 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23791 network=tcp +48:21.528 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:21.528 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23790 [Candidate]" term=2 +48:21.528 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:21.528 [DEBUG] TestAgent.server.raft: vote granted: from=b39458ac-e00a-d3b8-72fc-086c5821683e term=2 tally=1 +48:21.528 [INFO] TestAgent.server.raft: election won: tally=1 +48:21.528 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23790 [Leader]" +48:21.528 [INFO] TestAgent.server: cluster leadership acquired +48:21.528 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23790 +48:21.529 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:21.531 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:21.531 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:21.531 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:21.531 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:21.531 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:21.531 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +48:21.531 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +48:21.531 [INFO] TestAgent.server: New leader elected: payload=node1 +48:21.531 [INFO] TestAgent.server: federation state anti-entropy synced +48:21.540 [INFO] TestAgent: Requesting shutdown +48:21.540 [INFO] TestAgent.server: shutting down server +48:21.540 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:21.540 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:21.540 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:21.540 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:21.540 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:21.540 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:21.540 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:21.540 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:21.540 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:21.541 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:21.541 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:21.541 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:21.541Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:21.541Z [INFO] agent.router.manager: shutting down +48:21.541 [INFO] TestAgent: consul server down +48:21.541 [INFO] TestAgent: shutdown complete +48:21.541 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23785 network=tcp +48:21.541 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23785 network=udp +48:21.541 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23786 network=tcp +48:22.044 [INFO] TestAgent: Waiting for endpoints to shut down +48:22.044 [INFO] TestAgent: Endpoints down +[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:48:21.419Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:21.457Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:21.459 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8f78d8cc-ebac-8ba6-9c51-38c7b8437bc0 Address:127.0.0.1:23797}]" +48:21.459 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +48:21.460 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2022-12-04T05:48:21.460Z [INFO] agent.router: Initializing LAN area manager +48:21.460 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23797 [Follower]" leader= +48:21.464 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23792 network=udp +48:21.465 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:23797) (DC: dc1)" +48:21.465 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +48:21.465 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23792 network=tcp +48:21.465 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23793 network=tcp +48:21.465 [INFO] TestAgent: started state syncer +48:21.465 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23798 network=tcp +48:21.516 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:21.516 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23797 [Candidate]" term=2 +48:21.516 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:21.516 [DEBUG] TestAgent.server.raft: vote granted: from=8f78d8cc-ebac-8ba6-9c51-38c7b8437bc0 term=2 tally=1 +48:21.516 [INFO] TestAgent.server.raft: election won: tally=1 +48:21.516 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23797 [Leader]" +48:21.516 [INFO] TestAgent.server: cluster leadership acquired +48:21.517 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23797 +48:21.517 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:21.519 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:21.519 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:21.519 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:21.519 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:21.519 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:21.519 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +48:21.519 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +48:21.519 [INFO] TestAgent.server: New leader elected: payload=node1 +48:21.520 [INFO] TestAgent.server: federation state anti-entropy synced +48:21.644 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:21.645 [INFO] TestAgent: Synced node info +48:21.645 [DEBUG] TestAgent: Node info in sync +48:21.716 [INFO] TestAgent: Requesting shutdown +48:21.716 [INFO] TestAgent.server: shutting down server +48:21.716 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:21.716 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:21.716 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:21.716 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:21.716 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:21.717 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:21.717 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:21.717 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:21.717 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:21.717 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:21.717 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:21.717Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:21.717Z [INFO] agent.router.manager: shutting down +48:21.717 [INFO] TestAgent: consul server down +48:21.717 [INFO] TestAgent: shutdown complete +48:21.717 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23792 network=tcp +48:21.717 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23792 network=udp +48:21.717 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23793 network=tcp +48:22.219 [INFO] TestAgent: Waiting for endpoints to shut down +48:22.219 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_AddServiceNoRemoteExec (0.00s) - --- PASS: TestAgent_AddServiceNoRemoteExec/service_manager (0.74s) - --- PASS: TestAgent_AddServiceNoRemoteExec/normal (0.75s) + --- PASS: TestAgent_AddServiceNoRemoteExec/normal (0.65s) + --- PASS: TestAgent_AddServiceNoRemoteExec/service_manager (0.82s) === RUN TestCacheRateLimit agent_test.go:749: DM-skipped --- SKIP: TestCacheRateLimit (0.00s) === RUN TestAddServiceIPv4TaggedDefault -[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:05.944Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:05.983Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:05.984 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fe13c5ed-9dd7-abd2-50df-68dc44302742 Address:127.0.0.1:16657}]" -12:05.985 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fe13c5ed-9dd7-abd2-50df-68dc44302742.dc1 127.0.0.1 -12:05.985 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16657 [Follower]" leader= -12:05.990 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fe13c5ed-9dd7-abd2-50df-68dc44302742 127.0.0.1 -2021-10-31T23:12:05.990Z [INFO] agent.router: Initializing LAN area manager -12:05.990 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16652 network=udp -12:05.990 [INFO] TestAgent.server: Adding LAN server: server="Node-fe13c5ed-9dd7-abd2-50df-68dc44302742 (Addr: tcp/127.0.0.1:16657) (DC: dc1)" -12:05.990 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fe13c5ed-9dd7-abd2-50df-68dc44302742.dc1 area=wan -12:05.990 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16652 network=tcp -12:05.991 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16653 network=tcp -12:05.991 [INFO] TestAgent: started state syncer -12:05.991 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16658 network=tcp -12:06.029 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:06.030 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16657 [Candidate]" term=2 -12:06.030 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:06.030 [DEBUG] TestAgent.server.raft: vote granted: from=fe13c5ed-9dd7-abd2-50df-68dc44302742 term=2 tally=1 -12:06.030 [INFO] TestAgent.server.raft: election won: tally=1 -12:06.030 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16657 [Leader]" -12:06.030 [INFO] TestAgent.server: cluster leadership acquired -12:06.030 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16657 -12:06.031 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:06.032 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:06.032 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:06.032 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:06.032 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:06.032 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:06.032 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe13c5ed-9dd7-abd2-50df-68dc44302742 -12:06.032 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fe13c5ed-9dd7-abd2-50df-68dc44302742 -12:06.033 [INFO] TestAgent.server: New leader elected: payload=Node-fe13c5ed-9dd7-abd2-50df-68dc44302742 -12:06.033 [INFO] TestAgent.server: federation state anti-entropy synced -12:06.162 [WARN] TestAgent: Service name will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: service=my_service -12:06.162 [INFO] TestAgent: Requesting shutdown -12:06.162 [INFO] TestAgent.server: shutting down server -12:06.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:06.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:06.162 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:06.162 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:06.162 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:06.162 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:06.162 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:06.162 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:06.162 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:06.162 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:06.162 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:06.162 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:06.162Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:06.162Z [INFO] agent.router.manager: shutting down -12:06.162 [INFO] TestAgent: consul server down -12:06.162 [INFO] TestAgent: shutdown complete -12:06.162 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16652 network=tcp -12:06.162 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16652 network=udp -12:06.162 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16653 network=tcp -12:06.665 [INFO] TestAgent: Waiting for endpoints to shut down -12:06.665 [INFO] TestAgent: Endpoints down ---- PASS: TestAddServiceIPv4TaggedDefault (0.75s) +2022-12-04T05:48:22.256Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:22.284Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:22.298 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6aac6468-61db-5328-578c-c7cdffc2011a Address:127.0.0.1:23804}]" +48:22.298 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6aac6468-61db-5328-578c-c7cdffc2011a.dc1 127.0.0.1 +48:22.299 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6aac6468-61db-5328-578c-c7cdffc2011a 127.0.0.1 +2022-12-04T05:48:22.299Z [INFO] agent.router: Initializing LAN area manager +48:22.299 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23799 network=udp +48:22.299 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23804 [Follower]" leader= +48:22.300 [INFO] TestAgent.server: Adding LAN server: server="Node-6aac6468-61db-5328-578c-c7cdffc2011a (Addr: tcp/127.0.0.1:23804) (DC: dc1)" +48:22.300 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6aac6468-61db-5328-578c-c7cdffc2011a.dc1 area=wan +48:22.300 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23799 network=tcp +48:22.300 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23800 network=tcp +48:22.301 [INFO] TestAgent: started state syncer +48:22.301 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23805 network=tcp +48:22.360 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:22.360 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23804 [Candidate]" term=2 +48:22.360 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:22.360 [DEBUG] TestAgent.server.raft: vote granted: from=6aac6468-61db-5328-578c-c7cdffc2011a term=2 tally=1 +48:22.360 [INFO] TestAgent.server.raft: election won: tally=1 +48:22.360 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23804 [Leader]" +48:22.360 [INFO] TestAgent.server: cluster leadership acquired +48:22.360 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23804 +48:22.361 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:22.363 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:22.363 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:22.363 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:22.363 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:22.363 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:22.363 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6aac6468-61db-5328-578c-c7cdffc2011a +48:22.363 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6aac6468-61db-5328-578c-c7cdffc2011a +48:22.363 [INFO] TestAgent.server: New leader elected: payload=Node-6aac6468-61db-5328-578c-c7cdffc2011a +48:22.364 [INFO] TestAgent.server: federation state anti-entropy synced +48:22.597 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:22.597 [INFO] TestAgent: Synced node info +48:22.598 [WARN] TestAgent: Service name will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: service=my_service +48:22.598 [INFO] TestAgent: Requesting shutdown +48:22.598 [INFO] TestAgent.server: shutting down server +48:22.598 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:22.598 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:22.598 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:22.598 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:22.598 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:22.598 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:22.598 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:22.598 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:22.598 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:22.598 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:22.598 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:22.598Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:22.598Z [INFO] agent.router.manager: shutting down +48:22.599 [INFO] TestAgent: consul server down +48:22.599 [INFO] TestAgent: shutdown complete +48:22.599 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23799 network=tcp +48:22.599 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23799 network=udp +48:22.599 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23800 network=tcp +48:23.099 [INFO] TestAgent: Waiting for endpoints to shut down +48:23.099 [INFO] TestAgent: Endpoints down +--- PASS: TestAddServiceIPv4TaggedDefault (0.88s) === RUN TestAddServiceIPv6TaggedDefault [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:06.700Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:06.729Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:06.739 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:39475edd-8569-b6c9-ac48-56ea4f39c3ef Address:127.0.0.1:16664}]" -12:06.740 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-39475edd-8569-b6c9-ac48-56ea4f39c3ef.dc1 127.0.0.1 -12:06.740 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-39475edd-8569-b6c9-ac48-56ea4f39c3ef 127.0.0.1 -2021-10-31T23:12:06.740Z [INFO] agent.router: Initializing LAN area manager -12:06.741 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16659 network=udp -12:06.741 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16664 [Follower]" leader= -12:06.741 [INFO] TestAgent.server: Adding LAN server: server="Node-39475edd-8569-b6c9-ac48-56ea4f39c3ef (Addr: tcp/127.0.0.1:16664) (DC: dc1)" -12:06.741 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-39475edd-8569-b6c9-ac48-56ea4f39c3ef.dc1 area=wan -12:06.741 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16659 network=tcp -12:06.750 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16660 network=tcp -12:06.750 [INFO] TestAgent: started state syncer -12:06.750 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16665 network=tcp -12:06.805 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:06.805 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16664 [Candidate]" term=2 -12:06.806 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:06.806 [DEBUG] TestAgent.server.raft: vote granted: from=39475edd-8569-b6c9-ac48-56ea4f39c3ef term=2 tally=1 -12:06.806 [INFO] TestAgent.server.raft: election won: tally=1 -12:06.806 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16664 [Leader]" -12:06.806 [INFO] TestAgent.server: cluster leadership acquired -12:06.806 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16664 -12:06.807 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:06.808 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:06.808 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:06.808 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:06.808 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:06.808 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:06.808 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-39475edd-8569-b6c9-ac48-56ea4f39c3ef -12:06.808 [INFO] TestAgent.server: member joined, marking health alive: member=Node-39475edd-8569-b6c9-ac48-56ea4f39c3ef -12:06.809 [INFO] TestAgent.server: New leader elected: payload=Node-39475edd-8569-b6c9-ac48-56ea4f39c3ef -12:06.809 [INFO] TestAgent.server: federation state anti-entropy synced -12:06.830 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:06.830 [INFO] TestAgent: Synced node info -12:06.830 [DEBUG] TestAgent: Node info in sync -12:07.206 [WARN] TestAgent: Service name will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: service=my_service -12:07.206 [INFO] TestAgent: Requesting shutdown -12:07.206 [INFO] TestAgent.server: shutting down server -12:07.206 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:07.206 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:07.206 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:07.206 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:07.206 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:07.206 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:07.206 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:07.206 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:07.206 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:07.206 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:07.206 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:07.206Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:07.206Z [INFO] agent.router.manager: shutting down -12:07.206 [INFO] TestAgent: consul server down -12:07.206 [INFO] TestAgent: shutdown complete -12:07.206 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16659 network=tcp -12:07.206 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16659 network=udp -12:07.206 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16660 network=tcp -12:07.709 [INFO] TestAgent: Waiting for endpoints to shut down -12:07.709 [INFO] TestAgent: Endpoints down ---- PASS: TestAddServiceIPv6TaggedDefault (1.04s) +2022-12-04T05:48:23.114Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:23.127Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:23.128 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8872ce2d-9ca2-e9b0-88fa-aeb716351fb5 Address:127.0.0.1:23811}]" +48:23.129 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8872ce2d-9ca2-e9b0-88fa-aeb716351fb5.dc1 127.0.0.1 +48:23.129 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8872ce2d-9ca2-e9b0-88fa-aeb716351fb5 127.0.0.1 +2022-12-04T05:48:23.130Z [INFO] agent.router: Initializing LAN area manager +48:23.130 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23806 network=udp +48:23.130 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23811 [Follower]" leader= +48:23.130 [INFO] TestAgent.server: Adding LAN server: server="Node-8872ce2d-9ca2-e9b0-88fa-aeb716351fb5 (Addr: tcp/127.0.0.1:23811) (DC: dc1)" +48:23.130 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8872ce2d-9ca2-e9b0-88fa-aeb716351fb5.dc1 area=wan +48:23.130 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23806 network=tcp +48:23.131 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23807 network=tcp +48:23.131 [INFO] TestAgent: started state syncer +48:23.131 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23812 network=tcp +48:23.172 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:23.172 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23811 [Candidate]" term=2 +48:23.172 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:23.172 [DEBUG] TestAgent.server.raft: vote granted: from=8872ce2d-9ca2-e9b0-88fa-aeb716351fb5 term=2 tally=1 +48:23.172 [INFO] TestAgent.server.raft: election won: tally=1 +48:23.172 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23811 [Leader]" +48:23.172 [INFO] TestAgent.server: cluster leadership acquired +48:23.172 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23811 +48:23.173 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:23.175 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:23.175 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:23.175 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:23.175 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:23.175 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:23.175 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8872ce2d-9ca2-e9b0-88fa-aeb716351fb5 +48:23.175 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8872ce2d-9ca2-e9b0-88fa-aeb716351fb5 +48:23.175 [INFO] TestAgent.server: New leader elected: payload=Node-8872ce2d-9ca2-e9b0-88fa-aeb716351fb5 +48:23.175 [INFO] TestAgent.server: federation state anti-entropy synced +48:23.216 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:23.216 [INFO] TestAgent: Synced node info +48:23.456 [WARN] TestAgent: Service name will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: service=my_service +48:23.456 [INFO] TestAgent: Requesting shutdown +48:23.456 [INFO] TestAgent.server: shutting down server +48:23.456 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:23.456 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:23.456 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:23.456 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:23.456 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:23.456 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:23.456 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:23.456 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:23.456 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:23.456 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:23.457 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:23.457Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:23.457Z [INFO] agent.router.manager: shutting down +48:23.457 [INFO] TestAgent: consul server down +48:23.457 [INFO] TestAgent: shutdown complete +48:23.457 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23806 network=tcp +48:23.457 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23806 network=udp +48:23.457 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23807 network=tcp +48:23.957 [INFO] TestAgent: Waiting for endpoints to shut down +48:23.957 [INFO] TestAgent: Endpoints down +--- PASS: TestAddServiceIPv6TaggedDefault (0.86s) === RUN TestAddServiceIPv4TaggedSet [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:07.757Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:07.828Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:07.835 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1b46511d-8f48-885a-4df8-bb451dd16ab5 Address:127.0.0.1:16671}]" -12:07.835 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1b46511d-8f48-885a-4df8-bb451dd16ab5.dc1 127.0.0.1 -12:07.836 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1b46511d-8f48-885a-4df8-bb451dd16ab5 127.0.0.1 -2021-10-31T23:12:07.836Z [INFO] agent.router: Initializing LAN area manager -12:07.836 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16666 network=udp -12:07.836 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16671 [Follower]" leader= -12:07.837 [INFO] TestAgent.server: Adding LAN server: server="Node-1b46511d-8f48-885a-4df8-bb451dd16ab5 (Addr: tcp/127.0.0.1:16671) (DC: dc1)" -12:07.837 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1b46511d-8f48-885a-4df8-bb451dd16ab5.dc1 area=wan -12:07.837 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16666 network=tcp -12:07.846 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16667 network=tcp -12:07.846 [INFO] TestAgent: started state syncer -12:07.846 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16672 network=tcp -12:07.882 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:07.882 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16671 [Candidate]" term=2 -12:07.882 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:07.882 [DEBUG] TestAgent.server.raft: vote granted: from=1b46511d-8f48-885a-4df8-bb451dd16ab5 term=2 tally=1 -12:07.882 [INFO] TestAgent.server.raft: election won: tally=1 -12:07.882 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16671 [Leader]" -12:07.882 [INFO] TestAgent.server: cluster leadership acquired -12:07.882 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16671 -12:07.883 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:07.885 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:07.885 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:07.885 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:07.885 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:07.885 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:07.885 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1b46511d-8f48-885a-4df8-bb451dd16ab5 -12:07.885 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1b46511d-8f48-885a-4df8-bb451dd16ab5 -12:07.885 [INFO] TestAgent.server: New leader elected: payload=Node-1b46511d-8f48-885a-4df8-bb451dd16ab5 -12:07.885 [INFO] TestAgent.server: federation state anti-entropy synced -12:08.058 [WARN] TestAgent: Service name will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: service=my_service -12:08.058 [INFO] TestAgent: Requesting shutdown -12:08.058 [INFO] TestAgent.server: shutting down server -12:08.058 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:08.058 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:08.058 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:08.058 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:08.058 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:08.058 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:08.058 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:08.058 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:08.058 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:08.058 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:08.058 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:08.058 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:08.058Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:08.058Z [INFO] agent.router.manager: shutting down -12:08.058 [INFO] TestAgent: consul server down -12:08.058 [INFO] TestAgent: shutdown complete -12:08.058 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16666 network=tcp -12:08.058 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16666 network=udp -12:08.058 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16667 network=tcp -12:08.561 [INFO] TestAgent: Waiting for endpoints to shut down -12:08.561 [INFO] TestAgent: Endpoints down ---- PASS: TestAddServiceIPv4TaggedSet (0.85s) +2022-12-04T05:48:23.983Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:24.016Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:24.018 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9e4f11db-cd69-ea1c-ad2f-2a1770544225 Address:127.0.0.1:23818}]" +48:24.018 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23818 [Follower]" leader= +48:24.018 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9e4f11db-cd69-ea1c-ad2f-2a1770544225.dc1 127.0.0.1 +48:24.019 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9e4f11db-cd69-ea1c-ad2f-2a1770544225 127.0.0.1 +2022-12-04T05:48:24.019Z [INFO] agent.router: Initializing LAN area manager +48:24.019 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23813 network=udp +48:24.019 [INFO] TestAgent.server: Adding LAN server: server="Node-9e4f11db-cd69-ea1c-ad2f-2a1770544225 (Addr: tcp/127.0.0.1:23818) (DC: dc1)" +48:24.019 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9e4f11db-cd69-ea1c-ad2f-2a1770544225.dc1 area=wan +48:24.020 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23813 network=tcp +48:24.020 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23814 network=tcp +48:24.020 [INFO] TestAgent: started state syncer +48:24.020 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23819 network=tcp +48:24.072 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:24.072 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23818 [Candidate]" term=2 +48:24.072 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:24.072 [DEBUG] TestAgent.server.raft: vote granted: from=9e4f11db-cd69-ea1c-ad2f-2a1770544225 term=2 tally=1 +48:24.072 [INFO] TestAgent.server.raft: election won: tally=1 +48:24.072 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23818 [Leader]" +48:24.072 [INFO] TestAgent.server: cluster leadership acquired +48:24.072 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23818 +48:24.073 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:24.075 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:24.075 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:24.075 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:24.075 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:24.075 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:24.075 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9e4f11db-cd69-ea1c-ad2f-2a1770544225 +48:24.075 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9e4f11db-cd69-ea1c-ad2f-2a1770544225 +48:24.075 [INFO] TestAgent.server: New leader elected: payload=Node-9e4f11db-cd69-ea1c-ad2f-2a1770544225 +48:24.075 [INFO] TestAgent.server: federation state anti-entropy synced +48:24.100 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:24.100 [INFO] TestAgent: Synced node info +48:24.152 [WARN] TestAgent: Service name will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: service=my_service +48:24.152 [INFO] TestAgent: Requesting shutdown +48:24.152 [INFO] TestAgent.server: shutting down server +48:24.152 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:24.152 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:24.152 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:24.152 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:24.152 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:24.152 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:24.153 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:24.153 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:24.153 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:24.153 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:24.153 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:24.153Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:24.153Z [INFO] agent.router.manager: shutting down +48:24.153 [INFO] TestAgent: consul server down +48:24.153 [INFO] TestAgent: shutdown complete +48:24.153 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23813 network=tcp +48:24.153 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23813 network=udp +48:24.153 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23814 network=tcp +48:24.656 [INFO] TestAgent: Waiting for endpoints to shut down +48:24.656 [INFO] TestAgent: Endpoints down +--- PASS: TestAddServiceIPv4TaggedSet (0.70s) === RUN TestAddServiceIPv6TaggedSet [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:08.608Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:08.658Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:08.660 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ce3a02ab-6e64-079e-f125-43b3767fc160 Address:127.0.0.1:16678}]" -12:08.661 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ce3a02ab-6e64-079e-f125-43b3767fc160.dc1 127.0.0.1 -12:08.661 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ce3a02ab-6e64-079e-f125-43b3767fc160 127.0.0.1 -2021-10-31T23:12:08.661Z [INFO] agent.router: Initializing LAN area manager -12:08.661 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16678 [Follower]" leader= -12:08.676 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16673 network=udp -12:08.676 [INFO] TestAgent.server: Adding LAN server: server="Node-ce3a02ab-6e64-079e-f125-43b3767fc160 (Addr: tcp/127.0.0.1:16678) (DC: dc1)" -12:08.676 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ce3a02ab-6e64-079e-f125-43b3767fc160.dc1 area=wan -12:08.676 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16673 network=tcp -12:08.676 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16674 network=tcp -12:08.676 [INFO] TestAgent: started state syncer -12:08.677 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16679 network=tcp -12:08.729 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:08.729 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16678 [Candidate]" term=2 -12:08.730 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:08.730 [DEBUG] TestAgent.server.raft: vote granted: from=ce3a02ab-6e64-079e-f125-43b3767fc160 term=2 tally=1 -12:08.730 [INFO] TestAgent.server.raft: election won: tally=1 -12:08.730 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16678 [Leader]" -12:08.730 [INFO] TestAgent.server: cluster leadership acquired -12:08.730 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16678 -12:08.731 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:08.732 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:08.732 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:08.732 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:08.732 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:08.732 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:08.733 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ce3a02ab-6e64-079e-f125-43b3767fc160 -12:08.733 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ce3a02ab-6e64-079e-f125-43b3767fc160 -12:08.733 [INFO] TestAgent.server: New leader elected: payload=Node-ce3a02ab-6e64-079e-f125-43b3767fc160 -12:08.734 [INFO] TestAgent.server: federation state anti-entropy synced -12:08.790 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:08.790 [INFO] TestAgent: Synced node info -12:09.022 [WARN] TestAgent: Service name will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: service=my_service -12:09.022 [INFO] TestAgent: Requesting shutdown -12:09.022 [INFO] TestAgent.server: shutting down server -12:09.022 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:09.022 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:09.022 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:09.022 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:09.022 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:09.022 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:09.022 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:09.022 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:09.022 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:09.022 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:09.022 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:09.022Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:09.022Z [INFO] agent.router.manager: shutting down -12:09.022 [INFO] TestAgent: consul server down -12:09.022 [INFO] TestAgent: shutdown complete -12:09.022 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16673 network=tcp -12:09.022 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16673 network=udp -12:09.022 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16674 network=tcp -12:09.525 [INFO] TestAgent: Waiting for endpoints to shut down -12:09.525 [INFO] TestAgent: Endpoints down ---- PASS: TestAddServiceIPv6TaggedSet (0.96s) +2022-12-04T05:48:24.682Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:24.720Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:24.722 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2ee45470-ee94-3ce5-215d-64e6588cc426 Address:127.0.0.1:23825}]" +48:24.722 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23825 [Follower]" leader= +48:24.723 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2ee45470-ee94-3ce5-215d-64e6588cc426.dc1 127.0.0.1 +48:24.732 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2ee45470-ee94-3ce5-215d-64e6588cc426 127.0.0.1 +2022-12-04T05:48:24.732Z [INFO] agent.router: Initializing LAN area manager +48:24.733 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23820 network=udp +48:24.733 [INFO] TestAgent.server: Adding LAN server: server="Node-2ee45470-ee94-3ce5-215d-64e6588cc426 (Addr: tcp/127.0.0.1:23825) (DC: dc1)" +48:24.733 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2ee45470-ee94-3ce5-215d-64e6588cc426.dc1 area=wan +48:24.733 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23820 network=tcp +48:24.734 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23821 network=tcp +48:24.734 [INFO] TestAgent: started state syncer +48:24.734 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23826 network=tcp +48:24.784 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:24.784 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23825 [Candidate]" term=2 +48:24.784 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:24.784 [DEBUG] TestAgent.server.raft: vote granted: from=2ee45470-ee94-3ce5-215d-64e6588cc426 term=2 tally=1 +48:24.784 [INFO] TestAgent.server.raft: election won: tally=1 +48:24.784 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23825 [Leader]" +48:24.784 [INFO] TestAgent.server: cluster leadership acquired +48:24.784 [INFO] TestAgent.server: New leader elected: payload=Node-2ee45470-ee94-3ce5-215d-64e6588cc426 +48:24.784 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23825 +48:24.785 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:24.787 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:24.787 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:24.787 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:24.787 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:24.787 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:24.787 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2ee45470-ee94-3ce5-215d-64e6588cc426 +48:24.787 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2ee45470-ee94-3ce5-215d-64e6588cc426 +48:24.788 [INFO] TestAgent.server: federation state anti-entropy synced +48:24.852 [WARN] TestAgent: Service name will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes.: service=my_service +48:24.852 [INFO] TestAgent: Requesting shutdown +48:24.852 [INFO] TestAgent.server: shutting down server +48:24.852 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:24.852 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:24.852 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:24.852 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:24.852 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:24.852 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:24.852 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:24.853 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:24.853 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:24.853 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:24.853 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:24.853 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:24.853Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:24.853Z [INFO] agent.router.manager: shutting down +48:24.853 [INFO] TestAgent: consul server down +48:24.853 [INFO] TestAgent: shutdown complete +48:24.853 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23820 network=tcp +48:24.853 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23820 network=udp +48:24.853 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23821 network=tcp +48:25.356 [INFO] TestAgent: Waiting for endpoints to shut down +48:25.356 [INFO] TestAgent: Endpoints down +--- PASS: TestAddServiceIPv6TaggedSet (0.70s) === RUN TestAgent_RemoveService === RUN TestAgent_RemoveService/normal === PAUSE TestAgent_RemoveService/normal @@ -11268,135 +11287,134 @@ [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:09.553Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:09.631Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:09.633 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8aa98556-476a-28d5-ca56-b5f53da8226d Address:127.0.0.1:16692}]" -12:09.637 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16692 [Follower]" leader= -12:09.638 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8aa98556-476a-28d5-ca56-b5f53da8226d.dc1 127.0.0.1 -12:09.638 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8aa98556-476a-28d5-ca56-b5f53da8226d 127.0.0.1 -2021-10-31T23:12:09.638Z [INFO] agent.router: Initializing LAN area manager -12:09.639 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16687 network=udp -12:09.639 [INFO] TestAgent.server: Adding LAN server: server="Node-8aa98556-476a-28d5-ca56-b5f53da8226d (Addr: tcp/127.0.0.1:16692) (DC: dc1)" -12:09.639 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8aa98556-476a-28d5-ca56-b5f53da8226d.dc1 area=wan -12:09.639 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16687 network=tcp -12:09.640 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16688 network=tcp -12:09.640 [INFO] TestAgent: started state syncer -12:09.640 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16693 network=tcp -12:09.677 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:09.677 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16692 [Candidate]" term=2 -12:09.678 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:09.678 [DEBUG] TestAgent.server.raft: vote granted: from=8aa98556-476a-28d5-ca56-b5f53da8226d term=2 tally=1 -12:09.678 [INFO] TestAgent.server.raft: election won: tally=1 -12:09.678 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16692 [Leader]" -12:09.678 [INFO] TestAgent.server: cluster leadership acquired -12:09.678 [INFO] TestAgent.server: New leader elected: payload=Node-8aa98556-476a-28d5-ca56-b5f53da8226d -12:09.678 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16692 -12:09.679 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:09.680 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:09.680 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:09.680 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:09.680 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:09.680 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:09.680 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8aa98556-476a-28d5-ca56-b5f53da8226d -12:09.680 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8aa98556-476a-28d5-ca56-b5f53da8226d -12:09.681 [INFO] TestAgent.server: federation state anti-entropy synced -12:09.823 [WARN] TestAgent: Failed to deregister service: service=redis error="Service {"redis" {}} does not exist" -12:09.823 [DEBUG] TestAgent: removed check: check=service:memcache -12:09.823 [DEBUG] TestAgent: removed check: check=check2 -12:09.823 [DEBUG] TestAgent: removed service: service=memcache -12:09.824 [DEBUG] TestAgent: removed check: check=service:redis:1 -12:09.824 [DEBUG] TestAgent: removed check: check=service:redis:2 -12:09.824 [DEBUG] TestAgent: removed service: service=redis -12:09.824 [INFO] TestAgent: Requesting shutdown -12:09.824 [INFO] TestAgent.server: shutting down server -12:09.824 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:09.824 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:09.824 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:09.824 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:09.824 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:09.824 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:09.824 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:09.824 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:09.824 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:09.824 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:09.824 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:09.824 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:09.824Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:09.824Z [INFO] agent.router.manager: shutting down -12:09.824 [INFO] TestAgent: consul server down -12:09.824 [INFO] TestAgent: shutdown complete -12:09.824 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16687 network=tcp -12:09.824 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16687 network=udp -12:09.824 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16688 network=tcp -12:10.325 [INFO] TestAgent: Waiting for endpoints to shut down -12:10.325 [INFO] TestAgent: Endpoints down -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:09.560Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:09.640Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:09.642 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a66858ea-2ba5-bd94-ed56-b7a0551c59d6 Address:127.0.0.1:16685}]" -12:09.643 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a66858ea-2ba5-bd94-ed56-b7a0551c59d6.dc1 127.0.0.1 -12:09.643 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a66858ea-2ba5-bd94-ed56-b7a0551c59d6 127.0.0.1 -2021-10-31T23:12:09.643Z [INFO] agent.router: Initializing LAN area manager -12:09.644 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16680 network=udp -12:09.644 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16685 [Follower]" leader= -12:09.644 [INFO] TestAgent.server: Adding LAN server: server="Node-a66858ea-2ba5-bd94-ed56-b7a0551c59d6 (Addr: tcp/127.0.0.1:16685) (DC: dc1)" -12:09.644 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a66858ea-2ba5-bd94-ed56-b7a0551c59d6.dc1 area=wan -12:09.644 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16680 network=tcp -12:09.645 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16681 network=tcp -12:09.645 [INFO] TestAgent: started state syncer -12:09.645 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16686 network=tcp -12:09.705 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:09.706 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16685 [Candidate]" term=2 -12:09.706 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:09.706 [DEBUG] TestAgent.server.raft: vote granted: from=a66858ea-2ba5-bd94-ed56-b7a0551c59d6 term=2 tally=1 -12:09.706 [INFO] TestAgent.server.raft: election won: tally=1 -12:09.706 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16685 [Leader]" -12:09.706 [INFO] TestAgent.server: cluster leadership acquired -12:09.706 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16685 -12:09.707 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:09.708 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:09.708 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:09.708 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:09.708 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:09.708 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:09.708 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a66858ea-2ba5-bd94-ed56-b7a0551c59d6 -12:09.708 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a66858ea-2ba5-bd94-ed56-b7a0551c59d6 -12:09.709 [INFO] TestAgent.server: New leader elected: payload=Node-a66858ea-2ba5-bd94-ed56-b7a0551c59d6 -12:09.709 [INFO] TestAgent.server: federation state anti-entropy synced -12:09.798 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:09.798 [INFO] TestAgent: Synced node info -12:10.074 [WARN] TestAgent: Failed to deregister service: service=redis error="Service {"redis" {}} does not exist" -12:10.074 [DEBUG] TestAgent: removed check: check=service:memcache -12:10.074 [DEBUG] TestAgent: removed check: check=check2 -12:10.074 [DEBUG] TestAgent: removed service: service=memcache -12:10.074 [DEBUG] TestAgent: removed check: check=service:redis:1 -12:10.074 [DEBUG] TestAgent: removed check: check=service:redis:2 -12:10.074 [DEBUG] TestAgent: removed service: service=redis -12:10.074 [INFO] TestAgent: Requesting shutdown -12:10.074 [INFO] TestAgent.server: shutting down server -12:10.074 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:10.074 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:10.074 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:10.074 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:10.074 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:10.074 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:10.074 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:10.074 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:10.074 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:10.074 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:10.074 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:10.075Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:10.075Z [INFO] agent.router.manager: shutting down -12:10.075 [INFO] TestAgent: consul server down -12:10.075 [INFO] TestAgent: shutdown complete -12:10.075 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16680 network=tcp -12:10.075 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16680 network=udp -12:10.075 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16681 network=tcp -12:10.577 [INFO] TestAgent: Waiting for endpoints to shut down -12:10.577 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:25.375Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:25.409Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:25.410 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0872f4cb-9025-aa07-68a1-8d3a1fa58d8f Address:127.0.0.1:23832}]" +48:25.411 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0872f4cb-9025-aa07-68a1-8d3a1fa58d8f.dc1 127.0.0.1 +48:25.412 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0872f4cb-9025-aa07-68a1-8d3a1fa58d8f 127.0.0.1 +2022-12-04T05:48:25.412Z [INFO] agent.router: Initializing LAN area manager +48:25.412 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23832 [Follower]" leader= +48:25.412 [INFO] TestAgent.server: Adding LAN server: server="Node-0872f4cb-9025-aa07-68a1-8d3a1fa58d8f (Addr: tcp/127.0.0.1:23832) (DC: dc1)" +48:25.412 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0872f4cb-9025-aa07-68a1-8d3a1fa58d8f.dc1 area=wan +48:25.416 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23827 network=udp +48:25.416 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23827 network=tcp +48:25.417 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23828 network=tcp +48:25.417 [INFO] TestAgent: started state syncer +48:25.417 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23833 network=tcp +48:25.480 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:25.480 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23832 [Candidate]" term=2 +48:25.480 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:25.480 [DEBUG] TestAgent.server.raft: vote granted: from=0872f4cb-9025-aa07-68a1-8d3a1fa58d8f term=2 tally=1 +48:25.480 [INFO] TestAgent.server.raft: election won: tally=1 +48:25.480 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23832 [Leader]" +48:25.480 [INFO] TestAgent.server: cluster leadership acquired +48:25.480 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23832 +48:25.481 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:25.483 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:25.483 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:25.483 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:25.483 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:25.483 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:25.483 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0872f4cb-9025-aa07-68a1-8d3a1fa58d8f +48:25.483 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0872f4cb-9025-aa07-68a1-8d3a1fa58d8f +48:25.483 [INFO] TestAgent.server: New leader elected: payload=Node-0872f4cb-9025-aa07-68a1-8d3a1fa58d8f +48:25.484 [INFO] TestAgent.server: federation state anti-entropy synced +48:25.736 [WARN] TestAgent: Failed to deregister service: service=redis error="Service {"redis" {}} does not exist" +48:25.736 [DEBUG] TestAgent: removed check: check=service:memcache +48:25.736 [DEBUG] TestAgent: removed check: check=check2 +48:25.736 [DEBUG] TestAgent: removed service: service=memcache +48:25.737 [DEBUG] TestAgent: removed check: check=service:redis:2 +48:25.737 [DEBUG] TestAgent: removed check: check=service:redis:1 +48:25.737 [DEBUG] TestAgent: removed service: service=redis +48:25.737 [INFO] TestAgent: Requesting shutdown +48:25.737 [INFO] TestAgent.server: shutting down server +48:25.737 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:25.737 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:25.737 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:25.737 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:25.737 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:25.737 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:25.737 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:25.737 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:25.737 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:25.737 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:25.737 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:25.737 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:25.737Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:25.737Z [INFO] agent.router.manager: shutting down +48:25.738 [INFO] TestAgent: consul server down +48:25.738 [INFO] TestAgent: shutdown complete +48:25.738 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23827 network=tcp +48:25.738 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23827 network=udp +48:25.738 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23828 network=tcp +48:26.240 [INFO] TestAgent: Waiting for endpoints to shut down +48:26.240 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:25.382Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:25.408Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:25.414 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e372e0e6-f3f9-7fa7-2e8a-872a4b8201b2 Address:127.0.0.1:23839}]" +48:25.414 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23839 [Follower]" leader= +48:25.414 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e372e0e6-f3f9-7fa7-2e8a-872a4b8201b2.dc1 127.0.0.1 +48:25.415 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e372e0e6-f3f9-7fa7-2e8a-872a4b8201b2 127.0.0.1 +2022-12-04T05:48:25.415Z [INFO] agent.router: Initializing LAN area manager +48:25.415 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23834 network=udp +48:25.415 [INFO] TestAgent.server: Adding LAN server: server="Node-e372e0e6-f3f9-7fa7-2e8a-872a4b8201b2 (Addr: tcp/127.0.0.1:23839) (DC: dc1)" +48:25.415 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e372e0e6-f3f9-7fa7-2e8a-872a4b8201b2.dc1 area=wan +48:25.415 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23834 network=tcp +48:25.424 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23835 network=tcp +48:25.424 [INFO] TestAgent: started state syncer +48:25.424 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23840 network=tcp +48:25.472 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:25.472 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23839 [Candidate]" term=2 +48:25.472 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:25.472 [DEBUG] TestAgent.server.raft: vote granted: from=e372e0e6-f3f9-7fa7-2e8a-872a4b8201b2 term=2 tally=1 +48:25.472 [INFO] TestAgent.server.raft: election won: tally=1 +48:25.472 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23839 [Leader]" +48:25.472 [INFO] TestAgent.server: cluster leadership acquired +48:25.472 [INFO] TestAgent.server: New leader elected: payload=Node-e372e0e6-f3f9-7fa7-2e8a-872a4b8201b2 +48:25.473 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23839 +48:25.473 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:25.475 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:25.475 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:25.475 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:25.475 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:25.475 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:25.475 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e372e0e6-f3f9-7fa7-2e8a-872a4b8201b2 +48:25.475 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e372e0e6-f3f9-7fa7-2e8a-872a4b8201b2 +48:25.476 [INFO] TestAgent.server: federation state anti-entropy synced +48:25.512 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:25.512 [INFO] TestAgent: Synced node info +48:25.772 [WARN] TestAgent: Failed to deregister service: service=redis error="Service {"redis" {}} does not exist" +48:25.772 [DEBUG] TestAgent: removed check: check=service:memcache +48:25.772 [DEBUG] TestAgent: removed check: check=check2 +48:25.772 [DEBUG] TestAgent: removed service: service=memcache +48:25.773 [DEBUG] TestAgent: removed check: check=service:redis:1 +48:25.773 [DEBUG] TestAgent: removed check: check=service:redis:2 +48:25.773 [DEBUG] TestAgent: removed service: service=redis +48:25.773 [INFO] TestAgent: Requesting shutdown +48:25.773 [INFO] TestAgent.server: shutting down server +48:25.773 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:25.773 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:25.773 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:25.773 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:25.773 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:25.773 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:25.773 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:25.773 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:25.773 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:25.773 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:25.773 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:25.773Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:25.773Z [INFO] agent.router.manager: shutting down +48:25.773 [INFO] TestAgent: consul server down +48:25.773 [INFO] TestAgent: shutdown complete +48:25.773 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23834 network=tcp +48:25.773 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23834 network=udp +48:25.773 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23835 network=tcp +48:26.276 [INFO] TestAgent: Waiting for endpoints to shut down +48:26.276 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RemoveService (0.00s) - --- PASS: TestAgent_RemoveService/service_manager (0.80s) - --- PASS: TestAgent_RemoveService/normal (1.05s) + --- PASS: TestAgent_RemoveService/normal (0.88s) + --- PASS: TestAgent_RemoveService/service_manager (0.92s) === RUN TestAgent_RemoveServiceRemovesAllChecks === RUN TestAgent_RemoveServiceRemovesAllChecks/normal === PAUSE TestAgent_RemoveServiceRemovesAllChecks/normal @@ -11404,131 +11422,137 @@ === PAUSE TestAgent_RemoveServiceRemovesAllChecks/service_manager === CONT TestAgent_RemoveServiceRemovesAllChecks/normal === CONT TestAgent_RemoveServiceRemovesAllChecks/service_manager -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:10.613Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:10.652Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:10.654 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d67cb7a9-dde5-93a1-66a6-97c314c31f52 Address:127.0.0.1:16699}]" -12:10.669 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -12:10.674 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16699 [Follower]" leader= -12:10.683 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2021-10-31T23:12:10.683Z [INFO] agent.router: Initializing LAN area manager -12:10.684 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16694 network=udp -12:10.685 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -12:10.685 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:16699) (DC: dc1)" -12:10.685 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16694 network=tcp -12:10.715 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16695 network=tcp -12:10.715 [INFO] TestAgent: started state syncer -12:10.716 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16700 network=tcp -12:10.717 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:10.717 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16699 [Candidate]" term=2 -12:10.726 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:10.726 [DEBUG] TestAgent.server.raft: vote granted: from=d67cb7a9-dde5-93a1-66a6-97c314c31f52 term=2 tally=1 -12:10.726 [INFO] TestAgent.server.raft: election won: tally=1 -12:10.726 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16699 [Leader]" -12:10.726 [INFO] TestAgent.server: cluster leadership acquired -12:10.726 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16699 -12:10.726 [INFO] TestAgent.server: New leader elected: payload=node1 -12:10.727 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:10.729 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:10.729 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:10.729 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:10.729 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:10.729 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:10.729 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -12:10.729 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -12:10.729 [INFO] TestAgent: Synced node info -12:10.729 [DEBUG] TestAgent: Node info in sync -12:10.738 [INFO] TestAgent.server: federation state anti-entropy synced -12:10.806 [DEBUG] TestAgent: removed check: check=chk1 -12:10.806 [DEBUG] TestAgent: removed check: check=chk2 -12:10.806 [DEBUG] TestAgent: removed service: service=redis -12:10.806 [INFO] TestAgent: Requesting shutdown -12:10.806 [INFO] TestAgent.server: shutting down server -12:10.806 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:10.806 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:10.806 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:10.806 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:10.806 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:10.806 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:10.806 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:10.806 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:10.806 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:10.806 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:10.806 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:10.806Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:10.806Z [INFO] agent.router.manager: shutting down -12:10.807 [INFO] TestAgent: consul server down -12:10.807 [INFO] TestAgent: shutdown complete -12:10.807 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16694 network=tcp -12:10.807 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16694 network=udp -12:10.807 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16695 network=tcp -12:11.309 [INFO] TestAgent: Waiting for endpoints to shut down -12:11.309 [INFO] TestAgent: Endpoints down -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:10.636Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:10.694Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:10.697 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:77c542c7-e591-ce00-248b-e1d2c53d3876 Address:127.0.0.1:16706}]" -12:10.714 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16706 [Follower]" leader= -12:10.715 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 -12:10.716 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 -2021-10-31T23:12:10.716Z [INFO] agent.router: Initializing LAN area manager -12:10.716 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16701 network=udp -12:10.716 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:16706) (DC: dc1)" -12:10.716 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan -12:10.716 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16701 network=tcp -12:10.717 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16702 network=tcp -12:10.717 [INFO] TestAgent: started state syncer -12:10.717 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16707 network=tcp -12:10.785 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:10.786 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16706 [Candidate]" term=2 -12:10.786 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:10.786 [DEBUG] TestAgent.server.raft: vote granted: from=77c542c7-e591-ce00-248b-e1d2c53d3876 term=2 tally=1 -12:10.786 [INFO] TestAgent.server.raft: election won: tally=1 -12:10.786 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16706 [Leader]" -12:10.786 [INFO] TestAgent.server: cluster leadership acquired -12:10.786 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16706 -12:10.787 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:10.788 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:10.788 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:10.788 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:10.788 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:10.788 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:10.788 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 -12:10.788 [INFO] TestAgent.server: member joined, marking health alive: member=node1 -12:10.789 [INFO] TestAgent.server: New leader elected: payload=node1 -12:10.789 [INFO] TestAgent.server: federation state anti-entropy synced -12:10.858 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:10.858 [INFO] TestAgent: Synced node info -12:10.866 [DEBUG] TestAgent: removed check: check=chk2 -12:10.866 [DEBUG] TestAgent: removed check: check=chk1 -12:10.866 [DEBUG] TestAgent: removed service: service=redis -12:10.866 [INFO] TestAgent: Requesting shutdown -12:10.866 [INFO] TestAgent.server: shutting down server -12:10.866 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:10.866 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:10.866 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:10.866 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:10.866 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:10.866 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:10.866 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:10.866 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:10.866 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:10.866 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:10.867 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:10.867Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:10.867Z [INFO] agent.router.manager: shutting down -12:10.867 [INFO] TestAgent: consul server down -12:10.867 [INFO] TestAgent: shutdown complete -12:10.867 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16701 network=tcp -12:10.867 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16701 network=udp -12:10.867 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16702 network=tcp -12:11.369 [INFO] TestAgent: Waiting for endpoints to shut down -12:11.369 [INFO] TestAgent: Endpoints down +[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:48:26.329Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:26.409Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:26.411 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d34ed327-deda-947e-4a25-041afd13cb19 Address:127.0.0.1:23846}]" +48:26.412 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +48:26.412 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23846 [Follower]" leader= +48:26.413 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2022-12-04T05:48:26.413Z [INFO] agent.router: Initializing LAN area manager +48:26.413 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23841 network=udp +48:26.413 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:23846) (DC: dc1)" +48:26.413 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +48:26.413 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23841 network=tcp +48:26.414 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23842 network=tcp +48:26.414 [INFO] TestAgent: started state syncer +48:26.414 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23847 network=tcp +48:26.448 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:26.448 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23846 [Candidate]" term=2 +48:26.448 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:26.448 [DEBUG] TestAgent.server.raft: vote granted: from=d34ed327-deda-947e-4a25-041afd13cb19 term=2 tally=1 +48:26.448 [INFO] TestAgent.server.raft: election won: tally=1 +48:26.448 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23846 [Leader]" +48:26.448 [INFO] TestAgent.server: cluster leadership acquired +48:26.449 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23846 +48:26.449 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:26.451 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:26.451 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:26.451 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:26.451 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:26.451 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:26.451 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +48:26.451 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +48:26.451 [INFO] TestAgent.server: New leader elected: payload=node1 +48:26.452 [INFO] TestAgent.server: federation state anti-entropy synced +48:26.592 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:26.593 [INFO] TestAgent: Synced node info +48:26.660 [DEBUG] TestAgent: removed check: check=chk1 +48:26.660 [DEBUG] TestAgent: removed check: check=chk2 +48:26.660 [DEBUG] TestAgent: removed service: service=redis +48:26.660 [INFO] TestAgent: Requesting shutdown +48:26.660 [INFO] TestAgent.server: shutting down server +48:26.660 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:26.660 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:26.660 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:26.660 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:26.660 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:26.661 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:26.661 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:26.661 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:26.661 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:26.661 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:26.661 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:26.661Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:26.661Z [INFO] agent.router.manager: shutting down +48:26.661 [INFO] TestAgent: consul server down +48:26.661 [INFO] TestAgent: shutdown complete +48:26.661 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23841 network=tcp +48:26.661 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23841 network=udp +48:26.661 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23842 network=tcp +48:27.164 [INFO] TestAgent: Waiting for endpoints to shut down +48:27.164 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:26.347Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:26.420Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:26.422 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a29f9f81-4e26-8985-c7d5-b74c2055403a Address:127.0.0.1:23853}]" +48:26.422 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: node1.dc1 127.0.0.1 +48:26.423 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: node1 127.0.0.1 +2022-12-04T05:48:26.423Z [INFO] agent.router: Initializing LAN area manager +48:26.423 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23848 network=udp +48:26.423 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23853 [Follower]" leader= +48:26.424 [INFO] TestAgent.server: Adding LAN server: server="node1 (Addr: tcp/127.0.0.1:23853) (DC: dc1)" +48:26.424 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=node1.dc1 area=wan +48:26.424 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23848 network=tcp +48:26.429 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23849 network=tcp +48:26.429 [INFO] TestAgent: started state syncer +48:26.429 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23854 network=tcp +48:26.492 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:26.492 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23853 [Candidate]" term=2 +48:26.492 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:26.492 [DEBUG] TestAgent.server.raft: vote granted: from=a29f9f81-4e26-8985-c7d5-b74c2055403a term=2 tally=1 +48:26.492 [INFO] TestAgent.server.raft: election won: tally=1 +48:26.492 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23853 [Leader]" +48:26.492 [INFO] TestAgent.server: cluster leadership acquired +48:26.493 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23853 +48:26.493 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:26.495 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:26.495 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:26.495 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:26.495 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:26.495 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:26.495 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=node1 +48:26.495 [INFO] TestAgent.server: member joined, marking health alive: member=node1 +48:26.496 [INFO] TestAgent.server: New leader elected: payload=node1 +48:26.496 [INFO] TestAgent.server: federation state anti-entropy synced +48:26.556 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:26.557 [INFO] TestAgent: Synced node info +48:26.557 [DEBUG] TestAgent: Node info in sync +48:26.572 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:26.572 [DEBUG] TestAgent: Node info in sync +48:26.748 [DEBUG] TestAgent: removed check: check=chk1 +48:26.748 [DEBUG] TestAgent: removed check: check=chk2 +48:26.748 [DEBUG] TestAgent: removed service: service=redis +48:26.748 [INFO] TestAgent: Requesting shutdown +48:26.748 [INFO] TestAgent.server: shutting down server +48:26.748 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:26.748 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:26.748 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:26.748 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:26.748 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:26.749 [DEBUG] TestAgent: Node info in sync +48:26.749 [INFO] TestAgent: Deregistered service: service=redis +48:26.749 [DEBUG] TestAgent: Node info in sync +48:26.749 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:26.749 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:26.749 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:26.749 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:26.749 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:26.749 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:26.749Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:26.749Z [INFO] agent.router.manager: shutting down +48:26.749 [INFO] TestAgent: consul server down +48:26.749 [INFO] TestAgent: shutdown complete +48:26.749 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23848 network=tcp +48:26.749 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23848 network=udp +48:26.749 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23849 network=tcp +48:27.252 [INFO] TestAgent: Waiting for endpoints to shut down +48:27.252 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_RemoveServiceRemovesAllChecks (0.00s) - --- PASS: TestAgent_RemoveServiceRemovesAllChecks/normal (0.73s) - --- PASS: TestAgent_RemoveServiceRemovesAllChecks/service_manager (0.79s) + --- PASS: TestAgent_RemoveServiceRemovesAllChecks/normal (0.89s) + --- PASS: TestAgent_RemoveServiceRemovesAllChecks/service_manager (0.97s) === RUN TestAgent_IndexChurn === PAUSE TestAgent_IndexChurn === RUN TestAgent_AddCheck @@ -11574,99 +11598,100 @@ === PAUSE TestAgent_PersistService/service_manager === CONT TestAgent_PersistService/normal === CONT TestAgent_PersistService/service_manager -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -12:12.052 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-a884b693-d4c2-1d35-2413-2631a852e4fe 127.0.0.1 -2021-10-31T23:12:12.052Z [INFO] agent.router: Initializing LAN area manager -12:12.052 [DEBUG] TestAgent: restored service definition from file: service=redis file=/tmp/consul-test/TestAgent_PersistService_service_manager-TestAgent-agent534097832/services/86a1b907d54bf7010394bf316e183e67 -12:12.053 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16729 network=udp -12:12.053 [INFO] TestAgent.client.serf.lan: serf: Attempting re-join to previously known node: Node-e35e1441-2e44-84f0-c703-82c035e8f3ab: 127.0.0.1:16718 -12:12.053 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16729 network=tcp -12:12.054 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:16718: connect: connection refused -12:12.054 [WARN] TestAgent.client.serf.lan: serf: Failed to re-join any previously known node -12:12.058 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16730 network=tcp -12:12.058 [INFO] TestAgent: started state syncer -12:12.059 [INFO] TestAgent: Requesting shutdown -12:12.059 [INFO] TestAgent.client: shutting down client -12:12.059 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -12:12.059 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16735 network=tcp -12:12.059 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -2021-10-31T23:12:12.059Z [WARN] agent.router.manager: No servers available -12:12.059 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -12:12.059 [INFO] TestAgent: consul client down -12:12.059 [INFO] TestAgent: shutdown complete -12:12.059 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16729 network=tcp -12:12.059 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16729 network=udp -12:12.059 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16730 network=tcp -12:12.561 [INFO] TestAgent: Waiting for endpoints to shut down -12:12.561 [INFO] TestAgent: Endpoints down -12:11.445 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-e35e1441-2e44-84f0-c703-82c035e8f3ab 127.0.0.1 -2021-10-31T23:12:11.445Z [INFO] agent.router: Initializing LAN area manager -12:11.445 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16715 network=udp -12:11.448 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16715 network=tcp -12:11.448 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16716 network=tcp -12:11.448 [INFO] TestAgent: started state syncer -2021-10-31T23:12:11.449Z [WARN] agent.router.manager: No servers available -12:11.449 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -12:11.449 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16721 network=tcp -12:11.450 [INFO] TestAgent: Requesting shutdown -12:11.451 [INFO] TestAgent.client: shutting down client -12:11.451 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -12:11.451 [INFO] TestAgent: consul client down -12:11.451 [INFO] TestAgent: shutdown complete -12:11.451 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16715 network=tcp -12:11.451 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16715 network=udp -12:11.451 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16716 network=tcp -12:11.953 [INFO] TestAgent: Waiting for endpoints to shut down -12:11.953 [INFO] TestAgent: Endpoints down -12:12.056 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-9a3b68f0-4d4b-a400-4d73-525f633e41bd 127.0.0.1 -2021-10-31T23:12:12.057Z [INFO] agent.router: Initializing LAN area manager -12:12.057 [DEBUG] TestAgent: restored service definition from file: service=redis file=/tmp/consul-test/TestAgent_PersistService_normal-TestAgent-agent149236269/services/86a1b907d54bf7010394bf316e183e67 -12:12.057 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16722 network=udp -12:12.057 [INFO] TestAgent.client.serf.lan: serf: Attempting re-join to previously known node: Node-1817df09-8617-5ef5-09f5-b2976dea11bc: 127.0.0.1:16711 -12:12.058 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16722 network=tcp -12:12.058 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16723 network=tcp -12:12.058 [INFO] TestAgent: started state syncer -12:12.059 [INFO] TestAgent: Requesting shutdown -12:12.059 [INFO] TestAgent.client: shutting down client -12:12.059 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -12:12.059 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16728 network=tcp -12:12.059 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -2021-10-31T23:12:12.059Z [WARN] agent.router.manager: No servers available -12:12.059 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -12:12.059 [INFO] TestAgent: consul client down -12:12.059 [INFO] TestAgent: shutdown complete -12:12.059 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16722 network=tcp -12:12.059 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16722 network=udp -12:12.059 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16723 network=tcp -12:12.066 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:16711: connect: connection refused -12:12.066 [WARN] TestAgent.client.serf.lan: serf: Failed to re-join any previously known node -12:12.562 [INFO] TestAgent: Waiting for endpoints to shut down -12:12.562 [INFO] TestAgent: Endpoints down -12:11.427 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-1817df09-8617-5ef5-09f5-b2976dea11bc 127.0.0.1 -2021-10-31T23:12:11.427Z [INFO] agent.router: Initializing LAN area manager -12:11.428 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16708 network=udp -12:11.428 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16708 network=tcp -12:11.428 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16709 network=tcp -12:11.428 [INFO] TestAgent: started state syncer -12:11.429 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16714 network=tcp -2021-10-31T23:12:11.429Z [WARN] agent.router.manager: No servers available -12:11.430 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -12:11.438 [INFO] TestAgent: Requesting shutdown -12:11.438 [INFO] TestAgent.client: shutting down client -12:11.438 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -12:11.438 [INFO] TestAgent: consul client down -12:11.438 [INFO] TestAgent: shutdown complete -12:11.438 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16708 network=tcp -12:11.438 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16708 network=udp -12:11.438 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16709 network=tcp -12:11.941 [INFO] TestAgent: Waiting for endpoints to shut down -12:11.941 [INFO] TestAgent: Endpoints down +48:27.855 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-f24f49b2-f706-df0e-25c6-dff478c335a3 127.0.0.1 +2022-12-04T05:48:27.855Z [INFO] agent.router: Initializing LAN area manager +48:27.856 [DEBUG] TestAgent: restored service definition from file: service=redis file=/tmp/consul-test/TestAgent_PersistService_normal-TestAgent-agent995684524/services/86a1b907d54bf7010394bf316e183e67 +48:27.856 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23869 network=udp +48:27.856 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23869 network=tcp +48:27.857 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23870 network=tcp +48:27.857 [INFO] TestAgent: started state syncer +48:27.857 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23875 network=tcp +48:27.857 [INFO] TestAgent: Requesting shutdown +48:27.857 [INFO] TestAgent.client: shutting down client +48:27.857 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +2022-12-04T05:48:27.857Z [WARN] agent.router.manager: No servers available +48:27.857 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +48:27.857 [INFO] TestAgent: consul client down +48:27.857 [INFO] TestAgent: shutdown complete +48:27.857 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23869 network=tcp +48:27.857 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23869 network=udp +48:27.857 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23870 network=tcp +48:27.864 [INFO] TestAgent.client.serf.lan: serf: Attempting re-join to previously known node: Node-c1dff128-929a-65ee-452a-c229f414d542: 127.0.0.1:23858 +48:27.868 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:23858: connect: connection refused +48:27.868 [WARN] TestAgent.client.serf.lan: serf: Failed to re-join any previously known node +48:28.360 [INFO] TestAgent: Waiting for endpoints to shut down +48:28.360 [INFO] TestAgent: Endpoints down +48:27.300 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-c1dff128-929a-65ee-452a-c229f414d542 127.0.0.1 +2022-12-04T05:48:27.300Z [INFO] agent.router: Initializing LAN area manager +48:27.300 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23855 network=udp +48:27.300 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23855 network=tcp +48:27.301 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23856 network=tcp +48:27.301 [INFO] TestAgent: started state syncer +48:27.301 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23861 network=tcp +2022-12-04T05:48:27.302Z [WARN] agent.router.manager: No servers available +48:27.302 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +48:27.302 [INFO] TestAgent: Requesting shutdown +48:27.302 [INFO] TestAgent.client: shutting down client +48:27.302 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +48:27.302 [INFO] TestAgent: consul client down +48:27.302 [INFO] TestAgent: shutdown complete +48:27.302 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23855 network=tcp +48:27.302 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23855 network=udp +48:27.302 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23856 network=tcp +48:27.804 [INFO] TestAgent: Waiting for endpoints to shut down +48:27.804 [INFO] TestAgent: Endpoints down +48:27.941 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-56503284-621f-bc37-9f20-847bb7794f76 127.0.0.1 +2022-12-04T05:48:27.941Z [INFO] agent.router: Initializing LAN area manager +48:27.941 [DEBUG] TestAgent: restored service definition from file: service=redis file=/tmp/consul-test/TestAgent_PersistService_service_manager-TestAgent-agent650535451/services/86a1b907d54bf7010394bf316e183e67 +48:27.941 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23876 network=udp +48:27.942 [INFO] TestAgent.client.serf.lan: serf: Attempting re-join to previously known node: Node-74140849-a9af-97cd-c22c-baec472d2b92: 127.0.0.1:23865 +48:27.942 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23876 network=tcp +48:27.943 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23877 network=tcp +48:27.943 [INFO] TestAgent: started state syncer +48:27.943 [INFO] TestAgent: Requesting shutdown +48:27.943 [INFO] TestAgent.client: shutting down client +48:27.943 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +48:27.943 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23882 network=tcp +48:27.943 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" +2022-12-04T05:48:27.943Z [WARN] agent.router.manager: No servers available +48:27.943 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +48:27.943 [INFO] TestAgent: consul client down +48:27.943 [INFO] TestAgent: shutdown complete +48:27.943 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23876 network=tcp +48:27.943 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23876 network=udp +48:27.943 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23877 network=tcp +48:27.943 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:23865: connect: connection refused +48:27.943 [WARN] TestAgent.client.serf.lan: serf: Failed to re-join any previously known node +48:28.444 [INFO] TestAgent: Waiting for endpoints to shut down +48:28.444 [INFO] TestAgent: Endpoints down +48:27.374 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-74140849-a9af-97cd-c22c-baec472d2b92 127.0.0.1 +2022-12-04T05:48:27.374Z [INFO] agent.router: Initializing LAN area manager +48:27.374 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23862 network=udp +48:27.374 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23862 network=tcp +48:27.375 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23863 network=tcp +48:27.375 [INFO] TestAgent: started state syncer +48:27.376 [INFO] TestAgent: Requesting shutdown +48:27.376 [INFO] TestAgent.client: shutting down client +48:27.376 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +48:27.376 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23868 network=tcp +48:27.376 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" +2022-12-04T05:48:27.376Z [WARN] agent.router.manager: No servers available +48:27.376 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +48:27.376 [INFO] TestAgent: consul client down +48:27.376 [INFO] TestAgent: shutdown complete +48:27.376 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23862 network=tcp +48:27.376 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23862 network=udp +48:27.376 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23863 network=tcp +48:27.880 [INFO] TestAgent: Waiting for endpoints to shut down +48:27.880 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_PersistService (0.00s) + --- PASS: TestAgent_PersistService/normal (1.11s) --- PASS: TestAgent_PersistService/service_manager (1.19s) - --- PASS: TestAgent_PersistService/normal (1.19s) === RUN TestAgent_persistedService_compat === RUN TestAgent_persistedService_compat/normal === PAUSE TestAgent_persistedService_compat/normal @@ -11677,123 +11702,121 @@ [WARN] freeport: 6 out of 34 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:12.596Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:12.672Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:12.686 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ecd84b81-e69a-1737-406b-5cf138ecea9a Address:127.0.0.1:16748}]" -12:12.687 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ecd84b81-e69a-1737-406b-5cf138ecea9a.dc1 127.0.0.1 -12:12.688 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ecd84b81-e69a-1737-406b-5cf138ecea9a 127.0.0.1 -2021-10-31T23:12:12.688Z [INFO] agent.router: Initializing LAN area manager -12:12.688 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16743 network=udp -12:12.688 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16748 [Follower]" leader= -12:12.688 [INFO] TestAgent.server: Adding LAN server: server="Node-ecd84b81-e69a-1737-406b-5cf138ecea9a (Addr: tcp/127.0.0.1:16748) (DC: dc1)" -12:12.688 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ecd84b81-e69a-1737-406b-5cf138ecea9a.dc1 area=wan -12:12.689 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16743 network=tcp -12:12.689 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16744 network=tcp -12:12.689 [INFO] TestAgent: started state syncer -12:12.689 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16749 network=tcp -12:12.745 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:12.746 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16748 [Candidate]" term=2 -12:12.746 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:12.746 [DEBUG] TestAgent.server.raft: vote granted: from=ecd84b81-e69a-1737-406b-5cf138ecea9a term=2 tally=1 -12:12.746 [INFO] TestAgent.server.raft: election won: tally=1 -12:12.746 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16748 [Leader]" -12:12.746 [INFO] TestAgent.server: cluster leadership acquired -12:12.746 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16748 -12:12.747 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:12.748 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:12.748 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:12.748 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:12.748 [INFO] TestAgent.server: New leader elected: payload=Node-ecd84b81-e69a-1737-406b-5cf138ecea9a -12:12.748 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:12.748 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:12.749 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ecd84b81-e69a-1737-406b-5cf138ecea9a -12:12.749 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ecd84b81-e69a-1737-406b-5cf138ecea9a -12:12.749 [INFO] TestAgent.server: federation state anti-entropy synced -12:13.038 [DEBUG] TestAgent: restored service definition from file: service=redis file=/tmp/consul-test/TestAgent_persistedService_compat_service_manager-TestAgent-agent412585242/services/86a1b907d54bf7010394bf316e183e67 -12:13.038 [INFO] TestAgent: Requesting shutdown -12:13.038 [INFO] TestAgent.server: shutting down server -12:13.038 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:13.038 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:13.038 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:13.038 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:13.038 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:13.038 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:13.038 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:13.038 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:13.038 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:13.038 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:13.038 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:13.038 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:13.039Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:13.039Z [INFO] agent.router.manager: shutting down -12:13.039 [INFO] TestAgent: consul server down -12:13.039 [INFO] TestAgent: shutdown complete -12:13.039 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16743 network=tcp -12:13.039 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16743 network=udp -12:13.039 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16744 network=tcp -12:13.541 [INFO] TestAgent: Waiting for endpoints to shut down -12:13.541 [INFO] TestAgent: Endpoints down -2021-10-31T23:12:12.592Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:12.675Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:12.677 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:59e0fd4f-ddfa-2fef-eb2f-624ee2a8a063 Address:127.0.0.1:16741}]" -12:12.677 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16741 [Follower]" leader= -12:12.678 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-59e0fd4f-ddfa-2fef-eb2f-624ee2a8a063.dc1 127.0.0.1 -12:12.678 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-59e0fd4f-ddfa-2fef-eb2f-624ee2a8a063 127.0.0.1 -2021-10-31T23:12:12.678Z [INFO] agent.router: Initializing LAN area manager -12:12.679 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16736 network=udp -12:12.679 [INFO] TestAgent.server: Adding LAN server: server="Node-59e0fd4f-ddfa-2fef-eb2f-624ee2a8a063 (Addr: tcp/127.0.0.1:16741) (DC: dc1)" -12:12.679 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-59e0fd4f-ddfa-2fef-eb2f-624ee2a8a063.dc1 area=wan -12:12.679 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16736 network=tcp -12:12.680 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16737 network=tcp -12:12.680 [INFO] TestAgent: started state syncer -12:12.680 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16742 network=tcp -12:12.737 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:12.737 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16741 [Candidate]" term=2 -12:12.738 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:12.738 [DEBUG] TestAgent.server.raft: vote granted: from=59e0fd4f-ddfa-2fef-eb2f-624ee2a8a063 term=2 tally=1 -12:12.738 [INFO] TestAgent.server.raft: election won: tally=1 -12:12.738 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16741 [Leader]" -12:12.738 [INFO] TestAgent.server: cluster leadership acquired -12:12.738 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16741 -12:12.739 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:12.740 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:12.740 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:12.740 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:12.740 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:12.740 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:12.740 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-59e0fd4f-ddfa-2fef-eb2f-624ee2a8a063 -12:12.740 [INFO] TestAgent.server: member joined, marking health alive: member=Node-59e0fd4f-ddfa-2fef-eb2f-624ee2a8a063 -12:12.741 [INFO] TestAgent.server: New leader elected: payload=Node-59e0fd4f-ddfa-2fef-eb2f-624ee2a8a063 -12:12.741 [INFO] TestAgent.server: federation state anti-entropy synced -12:12.802 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:12.802 [INFO] TestAgent: Synced node info -12:13.082 [DEBUG] TestAgent: restored service definition from file: service=redis file=/tmp/consul-test/TestAgent_persistedService_compat_normal-TestAgent-agent840682215/services/86a1b907d54bf7010394bf316e183e67 -12:13.082 [INFO] TestAgent: Requesting shutdown -12:13.082 [INFO] TestAgent.server: shutting down server -12:13.082 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:13.082 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:13.082 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:13.082 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:13.082 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:13.082 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:13.082 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:13.082 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:13.082 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:13.082 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:13.082 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:13.083Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:13.083Z [INFO] agent.router.manager: shutting down -12:13.083 [INFO] TestAgent: consul server down -12:13.083 [INFO] TestAgent: shutdown complete -12:13.083 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16736 network=tcp -12:13.083 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16736 network=udp -12:13.083 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16737 network=tcp -12:13.585 [INFO] TestAgent: Waiting for endpoints to shut down -12:13.585 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:28.492Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:28.533Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:28.535 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9dda006a-af04-adfe-a953-47e890958e82 Address:127.0.0.1:23895}]" +48:28.536 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9dda006a-af04-adfe-a953-47e890958e82.dc1 127.0.0.1 +48:28.536 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23895 [Follower]" leader= +48:28.540 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9dda006a-af04-adfe-a953-47e890958e82 127.0.0.1 +2022-12-04T05:48:28.540Z [INFO] agent.router: Initializing LAN area manager +48:28.541 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23890 network=udp +48:28.541 [INFO] TestAgent.server: Adding LAN server: server="Node-9dda006a-af04-adfe-a953-47e890958e82 (Addr: tcp/127.0.0.1:23895) (DC: dc1)" +48:28.541 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9dda006a-af04-adfe-a953-47e890958e82.dc1 area=wan +48:28.541 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23890 network=tcp +48:28.542 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23891 network=tcp +48:28.542 [INFO] TestAgent: started state syncer +48:28.542 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23896 network=tcp +48:28.592 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:28.592 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23895 [Candidate]" term=2 +48:28.592 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:28.592 [DEBUG] TestAgent.server.raft: vote granted: from=9dda006a-af04-adfe-a953-47e890958e82 term=2 tally=1 +48:28.592 [INFO] TestAgent.server.raft: election won: tally=1 +48:28.592 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23895 [Leader]" +48:28.592 [INFO] TestAgent.server: cluster leadership acquired +48:28.592 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23895 +48:28.593 [INFO] TestAgent.server: New leader elected: payload=Node-9dda006a-af04-adfe-a953-47e890958e82 +48:28.593 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:28.596 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:28.596 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:28.596 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:28.596 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:28.596 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:28.596 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9dda006a-af04-adfe-a953-47e890958e82 +48:28.596 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9dda006a-af04-adfe-a953-47e890958e82 +48:28.596 [INFO] TestAgent.server: federation state anti-entropy synced +48:28.664 [DEBUG] TestAgent: restored service definition from file: service=redis file=/tmp/consul-test/TestAgent_persistedService_compat_service_manager-TestAgent-agent644123397/services/86a1b907d54bf7010394bf316e183e67 +48:28.664 [INFO] TestAgent: Requesting shutdown +48:28.665 [INFO] TestAgent.server: shutting down server +48:28.665 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:28.665 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:28.665 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:28.665 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:28.665 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:28.665 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:28.665 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:28.665 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:28.665 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:28.665 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:28.665 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:28.665 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:28.665Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:28.665Z [INFO] agent.router.manager: shutting down +48:28.665 [INFO] TestAgent: consul server down +48:28.665 [INFO] TestAgent: shutdown complete +48:28.665 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23890 network=tcp +48:28.665 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23890 network=udp +48:28.665 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23891 network=tcp +48:29.168 [INFO] TestAgent: Waiting for endpoints to shut down +48:29.168 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:28.491Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:28.542Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:28.544 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a34b8376-f8ea-d835-857e-2e40aa223f42 Address:127.0.0.1:23888}]" +48:28.545 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a34b8376-f8ea-d835-857e-2e40aa223f42.dc1 127.0.0.1 +48:28.545 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a34b8376-f8ea-d835-857e-2e40aa223f42 127.0.0.1 +2022-12-04T05:48:28.545Z [INFO] agent.router: Initializing LAN area manager +48:28.546 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23883 network=udp +48:28.546 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23888 [Follower]" leader= +48:28.546 [INFO] TestAgent.server: Adding LAN server: server="Node-a34b8376-f8ea-d835-857e-2e40aa223f42 (Addr: tcp/127.0.0.1:23888) (DC: dc1)" +48:28.546 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a34b8376-f8ea-d835-857e-2e40aa223f42.dc1 area=wan +48:28.546 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23883 network=tcp +48:28.547 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23884 network=tcp +48:28.547 [INFO] TestAgent: started state syncer +48:28.547 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23889 network=tcp +48:28.592 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:28.592 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23888 [Candidate]" term=2 +48:28.592 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:28.592 [DEBUG] TestAgent.server.raft: vote granted: from=a34b8376-f8ea-d835-857e-2e40aa223f42 term=2 tally=1 +48:28.592 [INFO] TestAgent.server.raft: election won: tally=1 +48:28.592 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23888 [Leader]" +48:28.592 [INFO] TestAgent.server: cluster leadership acquired +48:28.592 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23888 +48:28.593 [INFO] TestAgent.server: New leader elected: payload=Node-a34b8376-f8ea-d835-857e-2e40aa223f42 +48:28.593 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:28.596 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:28.596 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:28.596 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:28.596 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:28.596 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:28.596 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a34b8376-f8ea-d835-857e-2e40aa223f42 +48:28.596 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a34b8376-f8ea-d835-857e-2e40aa223f42 +48:28.597 [INFO] TestAgent.server: federation state anti-entropy synced +48:28.732 [DEBUG] TestAgent: restored service definition from file: service=redis file=/tmp/consul-test/TestAgent_persistedService_compat_normal-TestAgent-agent934715582/services/86a1b907d54bf7010394bf316e183e67 +48:28.733 [INFO] TestAgent: Requesting shutdown +48:28.733 [INFO] TestAgent.server: shutting down server +48:28.733 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:28.733 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:28.733 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:28.733 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:28.733 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:28.733 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:28.733 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:28.733 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:28.733 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:28.733 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:28.733 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:28.733 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:28.733Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:28.733Z [INFO] agent.router.manager: shutting down +48:28.733 [INFO] TestAgent: consul server down +48:28.733 [INFO] TestAgent: shutdown complete +48:28.733 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23883 network=tcp +48:28.733 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23883 network=udp +48:28.733 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23884 network=tcp +48:29.234 [INFO] TestAgent: Waiting for endpoints to shut down +48:29.234 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_persistedService_compat (0.00s) - --- PASS: TestAgent_persistedService_compat/service_manager (0.98s) - --- PASS: TestAgent_persistedService_compat/normal (1.02s) + --- PASS: TestAgent_persistedService_compat/service_manager (0.72s) + --- PASS: TestAgent_persistedService_compat/normal (0.79s) === RUN TestAgent_PurgeService === RUN TestAgent_PurgeService/normal === PAUSE TestAgent_PurgeService/normal @@ -11804,126 +11827,126 @@ [WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:13.631Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:13.723Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:13.724 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c47a5724-d089-7bf6-c97c-8a7237321c44 Address:127.0.0.1:16762}]" -12:13.725 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16762 [Follower]" leader= -12:13.725 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c47a5724-d089-7bf6-c97c-8a7237321c44.dc1 127.0.0.1 -12:13.738 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c47a5724-d089-7bf6-c97c-8a7237321c44 127.0.0.1 -2021-10-31T23:12:13.738Z [INFO] agent.router: Initializing LAN area manager -12:13.738 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16757 network=udp -12:13.738 [INFO] TestAgent.server: Adding LAN server: server="Node-c47a5724-d089-7bf6-c97c-8a7237321c44 (Addr: tcp/127.0.0.1:16762) (DC: dc1)" -12:13.738 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c47a5724-d089-7bf6-c97c-8a7237321c44.dc1 area=wan -12:13.738 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16757 network=tcp -12:13.739 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16758 network=tcp -12:13.739 [INFO] TestAgent: started state syncer -12:13.739 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16763 network=tcp -12:13.777 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:13.777 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16762 [Candidate]" term=2 -12:13.778 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:13.778 [DEBUG] TestAgent.server.raft: vote granted: from=c47a5724-d089-7bf6-c97c-8a7237321c44 term=2 tally=1 -12:13.778 [INFO] TestAgent.server.raft: election won: tally=1 -12:13.778 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16762 [Leader]" -12:13.778 [INFO] TestAgent.server: cluster leadership acquired -12:13.778 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16762 -12:13.779 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:13.780 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:13.780 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:13.780 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:13.780 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:13.780 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:13.780 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c47a5724-d089-7bf6-c97c-8a7237321c44 -12:13.780 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c47a5724-d089-7bf6-c97c-8a7237321c44 -12:13.781 [INFO] TestAgent.server: New leader elected: payload=Node-c47a5724-d089-7bf6-c97c-8a7237321c44 -12:13.781 [INFO] TestAgent.server: federation state anti-entropy synced -12:13.898 [DEBUG] TestAgent: removed service: service=redis -12:13.898 [DEBUG] TestAgent: removed service: service=redis -12:13.898 [INFO] TestAgent: Requesting shutdown -12:13.898 [INFO] TestAgent.server: shutting down server -12:13.898 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:13.898 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:13.898 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:13.898 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:13.898 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:13.898 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:13.898 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:13.899 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:13.899 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:13.899 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:13.899 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:13.899 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:13.899Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:13.899Z [INFO] agent.router.manager: shutting down -12:13.899 [INFO] TestAgent: consul server down -12:13.899 [INFO] TestAgent: shutdown complete -12:13.899 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16757 network=tcp -12:13.899 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16757 network=udp -12:13.899 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16758 network=tcp -12:14.401 [INFO] TestAgent: Waiting for endpoints to shut down -12:14.401 [INFO] TestAgent: Endpoints down -2021-10-31T23:12:13.609Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:13.668Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:13.671 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c0f56109-1183-074a-77cf-4628e8d70c0c Address:127.0.0.1:16755}]" -12:13.672 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c0f56109-1183-074a-77cf-4628e8d70c0c.dc1 127.0.0.1 -12:13.672 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c0f56109-1183-074a-77cf-4628e8d70c0c 127.0.0.1 -2021-10-31T23:12:13.672Z [INFO] agent.router: Initializing LAN area manager -12:13.673 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16750 network=udp -12:13.673 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16755 [Follower]" leader= -12:13.673 [INFO] TestAgent.server: Adding LAN server: server="Node-c0f56109-1183-074a-77cf-4628e8d70c0c (Addr: tcp/127.0.0.1:16755) (DC: dc1)" -12:13.673 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c0f56109-1183-074a-77cf-4628e8d70c0c.dc1 area=wan -12:13.673 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16750 network=tcp -12:13.691 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16751 network=tcp -12:13.691 [INFO] TestAgent: started state syncer -12:13.691 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16756 network=tcp -12:13.720 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:13.721 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16755 [Candidate]" term=2 -12:13.721 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:13.721 [DEBUG] TestAgent.server.raft: vote granted: from=c0f56109-1183-074a-77cf-4628e8d70c0c term=2 tally=1 -12:13.721 [INFO] TestAgent.server.raft: election won: tally=1 -12:13.721 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16755 [Leader]" -12:13.721 [INFO] TestAgent.server: cluster leadership acquired -12:13.721 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16755 -12:13.721 [INFO] TestAgent.server: New leader elected: payload=Node-c0f56109-1183-074a-77cf-4628e8d70c0c -12:13.734 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:13.736 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:13.736 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:13.736 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:13.736 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:13.736 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:13.736 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c0f56109-1183-074a-77cf-4628e8d70c0c -12:13.736 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c0f56109-1183-074a-77cf-4628e8d70c0c -12:13.737 [INFO] TestAgent.server: federation state anti-entropy synced -12:13.746 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:13.746 [INFO] TestAgent: Synced node info -12:13.746 [DEBUG] TestAgent: Node info in sync -12:13.902 [DEBUG] TestAgent: removed service: service=redis -12:13.902 [DEBUG] TestAgent: removed service: service=redis -12:13.902 [INFO] TestAgent: Requesting shutdown -12:13.902 [INFO] TestAgent.server: shutting down server -12:13.902 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:13.902 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:13.902 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:13.902 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:13.902 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:13.902 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:13.902 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:13.902 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:13.902 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:13.902 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:13.902 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:13.903Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:13.903Z [INFO] agent.router.manager: shutting down -12:13.903 [INFO] TestAgent: consul server down -12:13.903 [INFO] TestAgent: shutdown complete -12:13.903 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16750 network=tcp -12:13.903 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16750 network=udp -12:13.903 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16751 network=tcp -12:14.405 [INFO] TestAgent: Waiting for endpoints to shut down -12:14.405 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:29.254Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:29.270Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:29.273 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e3c93ac5-fe17-8174-2ba4-4f7e2259c932 Address:127.0.0.1:23902}]" +48:29.281 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e3c93ac5-fe17-8174-2ba4-4f7e2259c932.dc1 127.0.0.1 +48:29.282 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e3c93ac5-fe17-8174-2ba4-4f7e2259c932 127.0.0.1 +2022-12-04T05:48:29.282Z [INFO] agent.router: Initializing LAN area manager +48:29.283 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23897 network=udp +48:29.284 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23902 [Follower]" leader= +48:29.284 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e3c93ac5-fe17-8174-2ba4-4f7e2259c932.dc1 area=wan +48:29.285 [INFO] TestAgent.server: Adding LAN server: server="Node-e3c93ac5-fe17-8174-2ba4-4f7e2259c932 (Addr: tcp/127.0.0.1:23902) (DC: dc1)" +48:29.285 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23897 network=tcp +48:29.287 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23898 network=tcp +48:29.287 [INFO] TestAgent: started state syncer +48:29.287 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23903 network=tcp +48:29.347 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:29.347 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23902 [Candidate]" term=2 +48:29.347 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:29.347 [DEBUG] TestAgent.server.raft: vote granted: from=e3c93ac5-fe17-8174-2ba4-4f7e2259c932 term=2 tally=1 +48:29.347 [INFO] TestAgent.server.raft: election won: tally=1 +48:29.347 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23902 [Leader]" +48:29.347 [INFO] TestAgent.server: cluster leadership acquired +48:29.347 [INFO] TestAgent.server: New leader elected: payload=Node-e3c93ac5-fe17-8174-2ba4-4f7e2259c932 +48:29.348 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23902 +48:29.348 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:29.350 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:29.350 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:29.350 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:29.350 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:29.350 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:29.350 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e3c93ac5-fe17-8174-2ba4-4f7e2259c932 +48:29.350 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e3c93ac5-fe17-8174-2ba4-4f7e2259c932 +48:29.351 [INFO] TestAgent.server: federation state anti-entropy synced +48:29.457 [DEBUG] TestAgent: removed service: service=redis +48:29.457 [DEBUG] TestAgent: removed service: service=redis +48:29.457 [INFO] TestAgent: Requesting shutdown +48:29.457 [INFO] TestAgent.server: shutting down server +48:29.457 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:29.457 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:29.457 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:29.457 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:29.457 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:29.457 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:29.457 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:29.457 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:29.458 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +2022-12-04T05:48:29.458Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:29.458Z [INFO] agent.router.manager: shutting down +48:29.458 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:29.458 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:29.458 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:29.458 [INFO] TestAgent: consul server down +48:29.458 [INFO] TestAgent: shutdown complete +48:29.458 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23897 network=tcp +48:29.458 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23897 network=udp +48:29.458 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23898 network=tcp +48:29.958 [INFO] TestAgent: Waiting for endpoints to shut down +48:29.958 [INFO] TestAgent: Endpoints down +[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:48:29.260Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:29.289Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:29.295 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:562d18dc-8d3a-286b-c598-bcf90d5bc9f4 Address:127.0.0.1:23909}]" +48:29.295 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-562d18dc-8d3a-286b-c598-bcf90d5bc9f4.dc1 127.0.0.1 +48:29.295 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23909 [Follower]" leader= +48:29.296 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-562d18dc-8d3a-286b-c598-bcf90d5bc9f4 127.0.0.1 +2022-12-04T05:48:29.296Z [INFO] agent.router: Initializing LAN area manager +48:29.296 [INFO] TestAgent.server: Adding LAN server: server="Node-562d18dc-8d3a-286b-c598-bcf90d5bc9f4 (Addr: tcp/127.0.0.1:23909) (DC: dc1)" +48:29.296 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-562d18dc-8d3a-286b-c598-bcf90d5bc9f4.dc1 area=wan +48:29.296 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23904 network=udp +48:29.296 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23904 network=tcp +48:29.297 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23905 network=tcp +48:29.297 [INFO] TestAgent: started state syncer +48:29.297 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23910 network=tcp +48:29.339 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:29.339 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23909 [Candidate]" term=2 +48:29.339 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:29.339 [DEBUG] TestAgent.server.raft: vote granted: from=562d18dc-8d3a-286b-c598-bcf90d5bc9f4 term=2 tally=1 +48:29.339 [INFO] TestAgent.server.raft: election won: tally=1 +48:29.339 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23909 [Leader]" +48:29.339 [INFO] TestAgent.server: cluster leadership acquired +48:29.339 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23909 +48:29.340 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:29.341 [INFO] TestAgent.server: New leader elected: payload=Node-562d18dc-8d3a-286b-c598-bcf90d5bc9f4 +48:29.342 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:29.342 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:29.342 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:29.342 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:29.342 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:29.342 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-562d18dc-8d3a-286b-c598-bcf90d5bc9f4 +48:29.342 [INFO] TestAgent.server: member joined, marking health alive: member=Node-562d18dc-8d3a-286b-c598-bcf90d5bc9f4 +48:29.343 [INFO] TestAgent.server: federation state anti-entropy synced +48:29.349 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:29.350 [INFO] TestAgent: Synced node info +48:29.350 [DEBUG] TestAgent: Node info in sync +48:29.649 [DEBUG] TestAgent: removed service: service=redis +48:29.649 [DEBUG] TestAgent: removed service: service=redis +48:29.649 [INFO] TestAgent: Requesting shutdown +48:29.649 [INFO] TestAgent.server: shutting down server +48:29.649 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:29.649 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:29.649 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:29.649 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:29.649 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:29.650 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:29.650 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:29.650 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:29.650 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:29.650 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:29.650 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:29.650Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:29.650Z [INFO] agent.router.manager: shutting down +48:29.650 [INFO] TestAgent: consul server down +48:29.650 [INFO] TestAgent: shutdown complete +48:29.650 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23904 network=tcp +48:29.650 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23904 network=udp +48:29.650 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23905 network=tcp +48:30.150 [INFO] TestAgent: Waiting for endpoints to shut down +48:30.150 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_PurgeService (0.00s) - --- PASS: TestAgent_PurgeService/service_manager (0.81s) - --- PASS: TestAgent_PurgeService/normal (0.82s) + --- PASS: TestAgent_PurgeService/normal (0.72s) + --- PASS: TestAgent_PurgeService/service_manager (0.92s) === RUN TestAgent_PurgeServiceOnDuplicate === RUN TestAgent_PurgeServiceOnDuplicate/normal === PAUSE TestAgent_PurgeServiceOnDuplicate/normal @@ -11931,101 +11954,99 @@ === PAUSE TestAgent_PurgeServiceOnDuplicate/service_manager === CONT TestAgent_PurgeServiceOnDuplicate/normal === CONT TestAgent_PurgeServiceOnDuplicate/service_manager -[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -12:15.087 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node-615dac49-b213-3821-076d-4e0445cb2c11 127.0.0.1 -2021-10-31T23:12:15.087Z [INFO] agent.router: Initializing LAN area manager -12:15.087 [DEBUG] Agent2: service exists, not restoring from file: service=redis file=/tmp/consul-test/TestAgent_PurgeServiceOnDuplicate_normal-TestAgent-agent267328267/services/86a1b907d54bf7010394bf316e183e67 -12:15.087 [INFO] Agent2: Started DNS server: address=127.0.0.1:16778 network=udp -12:15.087 [INFO] Agent2.client.serf.lan: serf: Attempting re-join to previously known node: Node-497cd1c7-76d6-cdf8-196e-48e2043e6797: 127.0.0.1:16767 -12:15.088 [INFO] Agent2: Started DNS server: address=127.0.0.1:16778 network=tcp -12:15.088 [INFO] Agent2: Started HTTP server: address=127.0.0.1:16779 network=tcp -12:15.088 [INFO] Agent2: started state syncer -12:15.089 [INFO] Agent2: Requesting shutdown -12:15.089 [INFO] Agent2.client: shutting down client -12:15.089 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave -12:15.089 [INFO] Agent2: Started gRPC server: address=127.0.0.1:16784 network=tcp -12:15.089 [ERROR] Agent2: gRPC server failed: error="grpc: the server has been stopped" -2021-10-31T23:12:15.089Z [WARN] agent.router.manager: No servers available -12:15.089 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" -12:15.089 [INFO] Agent2: consul client down -12:15.089 [INFO] Agent2: shutdown complete -12:15.089 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:16778 network=tcp -12:15.089 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:16778 network=udp -12:15.089 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:16779 network=tcp -12:15.090 [DEBUG] Agent2.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:16767: connect: connection refused -12:15.090 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node -12:15.589 [INFO] Agent2: Waiting for endpoints to shut down -12:15.589 [INFO] Agent2: Endpoints down -12:14.479 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-497cd1c7-76d6-cdf8-196e-48e2043e6797 127.0.0.1 -2021-10-31T23:12:14.479Z [INFO] agent.router: Initializing LAN area manager -12:14.479 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16764 network=udp -12:14.479 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16764 network=tcp -12:14.480 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16765 network=tcp -12:14.480 [INFO] TestAgent: started state syncer -12:14.480 [INFO] TestAgent: Requesting shutdown -12:14.480 [INFO] TestAgent.client: shutting down client -12:14.480 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -12:14.480 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16770 network=tcp -12:14.480 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -2021-10-31T23:12:14.480Z [WARN] agent.router.manager: No servers available -12:14.480 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -12:14.480 [INFO] TestAgent: consul client down -12:14.480 [INFO] TestAgent: shutdown complete -12:14.480 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16764 network=tcp -12:14.480 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16764 network=udp -12:14.480 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16765 network=tcp -12:14.983 [INFO] TestAgent: Waiting for endpoints to shut down -12:14.983 [INFO] TestAgent: Endpoints down -[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! -12:15.139 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node-3531d385-d4d3-a4ad-f8dc-81061f886433 127.0.0.1 -2021-10-31T23:12:15.139Z [INFO] agent.router: Initializing LAN area manager -12:15.139 [DEBUG] Agent2: service exists, not restoring from file: service=redis file=/tmp/consul-test/TestAgent_PurgeServiceOnDuplicate_service_manager-TestAgent-agent100764910/services/86a1b907d54bf7010394bf316e183e67 -12:15.140 [INFO] Agent2: Started DNS server: address=127.0.0.1:16785 network=udp -12:15.140 [INFO] Agent2.client.serf.lan: serf: Attempting re-join to previously known node: Node-e7e5eeb7-7b5b-f4a1-6785-5e742b9ca9fc: 127.0.0.1:16774 -12:15.140 [INFO] Agent2: Started DNS server: address=127.0.0.1:16785 network=tcp -12:15.141 [INFO] Agent2: Started HTTP server: address=127.0.0.1:16786 network=tcp -12:15.141 [INFO] Agent2: started state syncer -12:15.141 [INFO] Agent2: Started gRPC server: address=127.0.0.1:16791 network=tcp -12:15.142 [DEBUG] Agent2.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:16774: connect: connection refused -12:15.142 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node -2021-10-31T23:12:15.145Z [WARN] agent.router.manager: No servers available -12:15.145 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" -12:15.150 [INFO] Agent2: Requesting shutdown -12:15.150 [INFO] Agent2.client: shutting down client -12:15.150 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave -12:15.150 [INFO] Agent2: consul client down -12:15.150 [INFO] Agent2: shutdown complete -12:15.150 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:16785 network=tcp -12:15.150 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:16785 network=udp -12:15.150 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:16786 network=tcp -12:15.653 [INFO] Agent2: Waiting for endpoints to shut down -12:15.653 [INFO] Agent2: Endpoints down -12:14.498 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-e7e5eeb7-7b5b-f4a1-6785-5e742b9ca9fc 127.0.0.1 -2021-10-31T23:12:14.498Z [INFO] agent.router: Initializing LAN area manager -12:14.498 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16771 network=udp -12:14.498 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16771 network=tcp -12:14.499 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16772 network=tcp -12:14.499 [INFO] TestAgent: started state syncer -12:14.500 [INFO] TestAgent: Requesting shutdown -12:14.500 [INFO] TestAgent.client: shutting down client -12:14.500 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -12:14.500 [INFO] TestAgent: consul client down -12:14.500 [INFO] TestAgent: shutdown complete -12:14.500 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16771 network=tcp -12:14.500 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16771 network=udp -12:14.500 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16772 network=tcp -12:14.500 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16777 network=tcp -12:14.500 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -2021-10-31T23:12:14.500Z [WARN] agent.router.manager: No servers available -12:14.500 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -12:15.001 [INFO] TestAgent: Waiting for endpoints to shut down -12:15.001 [INFO] TestAgent: Endpoints down +48:30.729 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node-48dd1fa4-a147-c1c4-7bf1-c0fd30cf93d3 127.0.0.1 +2022-12-04T05:48:30.729Z [INFO] agent.router: Initializing LAN area manager +48:30.729 [DEBUG] Agent2: service exists, not restoring from file: service=redis file=/tmp/consul-test/TestAgent_PurgeServiceOnDuplicate_normal-TestAgent-agent116592850/services/86a1b907d54bf7010394bf316e183e67 +48:30.729 [INFO] Agent2: Started DNS server: address=127.0.0.1:23925 network=udp +48:30.729 [INFO] Agent2.client.serf.lan: serf: Attempting re-join to previously known node: Node-58c4aa5b-6a5b-3375-1c44-81240caf0674: 127.0.0.1:23914 +48:30.730 [INFO] Agent2: Started DNS server: address=127.0.0.1:23925 network=tcp +48:30.730 [INFO] Agent2: Started HTTP server: address=127.0.0.1:23926 network=tcp +48:30.730 [INFO] Agent2: started state syncer +48:30.731 [INFO] Agent2: Requesting shutdown +48:30.731 [INFO] Agent2.client: shutting down client +48:30.731 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave +48:30.731 [INFO] Agent2: Started gRPC server: address=127.0.0.1:23931 network=tcp +48:30.731 [ERROR] Agent2: gRPC server failed: error="grpc: the server has been stopped" +2022-12-04T05:48:30.731Z [WARN] agent.router.manager: No servers available +48:30.731 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" +48:30.731 [INFO] Agent2: consul client down +48:30.731 [INFO] Agent2: shutdown complete +48:30.731 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:23925 network=tcp +48:30.731 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:23925 network=udp +48:30.731 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:23926 network=tcp +48:30.731 [DEBUG] Agent2.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:23914: connect: connection refused +48:30.731 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node +48:31.231 [INFO] Agent2: Waiting for endpoints to shut down +48:31.231 [INFO] Agent2: Endpoints down +48:30.179 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-58c4aa5b-6a5b-3375-1c44-81240caf0674 127.0.0.1 +2022-12-04T05:48:30.179Z [INFO] agent.router: Initializing LAN area manager +48:30.179 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23911 network=udp +48:30.180 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23911 network=tcp +48:30.180 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23912 network=tcp +48:30.180 [INFO] TestAgent: started state syncer +48:30.181 [INFO] TestAgent: Requesting shutdown +48:30.181 [INFO] TestAgent.client: shutting down client +48:30.181 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +48:30.181 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23917 network=tcp +48:30.181 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" +2022-12-04T05:48:30.181Z [WARN] agent.router.manager: No servers available +48:30.181 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +48:30.181 [INFO] TestAgent: consul client down +48:30.181 [INFO] TestAgent: shutdown complete +48:30.181 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23911 network=tcp +48:30.181 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23911 network=udp +48:30.181 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23912 network=tcp +48:30.681 [INFO] TestAgent: Waiting for endpoints to shut down +48:30.681 [INFO] TestAgent: Endpoints down +48:30.736 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node-74602b06-5d78-be00-99aa-ad8709770a97 127.0.0.1 +2022-12-04T05:48:30.736Z [INFO] agent.router: Initializing LAN area manager +48:30.736 [DEBUG] Agent2: service exists, not restoring from file: service=redis file=/tmp/consul-test/TestAgent_PurgeServiceOnDuplicate_service_manager-TestAgent-agent004314633/services/86a1b907d54bf7010394bf316e183e67 +48:30.736 [INFO] Agent2.client.serf.lan: serf: Attempting re-join to previously known node: Node-7e25a747-9864-6c7d-65bf-127343dbfc99: 127.0.0.1:23921 +48:30.736 [INFO] Agent2: Started DNS server: address=127.0.0.1:23932 network=udp +48:30.736 [INFO] Agent2: Started DNS server: address=127.0.0.1:23932 network=tcp +48:30.737 [INFO] Agent2: Started HTTP server: address=127.0.0.1:23933 network=tcp +48:30.737 [INFO] Agent2: started state syncer +48:30.737 [INFO] Agent2: Started gRPC server: address=127.0.0.1:23938 network=tcp +2022-12-04T05:48:30.737Z [WARN] agent.router.manager: No servers available +48:30.737 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" +48:30.737 [INFO] Agent2: Requesting shutdown +48:30.737 [INFO] Agent2.client: shutting down client +48:30.737 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave +48:30.738 [INFO] Agent2: consul client down +48:30.738 [INFO] Agent2: shutdown complete +48:30.738 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:23932 network=tcp +48:30.738 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:23932 network=udp +48:30.738 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:23933 network=tcp +48:30.738 [DEBUG] Agent2.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:23921: connect: connection refused +48:30.738 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node +48:31.238 [INFO] Agent2: Waiting for endpoints to shut down +48:31.238 [INFO] Agent2: Endpoints down +48:30.180 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-7e25a747-9864-6c7d-65bf-127343dbfc99 127.0.0.1 +2022-12-04T05:48:30.180Z [INFO] agent.router: Initializing LAN area manager +48:30.180 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23918 network=udp +48:30.180 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23918 network=tcp +48:30.181 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23919 network=tcp +48:30.181 [INFO] TestAgent: started state syncer +48:30.181 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23924 network=tcp +48:30.181 [INFO] TestAgent: Requesting shutdown +48:30.181 [INFO] TestAgent.client: shutting down client +48:30.181 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +2022-12-04T05:48:30.181Z [WARN] agent.router.manager: No servers available +48:30.181 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +48:30.182 [INFO] TestAgent: consul client down +48:30.182 [INFO] TestAgent: shutdown complete +48:30.182 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23918 network=tcp +48:30.182 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23918 network=udp +48:30.182 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23919 network=tcp +48:30.686 [INFO] TestAgent: Waiting for endpoints to shut down +48:30.686 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_PurgeServiceOnDuplicate (0.00s) - --- PASS: TestAgent_PurgeServiceOnDuplicate/normal (1.18s) - --- PASS: TestAgent_PurgeServiceOnDuplicate/service_manager (1.24s) + --- PASS: TestAgent_PurgeServiceOnDuplicate/normal (1.08s) + --- PASS: TestAgent_PurgeServiceOnDuplicate/service_manager (1.08s) === RUN TestAgent_PersistCheck === PAUSE TestAgent_PersistCheck === RUN TestAgent_PurgeCheck @@ -12045,122 +12066,125 @@ === PAUSE TestAgent_loadServices_token/service_manager === CONT TestAgent_loadServices_token/normal === CONT TestAgent_loadServices_token/service_manager -[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 34 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:15.697Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:15.761Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:15.768 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:98bba118-363a-d71c-f4dd-dd52cb8c6bc9 Address:127.0.0.1:16797}]" -12:15.773 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16797 [Follower]" leader= -12:15.799 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-98bba118-363a-d71c-f4dd-dd52cb8c6bc9.dc1 127.0.0.1 -12:15.800 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-98bba118-363a-d71c-f4dd-dd52cb8c6bc9 127.0.0.1 -2021-10-31T23:12:15.800Z [INFO] agent.router: Initializing LAN area manager -12:15.800 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16792 network=udp -12:15.800 [INFO] TestAgent.server: Adding LAN server: server="Node-98bba118-363a-d71c-f4dd-dd52cb8c6bc9 (Addr: tcp/127.0.0.1:16797) (DC: dc1)" -12:15.800 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-98bba118-363a-d71c-f4dd-dd52cb8c6bc9.dc1 area=wan -12:15.800 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16792 network=tcp -12:15.801 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16793 network=tcp -12:15.801 [INFO] TestAgent: started state syncer -12:15.801 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16798 network=tcp -12:15.833 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:15.834 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16797 [Candidate]" term=2 -12:15.834 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:15.834 [DEBUG] TestAgent.server.raft: vote granted: from=98bba118-363a-d71c-f4dd-dd52cb8c6bc9 term=2 tally=1 -12:15.834 [INFO] TestAgent.server.raft: election won: tally=1 -12:15.834 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16797 [Leader]" -12:15.834 [INFO] TestAgent.server: cluster leadership acquired -12:15.834 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16797 -12:15.835 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:15.836 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:15.836 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:15.836 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:15.836 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:15.836 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:15.836 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-98bba118-363a-d71c-f4dd-dd52cb8c6bc9 -12:15.836 [INFO] TestAgent.server: member joined, marking health alive: member=Node-98bba118-363a-d71c-f4dd-dd52cb8c6bc9 -12:15.837 [INFO] TestAgent.server: New leader elected: payload=Node-98bba118-363a-d71c-f4dd-dd52cb8c6bc9 -12:15.837 [INFO] TestAgent.server: federation state anti-entropy synced -12:15.852 [INFO] TestAgent: Requesting shutdown -12:15.852 [INFO] TestAgent.server: shutting down server -12:15.852 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:15.852 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:15.852 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:15.852 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:15.852 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:15.852 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:15.852 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:15.852 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:15.852 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:15.852 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:15.852 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:15.852 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:15.852Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:15.852Z [INFO] agent.router.manager: shutting down -12:15.852 [INFO] TestAgent: consul server down -12:15.852 [INFO] TestAgent: shutdown complete -12:15.852 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16792 network=tcp -12:15.852 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16792 network=udp -12:15.852 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16793 network=tcp -12:16.353 [INFO] TestAgent: Waiting for endpoints to shut down -12:16.353 [INFO] TestAgent: Endpoints down +[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:48:31.249Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:31.259Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:31.260 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:98227227-5900-6201-e2d4-f6102affe9ad Address:127.0.0.1:23951}]" +48:31.260 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23951 [Follower]" leader= +48:31.261 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-98227227-5900-6201-e2d4-f6102affe9ad.dc1 127.0.0.1 +48:31.261 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-98227227-5900-6201-e2d4-f6102affe9ad 127.0.0.1 +2022-12-04T05:48:31.261Z [INFO] agent.router: Initializing LAN area manager +48:31.261 [INFO] TestAgent.server: Adding LAN server: server="Node-98227227-5900-6201-e2d4-f6102affe9ad (Addr: tcp/127.0.0.1:23951) (DC: dc1)" +48:31.261 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-98227227-5900-6201-e2d4-f6102affe9ad.dc1 area=wan +48:31.261 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23946 network=udp +48:31.261 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23946 network=tcp +48:31.262 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23947 network=tcp +48:31.262 [INFO] TestAgent: started state syncer +48:31.262 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23952 network=tcp +48:31.314 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:31.314 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23951 [Candidate]" term=2 +48:31.314 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:31.314 [DEBUG] TestAgent.server.raft: vote granted: from=98227227-5900-6201-e2d4-f6102affe9ad term=2 tally=1 +48:31.314 [INFO] TestAgent.server.raft: election won: tally=1 +48:31.314 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23951 [Leader]" +48:31.314 [INFO] TestAgent.server: cluster leadership acquired +48:31.314 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23951 +48:31.314 [INFO] TestAgent.server: New leader elected: payload=Node-98227227-5900-6201-e2d4-f6102affe9ad +48:31.315 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:31.316 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:31.316 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:31.316 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:31.316 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:31.316 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:31.316 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-98227227-5900-6201-e2d4-f6102affe9ad +48:31.316 [INFO] TestAgent.server: member joined, marking health alive: member=Node-98227227-5900-6201-e2d4-f6102affe9ad +48:31.316 [INFO] TestAgent.server: federation state anti-entropy synced +48:31.344 [INFO] TestAgent: Requesting shutdown +48:31.344 [INFO] TestAgent.server: shutting down server +48:31.344 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:31.344 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:31.344 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:31.344 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:31.344 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:31.344 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:31.344 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:31.344 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:31.344 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:31.344 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:31.344 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:31.344 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:48:31.344Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:31.344Z [INFO] agent.router.manager: shutting down +48:31.344 [INFO] TestAgent: consul server down +48:31.344 [INFO] TestAgent: shutdown complete +48:31.344 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23946 network=tcp +48:31.344 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23946 network=udp +48:31.344 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23947 network=tcp +48:31.844 [INFO] TestAgent: Waiting for endpoints to shut down +48:31.844 [INFO] TestAgent: Endpoints down [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:15.694Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:15.740Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:15.763 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:172eaed8-6ee1-2309-2c77-2e1fc8fda6a2 Address:127.0.0.1:16804}]" -12:15.764 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-172eaed8-6ee1-2309-2c77-2e1fc8fda6a2.dc1 127.0.0.1 -12:15.765 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-172eaed8-6ee1-2309-2c77-2e1fc8fda6a2 127.0.0.1 -2021-10-31T23:12:15.765Z [INFO] agent.router: Initializing LAN area manager -12:15.765 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16799 network=udp -12:15.765 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16804 [Follower]" leader= -12:15.766 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-172eaed8-6ee1-2309-2c77-2e1fc8fda6a2.dc1 area=wan -12:15.766 [INFO] TestAgent.server: Adding LAN server: server="Node-172eaed8-6ee1-2309-2c77-2e1fc8fda6a2 (Addr: tcp/127.0.0.1:16804) (DC: dc1)" -12:15.766 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16799 network=tcp -12:15.767 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16800 network=tcp -12:15.768 [INFO] TestAgent: started state syncer -12:15.768 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16805 network=tcp -12:15.821 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:15.821 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16804 [Candidate]" term=2 -12:15.822 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:15.822 [DEBUG] TestAgent.server.raft: vote granted: from=172eaed8-6ee1-2309-2c77-2e1fc8fda6a2 term=2 tally=1 -12:15.822 [INFO] TestAgent.server.raft: election won: tally=1 -12:15.822 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16804 [Leader]" -12:15.822 [INFO] TestAgent.server: cluster leadership acquired -12:15.822 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16804 -12:15.823 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:15.824 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:15.824 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:15.824 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:15.824 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:15.824 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:15.824 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-172eaed8-6ee1-2309-2c77-2e1fc8fda6a2 -12:15.824 [INFO] TestAgent.server: member joined, marking health alive: member=Node-172eaed8-6ee1-2309-2c77-2e1fc8fda6a2 -12:15.824 [INFO] TestAgent.server: New leader elected: payload=Node-172eaed8-6ee1-2309-2c77-2e1fc8fda6a2 -12:15.824 [INFO] TestAgent.server: federation state anti-entropy synced -12:15.985 [INFO] TestAgent: Requesting shutdown -12:15.986 [INFO] TestAgent.server: shutting down server -12:15.986 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:15.986 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:15.986 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:15.986 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:15.986 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:15.986 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:15.986 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:15.986 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:15.986 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:15.986 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:15.986 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:15.986 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:15.986Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:15.986Z [INFO] agent.router.manager: shutting down -12:15.986 [INFO] TestAgent: consul server down -12:15.986 [INFO] TestAgent: shutdown complete -12:15.986 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16799 network=tcp -12:15.986 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16799 network=udp -12:15.986 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16800 network=tcp -12:16.490 [INFO] TestAgent: Waiting for endpoints to shut down -12:16.490 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:31.251Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:31.265Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:31.266 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:75d02f7f-30d5-1cf4-d42f-b170d8dfb4b7 Address:127.0.0.1:23944}]" +48:31.266 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23944 [Follower]" leader= +48:31.267 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-75d02f7f-30d5-1cf4-d42f-b170d8dfb4b7.dc1 127.0.0.1 +48:31.267 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-75d02f7f-30d5-1cf4-d42f-b170d8dfb4b7 127.0.0.1 +2022-12-04T05:48:31.267Z [INFO] agent.router: Initializing LAN area manager +48:31.268 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23939 network=udp +48:31.268 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-75d02f7f-30d5-1cf4-d42f-b170d8dfb4b7.dc1 area=wan +48:31.268 [INFO] TestAgent.server: Adding LAN server: server="Node-75d02f7f-30d5-1cf4-d42f-b170d8dfb4b7 (Addr: tcp/127.0.0.1:23944) (DC: dc1)" +48:31.268 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23939 network=tcp +48:31.268 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23940 network=tcp +48:31.268 [INFO] TestAgent: started state syncer +48:31.268 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23945 network=tcp +48:31.311 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:31.311 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23944 [Candidate]" term=2 +48:31.311 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:31.311 [DEBUG] TestAgent.server.raft: vote granted: from=75d02f7f-30d5-1cf4-d42f-b170d8dfb4b7 term=2 tally=1 +48:31.311 [INFO] TestAgent.server.raft: election won: tally=1 +48:31.311 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23944 [Leader]" +48:31.311 [INFO] TestAgent.server: cluster leadership acquired +48:31.311 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23944 +48:31.311 [INFO] TestAgent.server: New leader elected: payload=Node-75d02f7f-30d5-1cf4-d42f-b170d8dfb4b7 +48:31.312 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:31.313 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:31.313 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:31.313 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:31.313 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:31.313 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:31.313 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-75d02f7f-30d5-1cf4-d42f-b170d8dfb4b7 +48:31.313 [INFO] TestAgent.server: member joined, marking health alive: member=Node-75d02f7f-30d5-1cf4-d42f-b170d8dfb4b7 +48:31.314 [INFO] TestAgent.server: federation state anti-entropy synced +48:31.464 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:31.464 [INFO] TestAgent: Synced node info +48:31.464 [INFO] TestAgent: Synced service: service=rabbitmq +48:31.647 [INFO] TestAgent: Requesting shutdown +48:31.647 [INFO] TestAgent.server: shutting down server +48:31.647 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:31.647 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:31.647 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:31.647 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:31.647 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:31.648 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:31.648 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:31.648 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:31.648 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:31.648 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:31.648 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:31.648Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:31.648Z [INFO] agent.router.manager: shutting down +48:31.648 [INFO] TestAgent: consul server down +48:31.648 [INFO] TestAgent: shutdown complete +48:31.648 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23939 network=tcp +48:31.648 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23939 network=udp +48:31.648 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23940 network=tcp +48:32.149 [INFO] TestAgent: Waiting for endpoints to shut down +48:32.149 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_loadServices_token (0.00s) - --- PASS: TestAgent_loadServices_token/normal (0.70s) - --- PASS: TestAgent_loadServices_token/service_manager (0.84s) + --- PASS: TestAgent_loadServices_token/service_manager (0.61s) + --- PASS: TestAgent_loadServices_token/normal (0.91s) === RUN TestAgent_loadServices_sidecar === RUN TestAgent_loadServices_sidecar/normal === PAUSE TestAgent_loadServices_sidecar/normal @@ -12172,125 +12196,142 @@ [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:16.544Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:16.593Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:16.615 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:79ba4cb6-37d9-5970-83fd-464230fc949e Address:127.0.0.1:16811}]" -12:16.615 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-79ba4cb6-37d9-5970-83fd-464230fc949e.dc1 127.0.0.1 -12:16.616 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-79ba4cb6-37d9-5970-83fd-464230fc949e 127.0.0.1 -2021-10-31T23:12:16.616Z [INFO] agent.router: Initializing LAN area manager -12:16.616 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16806 network=udp -12:16.616 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16811 [Follower]" leader= -12:16.626 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16806 network=tcp -12:16.628 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16807 network=tcp -12:16.628 [INFO] TestAgent: started state syncer -12:16.628 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16812 network=tcp -12:16.628 [INFO] TestAgent.server: Adding LAN server: server="Node-79ba4cb6-37d9-5970-83fd-464230fc949e (Addr: tcp/127.0.0.1:16811) (DC: dc1)" -12:16.628 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-79ba4cb6-37d9-5970-83fd-464230fc949e.dc1 area=wan -12:16.673 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:16.673 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16811 [Candidate]" term=2 -12:16.674 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:16.674 [DEBUG] TestAgent.server.raft: vote granted: from=79ba4cb6-37d9-5970-83fd-464230fc949e term=2 tally=1 -12:16.674 [INFO] TestAgent.server.raft: election won: tally=1 -12:16.674 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16811 [Leader]" -12:16.674 [INFO] TestAgent.server: cluster leadership acquired -12:16.674 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16811 -12:16.675 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:16.676 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:16.676 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:16.676 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:16.676 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:16.676 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:16.676 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-79ba4cb6-37d9-5970-83fd-464230fc949e -12:16.676 [INFO] TestAgent.server: member joined, marking health alive: member=Node-79ba4cb6-37d9-5970-83fd-464230fc949e -12:16.677 [INFO] TestAgent.server: New leader elected: payload=Node-79ba4cb6-37d9-5970-83fd-464230fc949e -12:16.677 [INFO] TestAgent.server: federation state anti-entropy synced -12:16.858 [INFO] TestAgent: Requesting shutdown -12:16.858 [INFO] TestAgent.server: shutting down server -12:16.858 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:16.858 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:16.858 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:16.858 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:16.858 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:16.858 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:16.858 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:16.858 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:16.858 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:16.858 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:16.858 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:16.858 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:16.858Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:16.858Z [INFO] agent.router.manager: shutting down -12:16.858 [INFO] TestAgent: consul server down -12:16.858 [INFO] TestAgent: shutdown complete -12:16.858 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16806 network=tcp -12:16.858 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16806 network=udp -12:16.858 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16807 network=tcp -12:17.361 [INFO] TestAgent: Waiting for endpoints to shut down -12:17.361 [INFO] TestAgent: Endpoints down -2021-10-31T23:12:16.532Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:16.577Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:16.595 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3992a6bf-dc78-4578-d3ef-3204cafb93a9 Address:127.0.0.1:16818}]" -12:16.596 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3992a6bf-dc78-4578-d3ef-3204cafb93a9.dc1 127.0.0.1 -12:16.596 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3992a6bf-dc78-4578-d3ef-3204cafb93a9 127.0.0.1 -2021-10-31T23:12:16.596Z [INFO] agent.router: Initializing LAN area manager -12:16.597 [DEBUG] TestAgent: added local registration for service: service=rabbitmq-sidecar-proxy -12:16.597 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16813 network=udp -12:16.597 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16818 [Follower]" leader= -12:16.597 [INFO] TestAgent.server: Adding LAN server: server="Node-3992a6bf-dc78-4578-d3ef-3204cafb93a9 (Addr: tcp/127.0.0.1:16818) (DC: dc1)" -12:16.597 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3992a6bf-dc78-4578-d3ef-3204cafb93a9.dc1 area=wan -12:16.598 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16813 network=tcp -12:16.598 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16814 network=tcp -12:16.598 [INFO] TestAgent: started state syncer -12:16.598 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16819 network=tcp -12:16.668 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:16.668 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16818 [Candidate]" term=2 -12:16.668 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:16.668 [DEBUG] TestAgent.server.raft: vote granted: from=3992a6bf-dc78-4578-d3ef-3204cafb93a9 term=2 tally=1 -12:16.668 [INFO] TestAgent.server.raft: election won: tally=1 -12:16.668 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16818 [Leader]" -12:16.668 [INFO] TestAgent.server: cluster leadership acquired -12:16.668 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16818 -12:16.669 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:16.671 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:16.671 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:16.671 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:16.671 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:16.671 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:16.671 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3992a6bf-dc78-4578-d3ef-3204cafb93a9 -12:16.671 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3992a6bf-dc78-4578-d3ef-3204cafb93a9 -12:16.672 [INFO] TestAgent.server: New leader elected: payload=Node-3992a6bf-dc78-4578-d3ef-3204cafb93a9 -12:16.672 [INFO] TestAgent.server: federation state anti-entropy synced -12:16.955 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:16.955 [INFO] TestAgent: Synced node info -12:16.955 [INFO] TestAgent: Synced service: service=rabbitmq -12:16.956 [INFO] TestAgent: Synced service: service=rabbitmq-sidecar-proxy -12:16.956 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 -12:16.956 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 -12:16.961 [INFO] TestAgent: Requesting shutdown -12:16.962 [INFO] TestAgent.server: shutting down server -12:16.962 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:16.962 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:16.962 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:16.962 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:16.962 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:16.962 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:16.962 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:16.962 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:16.962 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:16.962 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:16.962 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:16.962Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:16.962Z [INFO] agent.router.manager: shutting down -12:16.962 [INFO] TestAgent: consul server down -12:16.962 [INFO] TestAgent: shutdown complete -12:16.962 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16813 network=tcp -12:16.962 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16813 network=udp -12:16.962 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16814 network=tcp -12:17.465 [INFO] TestAgent: Waiting for endpoints to shut down -12:17.465 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:32.166Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:32.181Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:32.183 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:99bc52a9-7f1c-cdc7-0b26-b3ce568af191 Address:127.0.0.1:23958}]" +48:32.184 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-99bc52a9-7f1c-cdc7-0b26-b3ce568af191.dc1 127.0.0.1 +48:32.184 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23958 [Follower]" leader= +48:32.184 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-99bc52a9-7f1c-cdc7-0b26-b3ce568af191 127.0.0.1 +2022-12-04T05:48:32.184Z [INFO] agent.router: Initializing LAN area manager +48:32.184 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-99bc52a9-7f1c-cdc7-0b26-b3ce568af191.dc1 area=wan +48:32.184 [INFO] TestAgent.server: Adding LAN server: server="Node-99bc52a9-7f1c-cdc7-0b26-b3ce568af191 (Addr: tcp/127.0.0.1:23958) (DC: dc1)" +48:32.185 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23953 network=udp +48:32.185 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23953 network=tcp +48:32.185 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23954 network=tcp +48:32.185 [INFO] TestAgent: started state syncer +48:32.185 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23959 network=tcp +48:32.249 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:32.249 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23958 [Candidate]" term=2 +48:32.249 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:32.249 [DEBUG] TestAgent.server.raft: vote granted: from=99bc52a9-7f1c-cdc7-0b26-b3ce568af191 term=2 tally=1 +48:32.249 [INFO] TestAgent.server.raft: election won: tally=1 +48:32.249 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23958 [Leader]" +48:32.250 [INFO] TestAgent.server: cluster leadership acquired +48:32.250 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23958 +48:32.250 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:32.252 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:32.252 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:32.252 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:32.252 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:32.252 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:32.252 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-99bc52a9-7f1c-cdc7-0b26-b3ce568af191 +48:32.252 [INFO] TestAgent.server: member joined, marking health alive: member=Node-99bc52a9-7f1c-cdc7-0b26-b3ce568af191 +48:32.252 [INFO] TestAgent.server: New leader elected: payload=Node-99bc52a9-7f1c-cdc7-0b26-b3ce568af191 +48:32.252 [INFO] TestAgent.server: federation state anti-entropy synced +48:32.374 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:32.375 [INFO] TestAgent: Synced node info +48:32.375 [INFO] TestAgent: Synced service: service=rabbitmq +48:32.375 [INFO] TestAgent: Synced service: service=rabbitmq-sidecar-proxy +48:32.375 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 +48:32.375 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 +48:32.375 [DEBUG] TestAgent: Node info in sync +48:32.375 [DEBUG] TestAgent: Service in sync: service=rabbitmq +48:32.375 [DEBUG] TestAgent: Service in sync: service=rabbitmq-sidecar-proxy +48:32.375 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 +48:32.375 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 +48:32.568 [INFO] TestAgent: Requesting shutdown +48:32.568 [INFO] TestAgent.server: shutting down server +48:32.568 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:32.568 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:32.568 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:32.568 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:32.568 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:32.568 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:32.568 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:32.568 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:32.568Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:32.568Z [INFO] agent.router.manager: shutting down +48:32.568 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:32.568 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:32.568 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:32.569 [INFO] TestAgent: consul server down +48:32.569 [INFO] TestAgent: shutdown complete +48:32.569 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23953 network=tcp +48:32.569 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23953 network=udp +48:32.569 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23954 network=tcp +48:33.072 [INFO] TestAgent: Waiting for endpoints to shut down +48:33.072 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:32.181Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:32.205Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:32.207 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5d519084-9141-e20a-1614-e88210158fa7 Address:127.0.0.1:23965}]" +48:32.207 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23965 [Follower]" leader= +48:32.208 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5d519084-9141-e20a-1614-e88210158fa7.dc1 127.0.0.1 +48:32.208 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5d519084-9141-e20a-1614-e88210158fa7 127.0.0.1 +2022-12-04T05:48:32.208Z [INFO] agent.router: Initializing LAN area manager +48:32.209 [DEBUG] TestAgent: added local registration for service: service=rabbitmq-sidecar-proxy +48:32.209 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23960 network=udp +48:32.209 [INFO] TestAgent.server: Adding LAN server: server="Node-5d519084-9141-e20a-1614-e88210158fa7 (Addr: tcp/127.0.0.1:23965) (DC: dc1)" +48:32.209 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5d519084-9141-e20a-1614-e88210158fa7.dc1 area=wan +48:32.209 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23960 network=tcp +48:32.210 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23961 network=tcp +48:32.210 [INFO] TestAgent: started state syncer +48:32.211 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23966 network=tcp +48:32.276 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:32.276 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23965 [Candidate]" term=2 +48:32.276 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:32.276 [DEBUG] TestAgent.server.raft: vote granted: from=5d519084-9141-e20a-1614-e88210158fa7 term=2 tally=1 +48:32.276 [INFO] TestAgent.server.raft: election won: tally=1 +48:32.276 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23965 [Leader]" +48:32.276 [INFO] TestAgent.server: cluster leadership acquired +48:32.276 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23965 +48:32.277 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:32.278 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:32.278 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:32.278 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:32.278 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:32.278 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:32.278 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5d519084-9141-e20a-1614-e88210158fa7 +48:32.278 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5d519084-9141-e20a-1614-e88210158fa7 +48:32.279 [INFO] TestAgent.server: New leader elected: payload=Node-5d519084-9141-e20a-1614-e88210158fa7 +48:32.279 [INFO] TestAgent.server: federation state anti-entropy synced +48:32.373 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:32.373 [INFO] TestAgent: Synced node info +48:32.374 [INFO] TestAgent: Synced service: service=rabbitmq +48:32.374 [INFO] TestAgent: Synced service: service=rabbitmq-sidecar-proxy +48:32.374 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 +48:32.374 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 +48:32.374 [DEBUG] TestAgent: Node info in sync +48:32.374 [DEBUG] TestAgent: Service in sync: service=rabbitmq +48:32.374 [DEBUG] TestAgent: Service in sync: service=rabbitmq-sidecar-proxy +48:32.374 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 +48:32.374 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 +48:32.456 [WARN] TestAgent: Check socket connection failed: check=service:rabbitmq-sidecar-proxy:1 error="dial tcp 127.0.0.1:21000: connect: connection refused" +48:32.456 [WARN] TestAgent: Check is now critical: check=service:rabbitmq-sidecar-proxy:1 +48:32.578 [INFO] TestAgent: Requesting shutdown +48:32.578 [INFO] TestAgent.server: shutting down server +48:32.578 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:32.578 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:32.578 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:32.578 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:32.578 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:32.578 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:32.578 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:32.578 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:32.578 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:32.578 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:32.578 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:32.578Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:32.578Z [INFO] agent.router.manager: shutting down +48:32.578 [INFO] TestAgent: consul server down +48:32.578 [INFO] TestAgent: shutdown complete +48:32.578 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23960 network=tcp +48:32.578 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23960 network=udp +48:32.578 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23961 network=tcp +48:33.080 [INFO] TestAgent: Waiting for endpoints to shut down +48:33.080 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_loadServices_sidecar (0.00s) - --- PASS: TestAgent_loadServices_sidecar/normal (0.87s) - --- PASS: TestAgent_loadServices_sidecar/service_manager (0.97s) + --- PASS: TestAgent_loadServices_sidecar/normal (0.92s) + --- PASS: TestAgent_loadServices_sidecar/service_manager (0.93s) === RUN TestAgent_loadServices_sidecarSeparateToken === RUN TestAgent_loadServices_sidecarSeparateToken/normal === PAUSE TestAgent_loadServices_sidecarSeparateToken/normal @@ -12300,132 +12341,132 @@ === CONT TestAgent_loadServices_sidecarSeparateToken/service_manager [WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:17.484Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:17.515Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:17.516 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6ba88639-1e86-9baf-f40a-2424e699ca3d Address:127.0.0.1:16825}]" -12:17.517 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6ba88639-1e86-9baf-f40a-2424e699ca3d.dc1 127.0.0.1 -12:17.518 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16825 [Follower]" leader= -12:17.518 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6ba88639-1e86-9baf-f40a-2424e699ca3d 127.0.0.1 -2021-10-31T23:12:17.518Z [INFO] agent.router: Initializing LAN area manager -12:17.518 [INFO] TestAgent.server: Adding LAN server: server="Node-6ba88639-1e86-9baf-f40a-2424e699ca3d (Addr: tcp/127.0.0.1:16825) (DC: dc1)" -12:17.518 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16820 network=udp -12:17.518 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16820 network=tcp -12:17.518 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6ba88639-1e86-9baf-f40a-2424e699ca3d.dc1 area=wan -12:17.519 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16821 network=tcp -12:17.519 [INFO] TestAgent: started state syncer -12:17.519 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16826 network=tcp -12:17.561 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:17.561 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16825 [Candidate]" term=2 -12:17.562 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:17.562 [DEBUG] TestAgent.server.raft: vote granted: from=6ba88639-1e86-9baf-f40a-2424e699ca3d term=2 tally=1 -12:17.562 [INFO] TestAgent.server.raft: election won: tally=1 -12:17.562 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16825 [Leader]" -12:17.562 [INFO] TestAgent.server: cluster leadership acquired -12:17.562 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16825 -12:17.563 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:17.563 [INFO] TestAgent.server: New leader elected: payload=Node-6ba88639-1e86-9baf-f40a-2424e699ca3d -12:17.564 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:17.564 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:17.564 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:17.564 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:17.564 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:17.564 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6ba88639-1e86-9baf-f40a-2424e699ca3d -12:17.564 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6ba88639-1e86-9baf-f40a-2424e699ca3d -12:17.565 [INFO] TestAgent.server: federation state anti-entropy synced -12:17.633 [INFO] TestAgent: Requesting shutdown -12:17.633 [INFO] TestAgent.server: shutting down server -12:17.633 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:17.633 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:17.633 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:17.633 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:17.633 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:17.633 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:17.633 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:17.633 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:17.633 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:17.633 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:17.633 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:17.633 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:17.634Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:17.634Z [INFO] agent.router.manager: shutting down -12:17.634 [INFO] TestAgent: consul server down -12:17.634 [INFO] TestAgent: shutdown complete -12:17.634 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16820 network=tcp -12:17.634 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16820 network=udp -12:17.634 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16821 network=tcp -12:18.137 [INFO] TestAgent: Waiting for endpoints to shut down -12:18.137 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:33.119Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:33.134Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:33.136 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2ee78303-4ed7-df51-87a9-d303011ff83f Address:127.0.0.1:23972}]" +48:33.137 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23972 [Follower]" leader= +48:33.137 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2ee78303-4ed7-df51-87a9-d303011ff83f.dc1 127.0.0.1 +48:33.137 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2ee78303-4ed7-df51-87a9-d303011ff83f 127.0.0.1 +2022-12-04T05:48:33.137Z [INFO] agent.router: Initializing LAN area manager +48:33.138 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23967 network=udp +48:33.138 [INFO] TestAgent.server: Adding LAN server: server="Node-2ee78303-4ed7-df51-87a9-d303011ff83f (Addr: tcp/127.0.0.1:23972) (DC: dc1)" +48:33.138 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2ee78303-4ed7-df51-87a9-d303011ff83f.dc1 area=wan +48:33.138 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23967 network=tcp +48:33.139 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23968 network=tcp +48:33.139 [INFO] TestAgent: started state syncer +48:33.139 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23973 network=tcp +48:33.179 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:33.179 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23972 [Candidate]" term=2 +48:33.179 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:33.179 [DEBUG] TestAgent.server.raft: vote granted: from=2ee78303-4ed7-df51-87a9-d303011ff83f term=2 tally=1 +48:33.179 [INFO] TestAgent.server.raft: election won: tally=1 +48:33.179 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23972 [Leader]" +48:33.179 [INFO] TestAgent.server: cluster leadership acquired +48:33.179 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23972 +48:33.180 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:33.182 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:33.182 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:33.182 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:33.182 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:33.182 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:33.182 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2ee78303-4ed7-df51-87a9-d303011ff83f +48:33.182 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2ee78303-4ed7-df51-87a9-d303011ff83f +48:33.182 [INFO] TestAgent.server: New leader elected: payload=Node-2ee78303-4ed7-df51-87a9-d303011ff83f +48:33.183 [INFO] TestAgent.server: federation state anti-entropy synced +48:33.212 [INFO] TestAgent: Requesting shutdown +48:33.212 [INFO] TestAgent.server: shutting down server +48:33.212 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:33.212 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:33.212 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:33.212 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:33.212 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:33.212 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:33.212 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:33.212 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:33.212 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:33.212 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:33.212 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:33.212 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:48:33.212Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:33.212Z [INFO] agent.router.manager: shutting down +48:33.212 [INFO] TestAgent: consul server down +48:33.212 [INFO] TestAgent: shutdown complete +48:33.212 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23967 network=tcp +48:33.212 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23967 network=udp +48:33.212 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23968 network=tcp +48:33.712 [INFO] TestAgent: Waiting for endpoints to shut down +48:33.712 [INFO] TestAgent: Endpoints down [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:17.519Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:17.555Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:17.571 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:99b3f1b0-9dd5-4ea5-3af7-69c09659b737 Address:127.0.0.1:16832}]" -12:17.571 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-99b3f1b0-9dd5-4ea5-3af7-69c09659b737.dc1 127.0.0.1 -12:17.572 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-99b3f1b0-9dd5-4ea5-3af7-69c09659b737 127.0.0.1 -2021-10-31T23:12:17.572Z [INFO] agent.router: Initializing LAN area manager -12:17.572 [DEBUG] TestAgent: added local registration for service: service=rabbitmq-sidecar-proxy -12:17.572 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16827 network=udp -12:17.572 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16832 [Follower]" leader= -12:17.573 [INFO] TestAgent.server: Adding LAN server: server="Node-99b3f1b0-9dd5-4ea5-3af7-69c09659b737 (Addr: tcp/127.0.0.1:16832) (DC: dc1)" -12:17.573 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-99b3f1b0-9dd5-4ea5-3af7-69c09659b737.dc1 area=wan -12:17.573 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16827 network=tcp -12:17.599 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16828 network=tcp -12:17.599 [INFO] TestAgent: started state syncer -12:17.599 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16833 network=tcp -12:17.609 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:17.609 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16832 [Candidate]" term=2 -12:17.610 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:17.610 [DEBUG] TestAgent.server.raft: vote granted: from=99b3f1b0-9dd5-4ea5-3af7-69c09659b737 term=2 tally=1 -12:17.610 [INFO] TestAgent.server.raft: election won: tally=1 -12:17.610 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16832 [Leader]" -12:17.610 [INFO] TestAgent.server: cluster leadership acquired -12:17.610 [INFO] TestAgent.server: New leader elected: payload=Node-99b3f1b0-9dd5-4ea5-3af7-69c09659b737 -12:17.610 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16832 -12:17.611 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:17.613 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:17.613 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:17.613 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:17.613 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:17.613 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:17.613 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-99b3f1b0-9dd5-4ea5-3af7-69c09659b737 -12:17.613 [INFO] TestAgent.server: member joined, marking health alive: member=Node-99b3f1b0-9dd5-4ea5-3af7-69c09659b737 -12:17.615 [INFO] TestAgent.server: federation state anti-entropy synced -12:17.794 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:17.794 [INFO] TestAgent: Synced node info -12:17.794 [INFO] TestAgent: Synced service: service=rabbitmq -12:17.795 [INFO] TestAgent: Synced service: service=rabbitmq-sidecar-proxy -12:17.795 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 -12:17.795 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 -12:17.795 [DEBUG] TestAgent: Node info in sync -12:17.795 [DEBUG] TestAgent: Service in sync: service=rabbitmq -12:17.795 [DEBUG] TestAgent: Service in sync: service=rabbitmq-sidecar-proxy -12:17.795 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 -12:17.795 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 -12:17.990 [INFO] TestAgent: Requesting shutdown -12:17.990 [INFO] TestAgent.server: shutting down server -12:17.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:17.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:17.990 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:17.990 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:17.990 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:17.990 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:17.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:17.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:17.990 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:17.990 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:17.990 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:17.990Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:17.990Z [INFO] agent.router.manager: shutting down -12:17.990 [INFO] TestAgent: consul server down -12:17.990 [INFO] TestAgent: shutdown complete -12:17.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16827 network=tcp -12:17.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16827 network=udp -12:17.990 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16828 network=tcp -12:18.497 [INFO] TestAgent: Waiting for endpoints to shut down -12:18.497 [INFO] TestAgent: Endpoints down +[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:48:33.123Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:33.141Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:33.143 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:163aaf75-c6ce-3666-d096-fd7b20f816f5 Address:127.0.0.1:23979}]" +48:33.144 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-163aaf75-c6ce-3666-d096-fd7b20f816f5.dc1 127.0.0.1 +48:33.144 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23979 [Follower]" leader= +48:33.144 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-163aaf75-c6ce-3666-d096-fd7b20f816f5 127.0.0.1 +2022-12-04T05:48:33.144Z [INFO] agent.router: Initializing LAN area manager +48:33.145 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-163aaf75-c6ce-3666-d096-fd7b20f816f5.dc1 area=wan +48:33.145 [INFO] TestAgent.server: Adding LAN server: server="Node-163aaf75-c6ce-3666-d096-fd7b20f816f5 (Addr: tcp/127.0.0.1:23979) (DC: dc1)" +48:33.145 [DEBUG] TestAgent: added local registration for service: service=rabbitmq-sidecar-proxy +48:33.145 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23974 network=udp +48:33.145 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23974 network=tcp +48:33.146 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23975 network=tcp +48:33.146 [INFO] TestAgent: started state syncer +48:33.146 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23980 network=tcp +48:33.180 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:33.180 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23979 [Candidate]" term=2 +48:33.181 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:33.181 [DEBUG] TestAgent.server.raft: vote granted: from=163aaf75-c6ce-3666-d096-fd7b20f816f5 term=2 tally=1 +48:33.181 [INFO] TestAgent.server.raft: election won: tally=1 +48:33.181 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23979 [Leader]" +48:33.181 [INFO] TestAgent.server: cluster leadership acquired +48:33.181 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23979 +48:33.182 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:33.183 [INFO] TestAgent.server: New leader elected: payload=Node-163aaf75-c6ce-3666-d096-fd7b20f816f5 +48:33.183 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:33.183 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:33.183 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:33.183 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:33.183 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:33.183 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-163aaf75-c6ce-3666-d096-fd7b20f816f5 +48:33.183 [INFO] TestAgent.server: member joined, marking health alive: member=Node-163aaf75-c6ce-3666-d096-fd7b20f816f5 +48:33.184 [INFO] TestAgent.server: federation state anti-entropy synced +48:33.308 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:33.308 [INFO] TestAgent: Synced node info +48:33.309 [INFO] TestAgent: Synced service: service=rabbitmq +48:33.309 [INFO] TestAgent: Synced service: service=rabbitmq-sidecar-proxy +48:33.309 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 +48:33.309 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 +48:33.309 [DEBUG] TestAgent: Node info in sync +48:33.309 [DEBUG] TestAgent: Service in sync: service=rabbitmq +48:33.309 [DEBUG] TestAgent: Service in sync: service=rabbitmq-sidecar-proxy +48:33.309 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 +48:33.309 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 +48:33.560 [INFO] TestAgent: Requesting shutdown +48:33.561 [INFO] TestAgent.server: shutting down server +48:33.561 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:33.561 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:33.561 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:33.561 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:33.561 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:33.561 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:33.561 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:33.561 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:33.561 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:33.561 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:33.561 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:33.561Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:33.561Z [INFO] agent.router.manager: shutting down +48:33.561 [INFO] TestAgent: consul server down +48:33.561 [INFO] TestAgent: shutdown complete +48:33.561 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23974 network=tcp +48:33.561 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23974 network=udp +48:33.561 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23975 network=tcp +48:34.064 [INFO] TestAgent: Waiting for endpoints to shut down +48:34.064 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_loadServices_sidecarSeparateToken (0.00s) - --- PASS: TestAgent_loadServices_sidecarSeparateToken/normal (0.67s) - --- PASS: TestAgent_loadServices_sidecarSeparateToken/service_manager (1.03s) + --- PASS: TestAgent_loadServices_sidecarSeparateToken/normal (0.63s) + --- PASS: TestAgent_loadServices_sidecarSeparateToken/service_manager (0.98s) === RUN TestAgent_loadServices_sidecarInheritMeta === RUN TestAgent_loadServices_sidecarInheritMeta/normal === PAUSE TestAgent_loadServices_sidecarInheritMeta/normal @@ -12436,121 +12477,136 @@ [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:18.544Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:18.599Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:18.600 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e0791f33-873a-844e-c5e8-9f1aff07adc4 Address:127.0.0.1:16839}]" -12:18.601 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e0791f33-873a-844e-c5e8-9f1aff07adc4.dc1 127.0.0.1 -12:18.601 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16839 [Follower]" leader= -12:18.613 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e0791f33-873a-844e-c5e8-9f1aff07adc4 127.0.0.1 -2021-10-31T23:12:18.614Z [INFO] agent.router: Initializing LAN area manager -12:18.614 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16834 network=udp -12:18.614 [INFO] TestAgent.server: Adding LAN server: server="Node-e0791f33-873a-844e-c5e8-9f1aff07adc4 (Addr: tcp/127.0.0.1:16839) (DC: dc1)" -12:18.614 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e0791f33-873a-844e-c5e8-9f1aff07adc4.dc1 area=wan -12:18.614 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16834 network=tcp -12:18.615 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16835 network=tcp -12:18.615 [INFO] TestAgent: started state syncer -12:18.615 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16840 network=tcp -12:18.674 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:18.674 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16839 [Candidate]" term=2 -12:18.674 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:18.674 [DEBUG] TestAgent.server.raft: vote granted: from=e0791f33-873a-844e-c5e8-9f1aff07adc4 term=2 tally=1 -12:18.674 [INFO] TestAgent.server.raft: election won: tally=1 -12:18.674 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16839 [Leader]" -12:18.674 [INFO] TestAgent.server: cluster leadership acquired -12:18.674 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16839 -12:18.675 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:18.676 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:18.676 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:18.676 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:18.676 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:18.676 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:18.677 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e0791f33-873a-844e-c5e8-9f1aff07adc4 -12:18.677 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e0791f33-873a-844e-c5e8-9f1aff07adc4 -12:18.677 [INFO] TestAgent.server: New leader elected: payload=Node-e0791f33-873a-844e-c5e8-9f1aff07adc4 -12:18.677 [INFO] TestAgent.server: federation state anti-entropy synced -12:18.800 [INFO] TestAgent: Requesting shutdown -12:18.800 [INFO] TestAgent.server: shutting down server -12:18.800 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:18.800 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:18.800 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:18.800 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:18.800 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:18.801 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:18.801 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:18.801 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:18.801 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:18.801 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:18.801 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:18.801 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:18.801Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:18.801Z [INFO] agent.router.manager: shutting down -12:18.801 [INFO] TestAgent: consul server down -12:18.801 [INFO] TestAgent: shutdown complete -12:18.801 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16834 network=tcp -12:18.801 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16834 network=udp -12:18.801 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16835 network=tcp -12:19.305 [INFO] TestAgent: Waiting for endpoints to shut down -12:19.305 [INFO] TestAgent: Endpoints down -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:18.535Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:18.568Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:18.578 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3a6a56b1-5120-70e5-f570-6ddf2f6da55d Address:127.0.0.1:16846}]" -12:18.579 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3a6a56b1-5120-70e5-f570-6ddf2f6da55d.dc1 127.0.0.1 -12:18.580 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3a6a56b1-5120-70e5-f570-6ddf2f6da55d 127.0.0.1 -2021-10-31T23:12:18.580Z [INFO] agent.router: Initializing LAN area manager -12:18.580 [DEBUG] TestAgent: added local registration for service: service=rabbitmq-sidecar-proxy -12:18.580 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16841 network=udp -12:18.580 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16846 [Follower]" leader= -12:18.580 [INFO] TestAgent.server: Adding LAN server: server="Node-3a6a56b1-5120-70e5-f570-6ddf2f6da55d (Addr: tcp/127.0.0.1:16846) (DC: dc1)" -12:18.580 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3a6a56b1-5120-70e5-f570-6ddf2f6da55d.dc1 area=wan -12:18.581 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16841 network=tcp -12:18.581 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16842 network=tcp -12:18.581 [INFO] TestAgent: started state syncer -12:18.581 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16847 network=tcp -12:18.649 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:18.649 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16846 [Candidate]" term=2 -12:18.650 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:18.650 [DEBUG] TestAgent.server.raft: vote granted: from=3a6a56b1-5120-70e5-f570-6ddf2f6da55d term=2 tally=1 -12:18.650 [INFO] TestAgent.server.raft: election won: tally=1 -12:18.650 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16846 [Leader]" -12:18.650 [INFO] TestAgent.server: cluster leadership acquired -12:18.650 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16846 -12:18.651 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:18.652 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:18.652 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:18.652 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:18.652 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:18.652 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:18.653 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3a6a56b1-5120-70e5-f570-6ddf2f6da55d -12:18.653 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3a6a56b1-5120-70e5-f570-6ddf2f6da55d -12:18.653 [INFO] TestAgent.server: New leader elected: payload=Node-3a6a56b1-5120-70e5-f570-6ddf2f6da55d -12:18.653 [INFO] TestAgent.server: federation state anti-entropy synced -12:18.930 [INFO] TestAgent: Requesting shutdown -12:18.930 [INFO] TestAgent.server: shutting down server -12:18.930 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:18.930 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:18.930 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:18.930 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:18.930 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:18.930 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:18.930 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:18.931 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:18.931 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:18.931 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:18.931 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:18.931 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:18.931Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:18.931Z [INFO] agent.router.manager: shutting down -12:18.931 [INFO] TestAgent: consul server down -12:18.931 [INFO] TestAgent: shutdown complete -12:18.931 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16841 network=tcp -12:18.931 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16841 network=udp -12:18.931 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16842 network=tcp -12:19.433 [INFO] TestAgent: Waiting for endpoints to shut down -12:19.433 [INFO] TestAgent: Endpoints down +[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:48:34.095Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:34.146Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:34.149 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b92f5c12-ae9b-6592-75b6-b73dbd76fd0a Address:127.0.0.1:23993}]" +48:34.150 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b92f5c12-ae9b-6592-75b6-b73dbd76fd0a.dc1 127.0.0.1 +48:34.151 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b92f5c12-ae9b-6592-75b6-b73dbd76fd0a 127.0.0.1 +2022-12-04T05:48:34.151Z [INFO] agent.router: Initializing LAN area manager +48:34.151 [DEBUG] TestAgent: added local registration for service: service=rabbitmq-sidecar-proxy +48:34.151 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23988 network=udp +48:34.151 [INFO] TestAgent.server: Adding LAN server: server="Node-b92f5c12-ae9b-6592-75b6-b73dbd76fd0a (Addr: tcp/127.0.0.1:23993) (DC: dc1)" +48:34.151 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b92f5c12-ae9b-6592-75b6-b73dbd76fd0a.dc1 area=wan +48:34.152 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23988 network=tcp +48:34.152 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23989 network=tcp +48:34.152 [INFO] TestAgent: started state syncer +48:34.153 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23994 network=tcp +48:34.156 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23993 [Follower]" leader= +48:34.216 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:34.216 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23993 [Candidate]" term=2 +48:34.216 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:34.216 [DEBUG] TestAgent.server.raft: vote granted: from=b92f5c12-ae9b-6592-75b6-b73dbd76fd0a term=2 tally=1 +48:34.216 [INFO] TestAgent.server.raft: election won: tally=1 +48:34.216 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23993 [Leader]" +48:34.216 [INFO] TestAgent.server: cluster leadership acquired +48:34.216 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23993 +48:34.217 [INFO] TestAgent.server: New leader elected: payload=Node-b92f5c12-ae9b-6592-75b6-b73dbd76fd0a +48:34.217 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:34.219 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:34.219 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:34.219 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:34.219 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:34.219 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:34.219 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b92f5c12-ae9b-6592-75b6-b73dbd76fd0a +48:34.219 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b92f5c12-ae9b-6592-75b6-b73dbd76fd0a +48:34.220 [INFO] TestAgent.server: federation state anti-entropy synced +48:34.556 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:34.557 [INFO] TestAgent: Synced node info +48:34.557 [INFO] TestAgent: Synced service: service=rabbitmq +48:34.558 [INFO] TestAgent: Synced service: service=rabbitmq-sidecar-proxy +48:34.558 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 +48:34.558 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 +48:34.591 [INFO] TestAgent: Requesting shutdown +48:34.591 [INFO] TestAgent.server: shutting down server +48:34.591 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:34.591 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:34.591 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:34.591 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:34.591 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:34.591 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:34.591 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:34.591 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:34.591 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:34.591 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:34.591 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:34.591Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:34.591Z [INFO] agent.router.manager: shutting down +48:34.592 [INFO] TestAgent: consul server down +48:34.592 [INFO] TestAgent: shutdown complete +48:34.592 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23988 network=tcp +48:34.592 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23988 network=udp +48:34.592 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23989 network=tcp +48:35.092 [INFO] TestAgent: Waiting for endpoints to shut down +48:35.092 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:34.091Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:34.155Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:34.157 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:74be6e26-a07b-ff22-5d9e-ca0ee18d2ace Address:127.0.0.1:23986}]" +48:34.157 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23986 [Follower]" leader= +48:34.158 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-74be6e26-a07b-ff22-5d9e-ca0ee18d2ace.dc1 127.0.0.1 +48:34.158 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-74be6e26-a07b-ff22-5d9e-ca0ee18d2ace 127.0.0.1 +2022-12-04T05:48:34.158Z [INFO] agent.router: Initializing LAN area manager +48:34.158 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23981 network=udp +48:34.158 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-74be6e26-a07b-ff22-5d9e-ca0ee18d2ace.dc1 area=wan +48:34.159 [INFO] TestAgent.server: Adding LAN server: server="Node-74be6e26-a07b-ff22-5d9e-ca0ee18d2ace (Addr: tcp/127.0.0.1:23986) (DC: dc1)" +48:34.159 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23981 network=tcp +48:34.159 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23982 network=tcp +48:34.159 [INFO] TestAgent: started state syncer +48:34.159 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23987 network=tcp +48:34.196 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:34.196 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23986 [Candidate]" term=2 +48:34.196 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:34.196 [DEBUG] TestAgent.server.raft: vote granted: from=74be6e26-a07b-ff22-5d9e-ca0ee18d2ace term=2 tally=1 +48:34.196 [INFO] TestAgent.server.raft: election won: tally=1 +48:34.196 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23986 [Leader]" +48:34.197 [INFO] TestAgent.server: cluster leadership acquired +48:34.197 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23986 +48:34.197 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:34.199 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:34.199 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:34.199 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:34.199 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:34.199 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:34.199 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-74be6e26-a07b-ff22-5d9e-ca0ee18d2ace +48:34.199 [INFO] TestAgent.server: member joined, marking health alive: member=Node-74be6e26-a07b-ff22-5d9e-ca0ee18d2ace +48:34.200 [INFO] TestAgent.server: New leader elected: payload=Node-74be6e26-a07b-ff22-5d9e-ca0ee18d2ace +48:34.200 [INFO] TestAgent.server: federation state anti-entropy synced +48:34.316 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:34.317 [INFO] TestAgent: Synced node info +48:34.317 [INFO] TestAgent: Synced service: service=rabbitmq +48:34.318 [INFO] TestAgent: Synced service: service=rabbitmq-sidecar-proxy +48:34.318 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 +48:34.318 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 +48:34.318 [DEBUG] TestAgent: Node info in sync +48:34.318 [DEBUG] TestAgent: Service in sync: service=rabbitmq +48:34.318 [DEBUG] TestAgent: Service in sync: service=rabbitmq-sidecar-proxy +48:34.318 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 +48:34.318 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 +48:34.596 [INFO] TestAgent: Requesting shutdown +48:34.596 [INFO] TestAgent.server: shutting down server +48:34.596 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:34.596 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:34.596 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:34.596 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:34.596 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:34.596 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:34.596 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:34.596 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:34.596 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:34.596 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:34.596 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:34.597Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:34.597Z [INFO] agent.router.manager: shutting down +48:34.597 [INFO] TestAgent: consul server down +48:34.597 [INFO] TestAgent: shutdown complete +48:34.597 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23981 network=tcp +48:34.597 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23981 network=udp +48:34.597 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23982 network=tcp +48:35.100 [INFO] TestAgent: Waiting for endpoints to shut down +48:35.100 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_loadServices_sidecarInheritMeta (0.00s) - --- PASS: TestAgent_loadServices_sidecarInheritMeta/normal (0.81s) - --- PASS: TestAgent_loadServices_sidecarInheritMeta/service_manager (0.93s) + --- PASS: TestAgent_loadServices_sidecarInheritMeta/service_manager (1.02s) + --- PASS: TestAgent_loadServices_sidecarInheritMeta/normal (1.04s) === RUN TestAgent_loadServices_sidecarOverrideMeta === RUN TestAgent_loadServices_sidecarOverrideMeta/normal === PAUSE TestAgent_loadServices_sidecarOverrideMeta/normal @@ -12558,134 +12614,123 @@ === PAUSE TestAgent_loadServices_sidecarOverrideMeta/service_manager === CONT TestAgent_loadServices_sidecarOverrideMeta/normal === CONT TestAgent_loadServices_sidecarOverrideMeta/service_manager -[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:19.504Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:19.538Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:19.540 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fd9c41c1-0678-3649-c5a8-3bbf6c9a385a Address:127.0.0.1:16860}]" -12:19.541 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fd9c41c1-0678-3649-c5a8-3bbf6c9a385a.dc1 127.0.0.1 -12:19.541 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16860 [Follower]" leader= -12:19.550 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fd9c41c1-0678-3649-c5a8-3bbf6c9a385a 127.0.0.1 -2021-10-31T23:12:19.550Z [INFO] agent.router: Initializing LAN area manager -12:19.550 [DEBUG] TestAgent: added local registration for service: service=rabbitmq-sidecar-proxy -12:19.550 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16855 network=udp -12:19.550 [INFO] TestAgent.server: Adding LAN server: server="Node-fd9c41c1-0678-3649-c5a8-3bbf6c9a385a (Addr: tcp/127.0.0.1:16860) (DC: dc1)" -12:19.550 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fd9c41c1-0678-3649-c5a8-3bbf6c9a385a.dc1 area=wan -12:19.551 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16855 network=tcp -12:19.551 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16856 network=tcp -12:19.551 [INFO] TestAgent: started state syncer -12:19.552 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16861 network=tcp -12:19.609 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:19.609 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16860 [Candidate]" term=2 -12:19.610 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:19.610 [DEBUG] TestAgent.server.raft: vote granted: from=fd9c41c1-0678-3649-c5a8-3bbf6c9a385a term=2 tally=1 -12:19.610 [INFO] TestAgent.server.raft: election won: tally=1 -12:19.610 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16860 [Leader]" -12:19.610 [INFO] TestAgent.server: cluster leadership acquired -12:19.610 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16860 -12:19.611 [INFO] TestAgent.server: New leader elected: payload=Node-fd9c41c1-0678-3649-c5a8-3bbf6c9a385a -12:19.611 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:19.612 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:19.612 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:19.612 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:19.612 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:19.612 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:19.612 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fd9c41c1-0678-3649-c5a8-3bbf6c9a385a -12:19.612 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fd9c41c1-0678-3649-c5a8-3bbf6c9a385a -12:19.613 [INFO] TestAgent.server: federation state anti-entropy synced -12:19.890 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:19.890 [INFO] TestAgent: Synced node info -12:19.891 [INFO] TestAgent: Synced service: service=rabbitmq-sidecar-proxy -12:19.891 [INFO] TestAgent: Synced service: service=rabbitmq -12:19.891 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 -12:19.891 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 -12:19.891 [INFO] TestAgent: Requesting shutdown -12:19.892 [INFO] TestAgent.server: shutting down server -12:19.892 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:19.892 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:19.892 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:19.892 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:19.892 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:19.892 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:19.892 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:19.892 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:19.892 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:19.892 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:19.892 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:19.892Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:19.892Z [INFO] agent.router.manager: shutting down -12:19.892 [INFO] TestAgent: consul server down -12:19.892 [INFO] TestAgent: shutdown complete -12:19.892 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16855 network=tcp -12:19.892 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16855 network=udp -12:19.892 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16856 network=tcp -12:20.393 [INFO] TestAgent: Waiting for endpoints to shut down -12:20.393 [INFO] TestAgent: Endpoints down -2021-10-31T23:12:19.521Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:19.567Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:19.569 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bbd6cfb2-d401-5e47-674b-1d668780ae0c Address:127.0.0.1:16853}]" -12:19.569 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bbd6cfb2-d401-5e47-674b-1d668780ae0c.dc1 127.0.0.1 -12:19.569 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16853 [Follower]" leader= -12:19.574 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bbd6cfb2-d401-5e47-674b-1d668780ae0c 127.0.0.1 -2021-10-31T23:12:19.574Z [INFO] agent.router: Initializing LAN area manager -12:19.575 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16848 network=udp -12:19.575 [INFO] TestAgent.server: Adding LAN server: server="Node-bbd6cfb2-d401-5e47-674b-1d668780ae0c (Addr: tcp/127.0.0.1:16853) (DC: dc1)" -12:19.575 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bbd6cfb2-d401-5e47-674b-1d668780ae0c.dc1 area=wan -12:19.575 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16848 network=tcp -12:19.576 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16849 network=tcp -12:19.576 [INFO] TestAgent: started state syncer -12:19.576 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16854 network=tcp -12:19.639 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:19.639 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16853 [Candidate]" term=2 -12:19.639 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:19.639 [DEBUG] TestAgent.server.raft: vote granted: from=bbd6cfb2-d401-5e47-674b-1d668780ae0c term=2 tally=1 -12:19.639 [INFO] TestAgent.server.raft: election won: tally=1 -12:19.639 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16853 [Leader]" -12:19.639 [INFO] TestAgent.server: cluster leadership acquired -12:19.639 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16853 -12:19.640 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:19.641 [INFO] TestAgent.server: New leader elected: payload=Node-bbd6cfb2-d401-5e47-674b-1d668780ae0c -12:19.642 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:19.642 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:19.642 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:19.642 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:19.642 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:19.642 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bbd6cfb2-d401-5e47-674b-1d668780ae0c -12:19.642 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bbd6cfb2-d401-5e47-674b-1d668780ae0c -12:19.642 [INFO] TestAgent.server: federation state anti-entropy synced -12:19.894 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:19.894 [INFO] TestAgent: Synced node info -12:19.895 [INFO] TestAgent: Synced service: service=rabbitmq-sidecar-proxy -12:19.895 [INFO] TestAgent: Synced service: service=rabbitmq -12:19.895 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:1 -12:19.895 [DEBUG] TestAgent: Check in sync: check=service:rabbitmq-sidecar-proxy:2 -12:19.941 [INFO] TestAgent: Requesting shutdown -12:19.941 [INFO] TestAgent.server: shutting down server -12:19.942 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:19.942 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:19.942 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:19.942 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:19.942 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:19.942 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:19.942 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:19.942 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:19.942 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:19.942 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:19.942 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:19.942Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:19.942Z [INFO] agent.router.manager: shutting down -12:19.942 [INFO] TestAgent: consul server down -12:19.942 [INFO] TestAgent: shutdown complete -12:19.942 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16848 network=tcp -12:19.942 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16848 network=udp -12:19.942 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16849 network=tcp -12:20.444 [INFO] TestAgent: Waiting for endpoints to shut down -12:20.444 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:35.146Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:35.180Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:35.182 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:548563be-8f18-3512-d59d-6c08c9797168 Address:127.0.0.1:24007}]" +48:35.182 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24007 [Follower]" leader= +48:35.183 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-548563be-8f18-3512-d59d-6c08c9797168.dc1 127.0.0.1 +48:35.183 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-548563be-8f18-3512-d59d-6c08c9797168 127.0.0.1 +2022-12-04T05:48:35.183Z [INFO] agent.router: Initializing LAN area manager +48:35.183 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-548563be-8f18-3512-d59d-6c08c9797168.dc1 area=wan +48:35.183 [INFO] TestAgent.server: Adding LAN server: server="Node-548563be-8f18-3512-d59d-6c08c9797168 (Addr: tcp/127.0.0.1:24007) (DC: dc1)" +48:35.183 [DEBUG] TestAgent: added local registration for service: service=rabbitmq-sidecar-proxy +48:35.184 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24002 network=udp +48:35.184 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24002 network=tcp +48:35.193 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24003 network=tcp +48:35.193 [INFO] TestAgent: started state syncer +48:35.193 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24008 network=tcp +48:35.224 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:35.224 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24007 [Candidate]" term=2 +48:35.224 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:35.224 [DEBUG] TestAgent.server.raft: vote granted: from=548563be-8f18-3512-d59d-6c08c9797168 term=2 tally=1 +48:35.224 [INFO] TestAgent.server.raft: election won: tally=1 +48:35.224 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24007 [Leader]" +48:35.225 [INFO] TestAgent.server: cluster leadership acquired +48:35.225 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24007 +48:35.225 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:35.227 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:35.227 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:35.227 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:35.227 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:35.227 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:35.227 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-548563be-8f18-3512-d59d-6c08c9797168 +48:35.227 [INFO] TestAgent.server: member joined, marking health alive: member=Node-548563be-8f18-3512-d59d-6c08c9797168 +48:35.228 [INFO] TestAgent.server: New leader elected: payload=Node-548563be-8f18-3512-d59d-6c08c9797168 +48:35.228 [INFO] TestAgent.server: federation state anti-entropy synced +48:35.372 [INFO] TestAgent: Requesting shutdown +48:35.372 [INFO] TestAgent.server: shutting down server +48:35.372 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:35.372 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:35.372 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:35.372 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:35.372 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:35.373 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:35.373 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:35.373 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:35.373 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:35.373 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:35.373 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:35.373 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:35.373Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:35.373Z [INFO] agent.router.manager: shutting down +48:35.373 [INFO] TestAgent: consul server down +48:35.373 [INFO] TestAgent: shutdown complete +48:35.373 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24002 network=tcp +48:35.373 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24002 network=udp +48:35.373 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24003 network=tcp +48:35.876 [INFO] TestAgent: Waiting for endpoints to shut down +48:35.876 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:35.137Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:35.163Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:35.168 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9c9e2be1-b642-2e0a-edb6-bdb7fe96e89f Address:127.0.0.1:24000}]" +48:35.168 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24000 [Follower]" leader= +48:35.168 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9c9e2be1-b642-2e0a-edb6-bdb7fe96e89f.dc1 127.0.0.1 +48:35.169 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9c9e2be1-b642-2e0a-edb6-bdb7fe96e89f 127.0.0.1 +2022-12-04T05:48:35.169Z [INFO] agent.router: Initializing LAN area manager +48:35.169 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23995 network=udp +48:35.169 [INFO] TestAgent.server: Adding LAN server: server="Node-9c9e2be1-b642-2e0a-edb6-bdb7fe96e89f (Addr: tcp/127.0.0.1:24000) (DC: dc1)" +48:35.169 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23995 network=tcp +48:35.170 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23996 network=tcp +48:35.170 [INFO] TestAgent: started state syncer +48:35.170 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24001 network=tcp +48:35.171 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9c9e2be1-b642-2e0a-edb6-bdb7fe96e89f.dc1 area=wan +48:35.235 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:35.235 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24000 [Candidate]" term=2 +48:35.236 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:35.236 [DEBUG] TestAgent.server.raft: vote granted: from=9c9e2be1-b642-2e0a-edb6-bdb7fe96e89f term=2 tally=1 +48:35.236 [INFO] TestAgent.server.raft: election won: tally=1 +48:35.236 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24000 [Leader]" +48:35.236 [INFO] TestAgent.server: cluster leadership acquired +48:35.236 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24000 +48:35.237 [INFO] TestAgent.server: New leader elected: payload=Node-9c9e2be1-b642-2e0a-edb6-bdb7fe96e89f +48:35.237 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:35.238 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:35.238 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:35.238 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:35.238 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:35.238 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:35.239 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9c9e2be1-b642-2e0a-edb6-bdb7fe96e89f +48:35.239 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9c9e2be1-b642-2e0a-edb6-bdb7fe96e89f +48:35.239 [INFO] TestAgent.server: federation state anti-entropy synced +48:35.505 [INFO] TestAgent: Requesting shutdown +48:35.505 [INFO] TestAgent.server: shutting down server +48:35.505 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:35.505 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:35.505 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:35.505 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:35.505 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:35.505 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:35.505 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:35.505 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:35.505 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:35.505 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:35.505 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:35.506 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:35.506Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:35.506Z [INFO] agent.router.manager: shutting down +48:35.506 [INFO] TestAgent: consul server down +48:35.506 [INFO] TestAgent: shutdown complete +48:35.506 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23995 network=tcp +48:35.506 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23995 network=udp +48:35.506 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23996 network=tcp +48:36.008 [INFO] TestAgent: Waiting for endpoints to shut down +48:36.008 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_loadServices_sidecarOverrideMeta (0.00s) - --- PASS: TestAgent_loadServices_sidecarOverrideMeta/service_manager (0.96s) - --- PASS: TestAgent_loadServices_sidecarOverrideMeta/normal (1.01s) + --- PASS: TestAgent_loadServices_sidecarOverrideMeta/service_manager (0.77s) + --- PASS: TestAgent_loadServices_sidecarOverrideMeta/normal (0.91s) === RUN TestAgent_unloadServices === RUN TestAgent_unloadServices/normal === PAUSE TestAgent_unloadServices/normal @@ -12696,261 +12741,262 @@ [WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:20.474Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:20.574Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:20.576 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:41280d84-174f-6567-d3b7-3a6af0ebb00c Address:127.0.0.1:16867}]" -12:20.577 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-41280d84-174f-6567-d3b7-3a6af0ebb00c.dc1 127.0.0.1 -12:20.581 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16867 [Follower]" leader= -12:20.582 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-41280d84-174f-6567-d3b7-3a6af0ebb00c 127.0.0.1 -2021-10-31T23:12:20.582Z [INFO] agent.router: Initializing LAN area manager -12:20.582 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16862 network=udp -12:20.582 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-41280d84-174f-6567-d3b7-3a6af0ebb00c.dc1 area=wan -12:20.582 [INFO] TestAgent.server: Adding LAN server: server="Node-41280d84-174f-6567-d3b7-3a6af0ebb00c (Addr: tcp/127.0.0.1:16867) (DC: dc1)" -12:20.582 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16862 network=tcp -12:20.583 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16863 network=tcp -12:20.583 [INFO] TestAgent: started state syncer -12:20.583 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16868 network=tcp -12:20.637 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:20.637 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16867 [Candidate]" term=2 -12:20.637 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:20.637 [DEBUG] TestAgent.server.raft: vote granted: from=41280d84-174f-6567-d3b7-3a6af0ebb00c term=2 tally=1 -12:20.637 [INFO] TestAgent.server.raft: election won: tally=1 -12:20.637 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16867 [Leader]" -12:20.637 [INFO] TestAgent.server: cluster leadership acquired -12:20.638 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16867 -12:20.638 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:20.639 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:20.640 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:20.640 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:20.640 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:20.640 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:20.640 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-41280d84-174f-6567-d3b7-3a6af0ebb00c -12:20.640 [INFO] TestAgent.server: member joined, marking health alive: member=Node-41280d84-174f-6567-d3b7-3a6af0ebb00c -12:20.640 [INFO] TestAgent.server: New leader elected: payload=Node-41280d84-174f-6567-d3b7-3a6af0ebb00c -12:20.640 [INFO] TestAgent.server: federation state anti-entropy synced -12:20.890 [DEBUG] TestAgent: removed service: service=redis -12:20.890 [INFO] TestAgent: Requesting shutdown -12:20.890 [INFO] TestAgent.server: shutting down server -12:20.890 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:20.890 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:20.890 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:20.890 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:20.890 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:20.890 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:20.890 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:20.890 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:20.890 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:20.890 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:20.890 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:20.890 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:20.890Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:20.890Z [INFO] agent.router.manager: shutting down -12:20.890 [INFO] TestAgent: consul server down -12:20.890 [INFO] TestAgent: shutdown complete -12:20.890 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16862 network=tcp -12:20.890 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16862 network=udp -12:20.890 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16863 network=tcp -12:21.391 [INFO] TestAgent: Waiting for endpoints to shut down -12:21.391 [INFO] TestAgent: Endpoints down -2021-10-31T23:12:20.497Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:20.579Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:20.581 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:13b3763f-415f-e5e1-9320-7647b5c2e752 Address:127.0.0.1:16874}]" -12:20.581 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16874 [Follower]" leader= -12:20.588 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-13b3763f-415f-e5e1-9320-7647b5c2e752.dc1 127.0.0.1 -12:20.589 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-13b3763f-415f-e5e1-9320-7647b5c2e752 127.0.0.1 -2021-10-31T23:12:20.589Z [INFO] agent.router: Initializing LAN area manager -12:20.589 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16869 network=udp -12:20.589 [INFO] TestAgent.server: Adding LAN server: server="Node-13b3763f-415f-e5e1-9320-7647b5c2e752 (Addr: tcp/127.0.0.1:16874) (DC: dc1)" -12:20.589 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-13b3763f-415f-e5e1-9320-7647b5c2e752.dc1 area=wan -12:20.589 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16869 network=tcp -12:20.590 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16870 network=tcp -12:20.590 [INFO] TestAgent: started state syncer -12:20.590 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16875 network=tcp -12:20.653 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:20.654 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16874 [Candidate]" term=2 -12:20.654 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:20.654 [DEBUG] TestAgent.server.raft: vote granted: from=13b3763f-415f-e5e1-9320-7647b5c2e752 term=2 tally=1 -12:20.654 [INFO] TestAgent.server.raft: election won: tally=1 -12:20.654 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16874 [Leader]" -12:20.654 [INFO] TestAgent.server: cluster leadership acquired -12:20.654 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16874 -12:20.654 [INFO] TestAgent.server: New leader elected: payload=Node-13b3763f-415f-e5e1-9320-7647b5c2e752 -12:20.655 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:20.656 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:20.656 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:20.656 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:20.656 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:20.656 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:20.656 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-13b3763f-415f-e5e1-9320-7647b5c2e752 -12:20.656 [INFO] TestAgent.server: member joined, marking health alive: member=Node-13b3763f-415f-e5e1-9320-7647b5c2e752 -12:20.657 [INFO] TestAgent.server: federation state anti-entropy synced -12:20.670 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:20.670 [INFO] TestAgent: Synced node info -12:20.959 [DEBUG] TestAgent: removed service: service=redis -12:20.959 [INFO] TestAgent: Requesting shutdown -12:20.959 [INFO] TestAgent.server: shutting down server -12:20.959 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:20.959 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:20.959 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:20.959 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:20.959 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:20.959 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:20.959 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:20.959 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:20.959 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:20.959 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:20.959 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:20.959Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:20.959Z [INFO] agent.router.manager: shutting down -12:20.959 [INFO] TestAgent: consul server down -12:20.959 [INFO] TestAgent: shutdown complete -12:20.959 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16869 network=tcp -12:20.959 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16869 network=udp -12:20.959 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16870 network=tcp -12:21.461 [INFO] TestAgent: Waiting for endpoints to shut down -12:21.461 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:36.025Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:36.039Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:36.041 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:91fd53f6-a98c-25b7-d431-466413c8b727 Address:127.0.0.1:24021}]" +48:36.041 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24021 [Follower]" leader= +48:36.042 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-91fd53f6-a98c-25b7-d431-466413c8b727.dc1 127.0.0.1 +48:36.042 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-91fd53f6-a98c-25b7-d431-466413c8b727 127.0.0.1 +2022-12-04T05:48:36.042Z [INFO] agent.router: Initializing LAN area manager +48:36.043 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24016 network=udp +48:36.043 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-91fd53f6-a98c-25b7-d431-466413c8b727.dc1 area=wan +48:36.043 [INFO] TestAgent.server: Adding LAN server: server="Node-91fd53f6-a98c-25b7-d431-466413c8b727 (Addr: tcp/127.0.0.1:24021) (DC: dc1)" +48:36.043 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24016 network=tcp +48:36.043 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24017 network=tcp +48:36.043 [INFO] TestAgent: started state syncer +48:36.043 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24022 network=tcp +48:36.103 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:36.103 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24021 [Candidate]" term=2 +48:36.103 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:36.103 [DEBUG] TestAgent.server.raft: vote granted: from=91fd53f6-a98c-25b7-d431-466413c8b727 term=2 tally=1 +48:36.103 [INFO] TestAgent.server.raft: election won: tally=1 +48:36.103 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24021 [Leader]" +48:36.104 [INFO] TestAgent.server: cluster leadership acquired +48:36.104 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24021 +48:36.104 [INFO] TestAgent.server: New leader elected: payload=Node-91fd53f6-a98c-25b7-d431-466413c8b727 +48:36.104 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:36.106 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:36.106 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:36.106 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:36.106 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:36.106 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:36.106 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-91fd53f6-a98c-25b7-d431-466413c8b727 +48:36.106 [INFO] TestAgent.server: member joined, marking health alive: member=Node-91fd53f6-a98c-25b7-d431-466413c8b727 +48:36.106 [INFO] TestAgent.server: federation state anti-entropy synced +48:36.112 [DEBUG] TestAgent: removed service: service=redis +48:36.112 [INFO] TestAgent: Requesting shutdown +48:36.112 [INFO] TestAgent.server: shutting down server +48:36.112 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:36.112 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:36.112 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:36.112 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:36.112 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:36.112 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:36.112 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:36.112 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:36.112 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:36.112 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:36.112 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:36.113 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:36.113Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:36.113Z [INFO] agent.router.manager: shutting down +48:36.113 [INFO] TestAgent: consul server down +48:36.113 [INFO] TestAgent: shutdown complete +48:36.113 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24016 network=tcp +48:36.113 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24016 network=udp +48:36.113 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24017 network=tcp +48:36.614 [INFO] TestAgent: Waiting for endpoints to shut down +48:36.614 [INFO] TestAgent: Endpoints down +[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:48:36.034Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:36.059Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:36.060 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5070f5dc-cd4f-b5d1-fb1d-27348c250263 Address:127.0.0.1:24014}]" +48:36.061 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24014 [Follower]" leader= +48:36.061 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5070f5dc-cd4f-b5d1-fb1d-27348c250263.dc1 127.0.0.1 +48:36.062 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5070f5dc-cd4f-b5d1-fb1d-27348c250263 127.0.0.1 +2022-12-04T05:48:36.062Z [INFO] agent.router: Initializing LAN area manager +48:36.062 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24009 network=udp +48:36.062 [INFO] TestAgent.server: Adding LAN server: server="Node-5070f5dc-cd4f-b5d1-fb1d-27348c250263 (Addr: tcp/127.0.0.1:24014) (DC: dc1)" +48:36.062 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5070f5dc-cd4f-b5d1-fb1d-27348c250263.dc1 area=wan +48:36.062 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24009 network=tcp +48:36.063 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24010 network=tcp +48:36.063 [INFO] TestAgent: started state syncer +48:36.063 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24015 network=tcp +48:36.130 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:36.130 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24014 [Candidate]" term=2 +48:36.130 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:36.130 [DEBUG] TestAgent.server.raft: vote granted: from=5070f5dc-cd4f-b5d1-fb1d-27348c250263 term=2 tally=1 +48:36.130 [INFO] TestAgent.server.raft: election won: tally=1 +48:36.130 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24014 [Leader]" +48:36.130 [INFO] TestAgent.server: cluster leadership acquired +48:36.131 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24014 +48:36.131 [INFO] TestAgent.server: New leader elected: payload=Node-5070f5dc-cd4f-b5d1-fb1d-27348c250263 +48:36.131 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:36.133 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:36.133 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:36.133 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:36.133 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:36.133 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:36.133 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5070f5dc-cd4f-b5d1-fb1d-27348c250263 +48:36.133 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5070f5dc-cd4f-b5d1-fb1d-27348c250263 +48:36.134 [INFO] TestAgent.server: federation state anti-entropy synced +48:36.452 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:36.452 [INFO] TestAgent: Synced node info +48:36.491 [DEBUG] TestAgent: removed service: service=redis +48:36.491 [INFO] TestAgent: Requesting shutdown +48:36.491 [INFO] TestAgent.server: shutting down server +48:36.491 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:36.491 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:36.491 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:36.491 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:36.491 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:36.491 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:36.491 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:36.491 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:36.491 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:36.491 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:36.491 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:36.491Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:36.491Z [INFO] agent.router.manager: shutting down +48:36.491 [INFO] TestAgent: consul server down +48:36.491 [INFO] TestAgent: shutdown complete +48:36.491 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24009 network=tcp +48:36.491 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24009 network=udp +48:36.491 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24010 network=tcp +48:36.992 [INFO] TestAgent: Waiting for endpoints to shut down +48:36.992 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_unloadServices (0.00s) - --- PASS: TestAgent_unloadServices/normal (0.95s) - --- PASS: TestAgent_unloadServices/service_manager (1.02s) + --- PASS: TestAgent_unloadServices/service_manager (0.61s) + --- PASS: TestAgent_unloadServices/normal (0.98s) === RUN TestAgent_Service_MaintenanceMode === PAUSE TestAgent_Service_MaintenanceMode === RUN TestAgent_Service_Reap -[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:21.488Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:21.514Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:21.516 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6929a247-c657-77f5-cb67-1748867ddffb Address:127.0.0.1:16881}]" -12:21.517 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6929a247-c657-77f5-cb67-1748867ddffb.dc1 127.0.0.1 -12:21.517 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6929a247-c657-77f5-cb67-1748867ddffb 127.0.0.1 -2021-10-31T23:12:21.517Z [INFO] agent.router: Initializing LAN area manager -12:21.517 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16881 [Follower]" leader= -12:21.518 [INFO] TestAgent.server: Adding LAN server: server="Node-6929a247-c657-77f5-cb67-1748867ddffb (Addr: tcp/127.0.0.1:16881) (DC: dc1)" -12:21.518 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6929a247-c657-77f5-cb67-1748867ddffb.dc1 area=wan -12:21.518 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16876 network=udp -12:21.518 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16876 network=tcp -12:21.518 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16877 network=tcp -12:21.518 [INFO] TestAgent: started state syncer -12:21.519 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16882 network=tcp -12:21.577 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:21.577 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16881 [Candidate]" term=2 -12:21.578 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:21.578 [DEBUG] TestAgent.server.raft: vote granted: from=6929a247-c657-77f5-cb67-1748867ddffb term=2 tally=1 -12:21.578 [INFO] TestAgent.server.raft: election won: tally=1 -12:21.578 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16881 [Leader]" -12:21.578 [INFO] TestAgent.server: cluster leadership acquired -12:21.578 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16881 -12:21.579 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:21.580 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:21.580 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:21.580 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:21.580 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:21.580 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:21.580 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6929a247-c657-77f5-cb67-1748867ddffb -12:21.580 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6929a247-c657-77f5-cb67-1748867ddffb -12:21.581 [INFO] TestAgent.server: New leader elected: payload=Node-6929a247-c657-77f5-cb67-1748867ddffb -12:21.581 [INFO] TestAgent.server: federation state anti-entropy synced -12:21.722 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:21.722 [INFO] TestAgent: Synced node info -12:21.723 [INFO] TestAgent: Synced service: service=redis -12:21.723 [DEBUG] TestAgent: Check in sync: check=service:redis -12:21.741 [WARN] TestAgent: Check missed TTL, is now critical: check=service:redis -12:21.817 [DEBUG] TestAgent: Check status updated: check=service:redis status=passing -12:21.845 [WARN] TestAgent: Check missed TTL, is now critical: check=service:redis -12:22.090 [DEBUG] TestAgent: removed check: check=service:redis -12:22.090 [DEBUG] TestAgent: removed service: service=redis -12:22.090 [INFO] TestAgent: Check for service has been critical for too long; deregistered service: service=redis check=service:redis -12:22.322 [INFO] TestAgent: Requesting shutdown -12:22.322 [INFO] TestAgent.server: shutting down server -12:22.322 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:22.322 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:22.322 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:22.322 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:22.322 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:22.323 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:22.323 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:22.323 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:22.323 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:22.323 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:22.323 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:22.323Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:22.323Z [INFO] agent.router.manager: shutting down -12:22.323 [INFO] TestAgent: consul server down -12:22.323 [INFO] TestAgent: shutdown complete -12:22.323 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16876 network=tcp -12:22.323 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16876 network=udp -12:22.323 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16877 network=tcp -12:22.825 [INFO] TestAgent: Waiting for endpoints to shut down -12:22.825 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Service_Reap (1.36s) +2022-12-04T05:48:37.022Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:37.051Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:37.057 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:087a248e-cb7e-cc7d-bd15-778b772a566b Address:127.0.0.1:24028}]" +48:37.057 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-087a248e-cb7e-cc7d-bd15-778b772a566b.dc1 127.0.0.1 +48:37.058 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-087a248e-cb7e-cc7d-bd15-778b772a566b 127.0.0.1 +2022-12-04T05:48:37.058Z [INFO] agent.router: Initializing LAN area manager +48:37.058 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24023 network=udp +48:37.058 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24028 [Follower]" leader= +48:37.058 [INFO] TestAgent.server: Adding LAN server: server="Node-087a248e-cb7e-cc7d-bd15-778b772a566b (Addr: tcp/127.0.0.1:24028) (DC: dc1)" +48:37.058 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-087a248e-cb7e-cc7d-bd15-778b772a566b.dc1 area=wan +48:37.058 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24023 network=tcp +48:37.059 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24024 network=tcp +48:37.059 [INFO] TestAgent: started state syncer +48:37.059 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24029 network=tcp +48:37.124 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:37.124 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24028 [Candidate]" term=2 +48:37.124 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:37.124 [DEBUG] TestAgent.server.raft: vote granted: from=087a248e-cb7e-cc7d-bd15-778b772a566b term=2 tally=1 +48:37.124 [INFO] TestAgent.server.raft: election won: tally=1 +48:37.124 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24028 [Leader]" +48:37.124 [INFO] TestAgent.server: cluster leadership acquired +48:37.124 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24028 +48:37.125 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:37.127 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:37.127 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:37.127 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:37.127 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:37.127 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:37.127 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-087a248e-cb7e-cc7d-bd15-778b772a566b +48:37.127 [INFO] TestAgent.server: member joined, marking health alive: member=Node-087a248e-cb7e-cc7d-bd15-778b772a566b +48:37.127 [INFO] TestAgent.server: New leader elected: payload=Node-087a248e-cb7e-cc7d-bd15-778b772a566b +48:37.128 [INFO] TestAgent.server: federation state anti-entropy synced +48:37.223 [WARN] TestAgent: Check missed TTL, is now critical: check=service:redis +48:37.299 [DEBUG] TestAgent: Check status updated: check=service:redis status=passing +48:37.328 [WARN] TestAgent: Check missed TTL, is now critical: check=service:redis +48:37.448 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:37.449 [INFO] TestAgent: Synced node info +48:37.449 [INFO] TestAgent: Synced service: service=redis +48:37.449 [DEBUG] TestAgent: Check in sync: check=service:redis +48:37.533 [DEBUG] TestAgent: removed check: check=service:redis +48:37.533 [DEBUG] TestAgent: removed service: service=redis +48:37.533 [INFO] TestAgent: Check for service has been critical for too long; deregistered service: service=redis check=service:redis +48:37.812 [INFO] TestAgent: Requesting shutdown +48:37.820 [INFO] TestAgent.server: shutting down server +48:37.820 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:37.820 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:37.820 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:37.820 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:37.820 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:37.821 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:37.821 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:37.821 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:37.821 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:37.821 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:37.821 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:48:37.821Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:37.821Z [INFO] agent.router.manager: shutting down +48:37.821 [INFO] TestAgent: consul server down +48:37.821 [INFO] TestAgent: shutdown complete +48:37.821 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24023 network=tcp +48:37.821 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24023 network=udp +48:37.821 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24024 network=tcp +48:38.324 [INFO] TestAgent: Waiting for endpoints to shut down +48:38.324 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Service_Reap (1.33s) === RUN TestAgent_Service_NoReap [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:22.866Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:22.951Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:22.953 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:68a67917-7f64-06ac-6893-d9a89968e262 Address:127.0.0.1:16888}]" -12:22.954 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16888 [Follower]" leader= -12:22.960 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-68a67917-7f64-06ac-6893-d9a89968e262.dc1 127.0.0.1 -12:22.961 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-68a67917-7f64-06ac-6893-d9a89968e262 127.0.0.1 -2021-10-31T23:12:22.961Z [INFO] agent.router: Initializing LAN area manager -12:22.961 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16883 network=udp -12:22.961 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-68a67917-7f64-06ac-6893-d9a89968e262.dc1 area=wan -12:22.962 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16883 network=tcp -12:22.963 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16884 network=tcp -12:22.963 [INFO] TestAgent: started state syncer -12:22.963 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16889 network=tcp -12:22.969 [INFO] TestAgent.server: Adding LAN server: server="Node-68a67917-7f64-06ac-6893-d9a89968e262 (Addr: tcp/127.0.0.1:16888) (DC: dc1)" -12:23.009 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:23.009 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16888 [Candidate]" term=2 -12:23.010 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:23.010 [DEBUG] TestAgent.server.raft: vote granted: from=68a67917-7f64-06ac-6893-d9a89968e262 term=2 tally=1 -12:23.010 [INFO] TestAgent.server.raft: election won: tally=1 -12:23.010 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16888 [Leader]" -12:23.010 [INFO] TestAgent.server: cluster leadership acquired -12:23.010 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16888 -12:23.011 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:23.012 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:23.012 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:23.012 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:23.012 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:23.012 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:23.012 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-68a67917-7f64-06ac-6893-d9a89968e262 -12:23.012 [INFO] TestAgent.server: member joined, marking health alive: member=Node-68a67917-7f64-06ac-6893-d9a89968e262 -12:23.012 [INFO] TestAgent.server: New leader elected: payload=Node-68a67917-7f64-06ac-6893-d9a89968e262 -12:23.013 [INFO] TestAgent.server: federation state anti-entropy synced -12:23.073 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:23.074 [INFO] TestAgent: Synced node info -12:23.237 [WARN] TestAgent: Check missed TTL, is now critical: check=service:redis -12:23.618 [INFO] TestAgent: Requesting shutdown -12:23.618 [INFO] TestAgent.server: shutting down server -12:23.618 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:23.618 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:23.618 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:23.618 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:23.618 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:23.619 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:23.619 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:23.619 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:23.619 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:23.619 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:23.619 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:23.619Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:23.619Z [INFO] agent.router.manager: shutting down -12:23.619 [INFO] TestAgent: consul server down -12:23.619 [INFO] TestAgent: shutdown complete -12:23.619 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16883 network=tcp -12:23.619 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16883 network=udp -12:23.619 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16884 network=tcp -12:24.119 [INFO] TestAgent: Waiting for endpoints to shut down -12:24.119 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Service_NoReap (1.29s) +2022-12-04T05:48:38.355Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:38.385Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:38.387 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bc1bd818-f4bb-41fe-b359-b1a034184c39 Address:127.0.0.1:24035}]" +48:38.388 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bc1bd818-f4bb-41fe-b359-b1a034184c39.dc1 127.0.0.1 +48:38.388 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24035 [Follower]" leader= +48:38.397 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bc1bd818-f4bb-41fe-b359-b1a034184c39 127.0.0.1 +2022-12-04T05:48:38.397Z [INFO] agent.router: Initializing LAN area manager +48:38.397 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24030 network=udp +48:38.397 [INFO] TestAgent.server: Adding LAN server: server="Node-bc1bd818-f4bb-41fe-b359-b1a034184c39 (Addr: tcp/127.0.0.1:24035) (DC: dc1)" +48:38.397 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bc1bd818-f4bb-41fe-b359-b1a034184c39.dc1 area=wan +48:38.398 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24030 network=tcp +48:38.398 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24031 network=tcp +48:38.398 [INFO] TestAgent: started state syncer +48:38.398 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24036 network=tcp +48:38.443 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:38.443 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24035 [Candidate]" term=2 +48:38.444 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:38.444 [DEBUG] TestAgent.server.raft: vote granted: from=bc1bd818-f4bb-41fe-b359-b1a034184c39 term=2 tally=1 +48:38.444 [INFO] TestAgent.server.raft: election won: tally=1 +48:38.444 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24035 [Leader]" +48:38.444 [INFO] TestAgent.server: cluster leadership acquired +48:38.444 [INFO] TestAgent.server: New leader elected: payload=Node-bc1bd818-f4bb-41fe-b359-b1a034184c39 +48:38.444 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24035 +48:38.445 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:38.446 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:38.446 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:38.446 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:38.446 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:38.446 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:38.446 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bc1bd818-f4bb-41fe-b359-b1a034184c39 +48:38.446 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bc1bd818-f4bb-41fe-b359-b1a034184c39 +48:38.447 [INFO] TestAgent.server: federation state anti-entropy synced +48:38.651 [WARN] TestAgent: Check missed TTL, is now critical: check=service:redis +48:38.687 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:38.687 [INFO] TestAgent: Synced node info +48:38.688 [INFO] TestAgent: Synced service: service=redis +48:38.688 [DEBUG] TestAgent: Check in sync: check=service:redis +48:39.032 [INFO] TestAgent: Requesting shutdown +48:39.032 [INFO] TestAgent.server: shutting down server +48:39.032 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:39.032 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:39.032 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:39.032 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:39.032 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:39.032 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:39.032 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:39.032 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:39.032 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:39.032 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:39.032 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:39.033Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:39.033Z [INFO] agent.router.manager: shutting down +48:39.033 [INFO] TestAgent: consul server down +48:39.033 [INFO] TestAgent: shutdown complete +48:39.033 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24030 network=tcp +48:39.033 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24030 network=udp +48:39.033 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24031 network=tcp +48:39.536 [INFO] TestAgent: Waiting for endpoints to shut down +48:39.536 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Service_NoReap (1.21s) === RUN TestAgent_AddService_restoresSnapshot === RUN TestAgent_AddService_restoresSnapshot/normal === PAUSE TestAgent_AddService_restoresSnapshot/normal @@ -12961,119 +13007,122 @@ [WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:24.151Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:24.174Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:24.175 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:43690ba0-e8c3-6c06-405d-4308315d1066 Address:127.0.0.1:16895}]" -12:24.176 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-43690ba0-e8c3-6c06-405d-4308315d1066.dc1 127.0.0.1 -12:24.177 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-43690ba0-e8c3-6c06-405d-4308315d1066 127.0.0.1 -2021-10-31T23:12:24.177Z [INFO] agent.router: Initializing LAN area manager -12:24.177 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16890 network=udp -12:24.177 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16895 [Follower]" leader= -12:24.177 [INFO] TestAgent.server: Adding LAN server: server="Node-43690ba0-e8c3-6c06-405d-4308315d1066 (Addr: tcp/127.0.0.1:16895) (DC: dc1)" -12:24.178 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16890 network=tcp -12:24.178 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16891 network=tcp -12:24.178 [INFO] TestAgent: started state syncer -12:24.179 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16896 network=tcp -12:24.182 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-43690ba0-e8c3-6c06-405d-4308315d1066.dc1 area=wan -12:24.229 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:24.229 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16895 [Candidate]" term=2 -12:24.230 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:24.230 [DEBUG] TestAgent.server.raft: vote granted: from=43690ba0-e8c3-6c06-405d-4308315d1066 term=2 tally=1 -12:24.230 [INFO] TestAgent.server.raft: election won: tally=1 -12:24.230 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16895 [Leader]" -12:24.230 [INFO] TestAgent.server: cluster leadership acquired -12:24.230 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16895 -12:24.231 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:24.233 [INFO] TestAgent.server: New leader elected: payload=Node-43690ba0-e8c3-6c06-405d-4308315d1066 -12:24.239 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:24.239 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:24.239 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:24.239 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:24.239 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:24.239 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-43690ba0-e8c3-6c06-405d-4308315d1066 -12:24.239 [INFO] TestAgent.server: member joined, marking health alive: member=Node-43690ba0-e8c3-6c06-405d-4308315d1066 -12:24.240 [INFO] TestAgent.server: federation state anti-entropy synced -12:24.355 [INFO] TestAgent: Requesting shutdown -12:24.355 [INFO] TestAgent.server: shutting down server -12:24.355 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:24.355 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:24.355 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:24.355 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:24.355 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:24.355 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:24.355 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:24.356 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:24.356 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:24.356 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:24.356 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:24.356 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:24.356Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:24.356Z [INFO] agent.router.manager: shutting down -12:24.356 [INFO] TestAgent: consul server down -12:24.356 [INFO] TestAgent: shutdown complete -12:24.356 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16890 network=tcp -12:24.356 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16890 network=udp -12:24.356 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16891 network=tcp -12:24.857 [INFO] TestAgent: Waiting for endpoints to shut down -12:24.857 [INFO] TestAgent: Endpoints down -2021-10-31T23:12:24.143Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:24.191Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:24.193 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0ab3bfd1-b599-056a-c562-7702ed654802 Address:127.0.0.1:16902}]" -12:24.193 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0ab3bfd1-b599-056a-c562-7702ed654802.dc1 127.0.0.1 -12:24.193 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16902 [Follower]" leader= -12:24.194 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0ab3bfd1-b599-056a-c562-7702ed654802 127.0.0.1 -2021-10-31T23:12:24.194Z [INFO] agent.router: Initializing LAN area manager -12:24.194 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16897 network=udp -12:24.194 [INFO] TestAgent.server: Adding LAN server: server="Node-0ab3bfd1-b599-056a-c562-7702ed654802 (Addr: tcp/127.0.0.1:16902) (DC: dc1)" -12:24.194 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0ab3bfd1-b599-056a-c562-7702ed654802.dc1 area=wan -12:24.194 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16897 network=tcp -12:24.195 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16898 network=tcp -12:24.195 [INFO] TestAgent: started state syncer -12:24.195 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16903 network=tcp -12:24.261 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:24.261 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16902 [Candidate]" term=2 -12:24.262 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:24.262 [DEBUG] TestAgent.server.raft: vote granted: from=0ab3bfd1-b599-056a-c562-7702ed654802 term=2 tally=1 -12:24.262 [INFO] TestAgent.server.raft: election won: tally=1 -12:24.262 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16902 [Leader]" -12:24.262 [INFO] TestAgent.server: cluster leadership acquired -12:24.262 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16902 -12:24.263 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:24.264 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:24.264 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:24.264 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:24.264 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:24.264 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:24.264 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0ab3bfd1-b599-056a-c562-7702ed654802 -12:24.264 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0ab3bfd1-b599-056a-c562-7702ed654802 -12:24.264 [INFO] TestAgent.server: New leader elected: payload=Node-0ab3bfd1-b599-056a-c562-7702ed654802 -12:24.265 [INFO] TestAgent.server: federation state anti-entropy synced -12:24.393 [INFO] TestAgent: Requesting shutdown -12:24.393 [INFO] TestAgent.server: shutting down server -12:24.393 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:24.393 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:24.393 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:24.393 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:24.393 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:24.393 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:24.393 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:24.393 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:24.393 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:24.393 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:24.393 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:24.393 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:24.393Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:24.393Z [INFO] agent.router.manager: shutting down -12:24.393 [INFO] TestAgent: consul server down -12:24.393 [INFO] TestAgent: shutdown complete -12:24.393 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16897 network=tcp -12:24.393 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16897 network=udp -12:24.393 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16898 network=tcp -12:24.893 [INFO] TestAgent: Waiting for endpoints to shut down -12:24.893 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:39.560Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:39.574Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:39.576 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6ccbd421-1f03-71e2-980d-bc100c40b3cc Address:127.0.0.1:24049}]" +48:39.579 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24049 [Follower]" leader= +48:39.582 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6ccbd421-1f03-71e2-980d-bc100c40b3cc.dc1 127.0.0.1 +48:39.583 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6ccbd421-1f03-71e2-980d-bc100c40b3cc 127.0.0.1 +2022-12-04T05:48:39.583Z [INFO] agent.router: Initializing LAN area manager +48:39.583 [INFO] TestAgent.server: Adding LAN server: server="Node-6ccbd421-1f03-71e2-980d-bc100c40b3cc (Addr: tcp/127.0.0.1:24049) (DC: dc1)" +48:39.583 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6ccbd421-1f03-71e2-980d-bc100c40b3cc.dc1 area=wan +48:39.583 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24044 network=udp +48:39.583 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24044 network=tcp +48:39.584 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24045 network=tcp +48:39.584 [INFO] TestAgent: started state syncer +48:39.584 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24050 network=tcp +48:39.624 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:39.624 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24049 [Candidate]" term=2 +48:39.624 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:39.624 [DEBUG] TestAgent.server.raft: vote granted: from=6ccbd421-1f03-71e2-980d-bc100c40b3cc term=2 tally=1 +48:39.624 [INFO] TestAgent.server.raft: election won: tally=1 +48:39.624 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24049 [Leader]" +48:39.624 [INFO] TestAgent.server: cluster leadership acquired +48:39.624 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24049 +48:39.625 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:39.627 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:39.627 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:39.627 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:39.627 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:39.627 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:39.627 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6ccbd421-1f03-71e2-980d-bc100c40b3cc +48:39.627 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6ccbd421-1f03-71e2-980d-bc100c40b3cc +48:39.627 [INFO] TestAgent.server: New leader elected: payload=Node-6ccbd421-1f03-71e2-980d-bc100c40b3cc +48:39.627 [INFO] TestAgent.server: federation state anti-entropy synced +48:39.708 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:39.708 [INFO] TestAgent: Synced node info +48:39.708 [DEBUG] TestAgent: Node info in sync +48:39.724 [INFO] TestAgent: Requesting shutdown +48:39.724 [INFO] TestAgent.server: shutting down server +48:39.724 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:39.724 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:39.724 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:39.724 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:39.724 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:39.724 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:39.724 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:39.724 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:39.724 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:39.724 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:39.724 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:39.725Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:39.725Z [INFO] agent.router.manager: shutting down +48:39.725 [INFO] TestAgent: consul server down +48:39.725 [INFO] TestAgent: shutdown complete +48:39.725 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24044 network=tcp +48:39.725 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24044 network=udp +48:39.725 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24045 network=tcp +48:40.228 [INFO] TestAgent: Waiting for endpoints to shut down +48:40.228 [INFO] TestAgent: Endpoints down +[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:48:39.565Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:39.588Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:39.593 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0dc19f54-5447-e32f-4a6a-5c2e76ef238e Address:127.0.0.1:24042}]" +48:39.593 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24042 [Follower]" leader= +48:39.594 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0dc19f54-5447-e32f-4a6a-5c2e76ef238e.dc1 127.0.0.1 +48:39.594 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0dc19f54-5447-e32f-4a6a-5c2e76ef238e 127.0.0.1 +2022-12-04T05:48:39.594Z [INFO] agent.router: Initializing LAN area manager +48:39.594 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24037 network=udp +48:39.594 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0dc19f54-5447-e32f-4a6a-5c2e76ef238e.dc1 area=wan +48:39.594 [INFO] TestAgent.server: Adding LAN server: server="Node-0dc19f54-5447-e32f-4a6a-5c2e76ef238e (Addr: tcp/127.0.0.1:24042) (DC: dc1)" +48:39.595 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24037 network=tcp +48:39.595 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24038 network=tcp +48:39.595 [INFO] TestAgent: started state syncer +48:39.595 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24043 network=tcp +48:39.636 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:39.636 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24042 [Candidate]" term=2 +48:39.636 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:39.636 [DEBUG] TestAgent.server.raft: vote granted: from=0dc19f54-5447-e32f-4a6a-5c2e76ef238e term=2 tally=1 +48:39.636 [INFO] TestAgent.server.raft: election won: tally=1 +48:39.636 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24042 [Leader]" +48:39.636 [INFO] TestAgent.server: cluster leadership acquired +48:39.636 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24042 +48:39.637 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:39.639 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:39.639 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:39.639 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:39.639 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:39.639 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:39.639 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0dc19f54-5447-e32f-4a6a-5c2e76ef238e +48:39.639 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0dc19f54-5447-e32f-4a6a-5c2e76ef238e +48:39.639 [INFO] TestAgent.server: New leader elected: payload=Node-0dc19f54-5447-e32f-4a6a-5c2e76ef238e +48:39.639 [INFO] TestAgent.server: federation state anti-entropy synced +48:39.852 [INFO] TestAgent: Requesting shutdown +48:39.852 [INFO] TestAgent.server: shutting down server +48:39.852 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:39.852 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:39.852 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:39.852 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:39.852 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:39.852 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:39.852 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:39.852 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:39.852 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:39.852 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:39.852 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:39.853 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:39.853Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:39.853Z [INFO] agent.router.manager: shutting down +48:39.853 [INFO] TestAgent: consul server down +48:39.853 [INFO] TestAgent: shutdown complete +48:39.853 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24037 network=tcp +48:39.853 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24037 network=udp +48:39.853 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24038 network=tcp +48:40.356 [INFO] TestAgent: Waiting for endpoints to shut down +48:40.356 [INFO] TestAgent: Endpoints down --- PASS: TestAgent_AddService_restoresSnapshot (0.00s) - --- PASS: TestAgent_AddService_restoresSnapshot/normal (0.74s) - --- PASS: TestAgent_AddService_restoresSnapshot/service_manager (0.77s) + --- PASS: TestAgent_AddService_restoresSnapshot/service_manager (0.69s) + --- PASS: TestAgent_AddService_restoresSnapshot/normal (0.82s) === RUN TestAgent_AddCheck_restoresSnapshot === PAUSE TestAgent_AddCheck_restoresSnapshot === RUN TestAgent_NodeMaintenanceMode @@ -13180,8 +13229,8 @@ === RUN TestConfigSourceFromName/REMOTE === RUN TestConfigSourceFromName/garbage === RUN TestConfigSourceFromName/_ ---- PASS: TestConfigSourceFromName (0.00s) - --- PASS: TestConfigSourceFromName/local (0.00s) +--- PASS: TestConfigSourceFromName (0.01s) + --- PASS: TestConfigSourceFromName/local (0.01s) --- PASS: TestConfigSourceFromName/remote (0.00s) --- PASS: TestConfigSourceFromName/#00 (0.00s) --- PASS: TestConfigSourceFromName/LOCAL (0.00s) @@ -13198,10 +13247,10 @@ === RUN TestAgent_JoinWAN_viaMeshGateway === PAUSE TestAgent_JoinWAN_viaMeshGateway === RUN TestAutoConfig_Integration -[WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 6 out of 13 pending ports are still in use; something probably didn't wait around for the port to be closed! + agent_test.go:4731: Forcing roots rotation on the server [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! - agent_test.go:4731: Forcing roots rotation on the server [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! @@ -13219,174 +13268,170 @@ [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:25.539Z [INFO] agent.auto_config: retrieving initial agent auto configuration remotely -12:25.678 [INFO] test-client.client.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 -2021-10-31T23:12:25.678Z [INFO] agent.router: Initializing LAN area manager -2021-10-31T23:12:25.678Z [INFO] agent.auto_config: auto-config started -12:25.678 [INFO] test-client: Started DNS server: address=127.0.0.1:16911 network=udp -2021-10-31T23:12:25.678Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.678Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.679Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.679Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.679Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.679Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.679Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.679Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.681Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -12:25.682 [INFO] test-client: Started DNS server: address=127.0.0.1:16911 network=tcp -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -12:25.682 [INFO] test-client: Started HTTP server: address=127.0.0.1:16912 network=tcp -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -12:25.682 [INFO] test-client: started state syncer -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -12:25.682 [INFO] test-client: Retry join is supported for the following discovery methods: cluster=LAN discovery_methods="aliyun aws gce mdns os packet" -12:25.682 [INFO] test-client: Joining cluster...: cluster=LAN -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -12:25.682 [INFO] test-client: (LAN) joining: lan_addresses=[127.0.0.1:16907] -2021-10-31T23:12:25.682Z [WARN] agent.router.manager: No servers available -12:25.682 [ERROR] test-client.anti_entropy: failed to sync remote state: error="No known Consul servers" -12:25.682 [INFO] test-client: Started gRPC server: address=127.0.0.1:16917 network=tcp -12:25.684 [DEBUG] test-client.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:16907 -12:25.686 [INFO] test-client.client.serf.lan: serf: EventMemberJoin: Node-75f7aedc-4779-6c4d-c52d-a410543d511d 127.0.0.1 -12:25.686 [INFO] test-client: (LAN) joined: number_of_nodes=1 -12:25.686 [DEBUG] test-client: systemd notify failed: error="No socket" -12:25.686 [INFO] test-client: Join cluster completed. Synced with initial agents: cluster=LAN num_agents=1 -12:25.686 [INFO] test-client.client: adding server: server="Node-75f7aedc-4779-6c4d-c52d-a410543d511d (Addr: tcp/127.0.0.1:16909) (DC: dc1)" -12:25.730 [DEBUG] test-client.client: transitioned out of legacy ACL mode -12:25.730 [INFO] test-client.client.serf.lan: serf: EventMemberUpdate: test-client -12:25.834 [DEBUG] test-client.client.serf.lan: serf: messageUserEventType: consul:new-leader -12:26.038 [DEBUG] test-client.client.serf.lan: serf: messageUserEventType: consul:new-leader -12:26.038 [DEBUG] test-client.client.serf.lan: serf: messageJoinType: test-client -12:26.038 [DEBUG] test-client.client.serf.lan: serf: messageUserEventType: consul:new-leader -12:26.038 [DEBUG] test-client.client.serf.lan: serf: messageJoinType: test-client -12:26.234 [DEBUG] test-client.client.serf.lan: serf: messageUserEventType: consul:new-leader -12:26.234 [DEBUG] test-client.client.serf.lan: serf: messageJoinType: test-client -12:26.438 [DEBUG] test-client.client.serf.lan: serf: messageJoinType: test-client -12:26.832 [DEBUG] test-client: Skipping remote check since it is managed automatically: check=serfHealth -12:26.832 [INFO] test-client: Synced node info -12:26.832 [DEBUG] test-client: Node info in sync -12:29.087 [INFO] test-client: Requesting shutdown -12:29.087 [INFO] test-client.client: shutting down client -12:29.087 [WARN] test-client.client.serf.lan: serf: Shutdown without a Leave -12:29.087 [INFO] test-client: consul client down -12:29.087 [INFO] test-client: shutdown complete -12:29.087 [INFO] test-client: Stopping server: protocol=DNS address=127.0.0.1:16911 network=tcp -12:29.087 [INFO] test-client: Stopping server: protocol=DNS address=127.0.0.1:16911 network=udp -12:29.087 [INFO] test-client: Stopping server: protocol=HTTP address=127.0.0.1:16912 network=tcp -12:29.087 [ERROR] test-client.client: RPC failed to server: method=ConnectCA.Roots server=127.0.0.1:16909 error="rpc error making call: EOF" -12:29.590 [INFO] test-client: Waiting for endpoints to shut down -12:29.590 [INFO] test-client: Endpoints down -2021-10-31T23:12:24.962Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:25.016Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:25.031 [INFO] TestAgent-Server.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:75f7aedc-4779-6c4d-c52d-a410543d511d Address:127.0.0.1:16909}]" -12:25.031 [INFO] TestAgent-Server.server.serf.wan: serf: EventMemberJoin: Node-75f7aedc-4779-6c4d-c52d-a410543d511d.dc1 127.0.0.1 -12:25.032 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberJoin: Node-75f7aedc-4779-6c4d-c52d-a410543d511d 127.0.0.1 -2021-10-31T23:12:25.032Z [INFO] agent.router: Initializing LAN area manager -12:25.032 [INFO] TestAgent-Server: Started DNS server: address=127.0.0.1:16904 network=udp -12:25.032 [INFO] TestAgent-Server.server.raft: entering follower state: follower="Node at 127.0.0.1:16909 [Follower]" leader= -12:25.032 [INFO] TestAgent-Server.server: Adding LAN server: server="Node-75f7aedc-4779-6c4d-c52d-a410543d511d (Addr: tcp/127.0.0.1:16909) (DC: dc1)" -12:25.032 [INFO] TestAgent-Server.server: Handled event for server in area: event=member-join server=Node-75f7aedc-4779-6c4d-c52d-a410543d511d.dc1 area=wan -12:25.033 [INFO] TestAgent-Server: Started DNS server: address=127.0.0.1:16904 network=tcp -12:25.033 [INFO] TestAgent-Server: Started HTTP server: address=127.0.0.1:16905 network=tcp -12:25.033 [INFO] TestAgent-Server: started state syncer -12:25.033 [INFO] TestAgent-Server: Started gRPC server: address=127.0.0.1:16910 network=tcp -12:25.081 [WARN] TestAgent-Server.server.raft: heartbeat timeout reached, starting election: last-leader= -12:25.082 [INFO] TestAgent-Server.server.raft: entering candidate state: node="Node at 127.0.0.1:16909 [Candidate]" term=2 -12:25.082 [DEBUG] TestAgent-Server.server.raft: votes: needed=1 -12:25.082 [DEBUG] TestAgent-Server.server.raft: vote granted: from=75f7aedc-4779-6c4d-c52d-a410543d511d term=2 tally=1 -12:25.082 [INFO] TestAgent-Server.server.raft: election won: tally=1 -12:25.082 [INFO] TestAgent-Server.server.raft: entering leader state: leader="Node at 127.0.0.1:16909 [Leader]" -12:25.082 [INFO] TestAgent-Server.server: cluster leadership acquired -12:25.082 [INFO] TestAgent-Server.server: initializing acls -12:25.082 [INFO] TestAgent-Server.server: Created ACL 'global-management' policy -12:25.083 [INFO] TestAgent-Server.server: initializing acls -12:25.083 [INFO] TestAgent-Server.server: New leader elected: payload=Node-75f7aedc-4779-6c4d-c52d-a410543d511d -12:25.083 [INFO] TestAgent-Server.server: Bootstrapped ACL master token from configuration -12:25.083 [INFO] TestAgent-Server.server: Created ACL anonymous token from configuration -12:25.083 [INFO] TestAgent-Server.leader: started routine: routine="legacy ACL token upgrade" -12:25.083 [INFO] TestAgent-Server.leader: started routine: routine="acl token reaping" -12:25.084 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberUpdate: Node-75f7aedc-4779-6c4d-c52d-a410543d511d -12:25.084 [INFO] TestAgent-Server.server.serf.wan: serf: EventMemberUpdate: Node-75f7aedc-4779-6c4d-c52d-a410543d511d.dc1 -12:25.084 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:25.085 [INFO] TestAgent-Server.server: Bootstrapped ACL master token from configuration -12:25.085 [DEBUG] TestAgent-Server.server: transitioning out of legacy ACL mode -12:25.085 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberUpdate: Node-75f7aedc-4779-6c4d-c52d-a410543d511d -12:25.085 [INFO] TestAgent-Server.server.serf.wan: serf: EventMemberUpdate: Node-75f7aedc-4779-6c4d-c52d-a410543d511d.dc1 -12:25.085 [INFO] TestAgent-Server.server: Handled event for server in area: event=member-update server=Node-75f7aedc-4779-6c4d-c52d-a410543d511d.dc1 area=wan -12:25.085 [INFO] TestAgent-Server.server: Handled event for server in area: event=member-update server=Node-75f7aedc-4779-6c4d-c52d-a410543d511d.dc1 area=wan -12:25.085 [INFO] TestAgent-Server.server: Updating LAN server: server="Node-75f7aedc-4779-6c4d-c52d-a410543d511d (Addr: tcp/127.0.0.1:16909) (DC: dc1)" -12:25.085 [INFO] TestAgent-Server.server: Updating LAN server: server="Node-75f7aedc-4779-6c4d-c52d-a410543d511d (Addr: tcp/127.0.0.1:16909) (DC: dc1)" -12:25.090 [INFO] TestAgent-Server.server.connect: initialized primary datacenter CA with provider: provider=consul -12:25.090 [INFO] TestAgent-Server.leader: started routine: routine="federation state anti-entropy" -12:25.090 [INFO] TestAgent-Server.leader: started routine: routine="federation state pruning" -12:25.090 [INFO] TestAgent-Server.leader: started routine: routine="intermediate cert renew watch" -12:25.090 [INFO] TestAgent-Server.leader: started routine: routine="CA root pruning" -12:25.090 [DEBUG] TestAgent-Server.server: Skipping self join check for node since the cluster is too small: node=Node-75f7aedc-4779-6c4d-c52d-a410543d511d -12:25.090 [INFO] TestAgent-Server.server: member joined, marking health alive: member=Node-75f7aedc-4779-6c4d-c52d-a410543d511d -12:25.091 [DEBUG] TestAgent-Server.server: Skipping self join check for node since the cluster is too small: node=Node-75f7aedc-4779-6c4d-c52d-a410543d511d -12:25.091 [DEBUG] TestAgent-Server.server: Skipping self join check for node since the cluster is too small: node=Node-75f7aedc-4779-6c4d-c52d-a410543d511d -12:25.091 [INFO] TestAgent-Server.server: federation state anti-entropy synced -12:25.390 [DEBUG] TestAgent-Server: Skipping remote check since it is managed automatically: check=serfHealth -12:25.390 [INFO] TestAgent-Server: Synced node info -12:25.434 [DEBUG] TestAgent-Server.acl: dropping node from result due to ACLs: node=Node-75f7aedc-4779-6c4d-c52d-a410543d511d -12:25.684 [DEBUG] TestAgent-Server.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:36590 -12:25.685 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 -12:25.685 [INFO] TestAgent-Server.server: member joined, marking health alive: member=test-client -12:25.735 [DEBUG] connect.ca.consul: consul CA provider configured: id=f3:37:c1:83:df:38:f7:28:97:ac:c3:d6:84:1e:8a:63:dd:e4:e5:e2:cb:c8:81:97:33:fb:ec:ff:e7:4b:7e:46 is_primary=true -12:25.736 [INFO] TestAgent-Server.server.connect: CA rotated to new root under provider: provider=consul -12:25.882 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberUpdate: test-client -12:25.882 [DEBUG] TestAgent-Server.server.serf.lan: serf: messageJoinType: test-client -12:26.042 [DEBUG] TestAgent-Server.server.serf.lan: serf: messageJoinType: test-client -12:26.106 [DEBUG] TestAgent-Server.server.serf.lan: serf: messageJoinType: test-client -12:26.282 [DEBUG] TestAgent-Server.server.serf.lan: serf: messageJoinType: test-client -12:26.990 [DEBUG] TestAgent-Server: Skipping remote check since it is managed automatically: check=serfHealth -12:26.990 [DEBUG] TestAgent-Server: Node info in sync -12:26.990 [DEBUG] TestAgent-Server: Node info in sync -12:29.590 [INFO] TestAgent-Server: Requesting shutdown -12:29.590 [INFO] TestAgent-Server.server: shutting down server -12:29.590 [DEBUG] TestAgent-Server.leader: stopping routine: routine="legacy ACL token upgrade" -12:29.590 [DEBUG] TestAgent-Server.leader: stopping routine: routine="acl token reaping" -12:29.590 [DEBUG] TestAgent-Server.leader: stopping routine: routine="federation state anti-entropy" -12:29.590 [DEBUG] TestAgent-Server.leader: stopping routine: routine="federation state pruning" -12:29.590 [DEBUG] TestAgent-Server.leader: stopping routine: routine="intermediate cert renew watch" -12:29.590 [DEBUG] TestAgent-Server.leader: stopping routine: routine="CA root pruning" -12:29.590 [WARN] TestAgent-Server.server.serf.lan: serf: Shutdown without a Leave -12:29.590 [INFO] TestAgent-Server.server: shutting down Connect CA roots monitor -12:29.590 [ERROR] TestAgent-Server.server: error performing anti-entropy sync of federation state: error="context canceled" -12:29.590 [DEBUG] TestAgent-Server.leader: stopped routine: routine="federation state anti-entropy" -12:29.590 [DEBUG] TestAgent-Server.leader: stopped routine: routine="legacy ACL token upgrade" -12:29.590 [DEBUG] TestAgent-Server.leader: stopped routine: routine="acl token reaping" -12:29.590 [DEBUG] TestAgent-Server.leader: stopped routine: routine="federation state pruning" -12:29.590 [DEBUG] TestAgent-Server.leader: stopped routine: routine="intermediate cert renew watch" -12:29.590 [DEBUG] TestAgent-Server.leader: stopped routine: routine="CA root pruning" -12:29.590 [WARN] TestAgent-Server.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:29.590Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:29.590Z [INFO] agent.router.manager: shutting down -12:29.590 [INFO] TestAgent-Server: consul server down -12:29.590 [INFO] TestAgent-Server: shutdown complete -12:29.590 [INFO] TestAgent-Server: Stopping server: protocol=DNS address=127.0.0.1:16904 network=tcp -12:29.590 [INFO] TestAgent-Server: Stopping server: protocol=DNS address=127.0.0.1:16904 network=udp -12:29.591 [INFO] TestAgent-Server: Stopping server: protocol=HTTP address=127.0.0.1:16905 network=tcp -12:30.093 [INFO] TestAgent-Server: Waiting for endpoints to shut down -12:30.094 [INFO] TestAgent-Server: Endpoints down ---- PASS: TestAutoConfig_Integration (5.20s) +2022-12-04T05:48:40.599Z [INFO] agent.auto_config: retrieving initial agent auto configuration remotely +48:40.680 [INFO] test-client.client.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 +2022-12-04T05:48:40.680Z [INFO] agent.router: Initializing LAN area manager +2022-12-04T05:48:40.680Z [INFO] agent.auto_config: auto-config started +48:40.681 [INFO] test-client: Started DNS server: address=127.0.0.1:24058 network=udp +2022-12-04T05:48:40.681Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.681Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.681Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.681Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.681Z [WARN] agent.router.manager: No servers available +48:40.681 [INFO] test-client: Started DNS server: address=127.0.0.1:24058 network=tcp +2022-12-04T05:48:40.681Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.681Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.681Z [WARN] agent.router.manager: No servers available +48:40.682 [INFO] test-client: Started HTTP server: address=127.0.0.1:24059 network=tcp +48:40.682 [INFO] test-client: started state syncer +2022-12-04T05:48:40.682Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.682Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.682Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.682Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.682Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.682Z [WARN] agent.router.manager: No servers available +48:40.682 [INFO] test-client: Started gRPC server: address=127.0.0.1:24064 network=tcp +48:40.682 [INFO] test-client: Retry join is supported for the following discovery methods: cluster=LAN discovery_methods="aliyun aws gce mdns os packet" +48:40.682 [INFO] test-client: Joining cluster...: cluster=LAN +48:40.682 [INFO] test-client: (LAN) joining: lan_addresses=[127.0.0.1:24054] +2022-12-04T05:48:40.682Z [WARN] agent.router.manager: No servers available +48:40.682 [ERROR] test-client.anti_entropy: failed to sync remote state: error="No known Consul servers" +2022-12-04T05:48:40.682Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.682Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.682Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.682Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.682Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.682Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.682Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.683Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.683Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.683Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.683Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.683Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.683Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.683Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.683Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.683Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.683Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.683Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.683Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:40.683Z [WARN] agent.router.manager: No servers available +48:40.684 [DEBUG] test-client.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:24054 +48:40.685 [INFO] test-client.client.serf.lan: serf: EventMemberJoin: Node-b0aee4fd-1c89-0e92-5629-f30e1a75558c 127.0.0.1 +48:40.686 [INFO] test-client: (LAN) joined: number_of_nodes=1 +48:40.686 [DEBUG] test-client: systemd notify failed: error="No socket" +48:40.686 [INFO] test-client: Join cluster completed. Synced with initial agents: cluster=LAN num_agents=1 +48:40.686 [INFO] test-client.client: adding server: server="Node-b0aee4fd-1c89-0e92-5629-f30e1a75558c (Addr: tcp/127.0.0.1:24056) (DC: dc1)" +48:40.732 [DEBUG] test-client.client: transitioned out of legacy ACL mode +48:40.732 [INFO] test-client.client.serf.lan: serf: EventMemberUpdate: test-client +48:40.836 [DEBUG] test-client.client.serf.lan: serf: messageUserEventType: consul:new-leader +48:41.037 [DEBUG] test-client.client.serf.lan: serf: messageUserEventType: consul:new-leader +48:41.037 [DEBUG] test-client.client.serf.lan: serf: messageJoinType: test-client +48:41.235 [DEBUG] test-client.client.serf.lan: serf: messageUserEventType: consul:new-leader +48:41.235 [DEBUG] test-client.client.serf.lan: serf: messageJoinType: test-client +48:41.435 [DEBUG] test-client.client.serf.lan: serf: messageJoinType: test-client +48:41.435 [DEBUG] test-client.client.serf.lan: serf: messageUserEventType: consul:new-leader +48:41.435 [DEBUG] test-client.client.serf.lan: serf: messageJoinType: test-client +48:41.897 [DEBUG] test-client: Skipping remote check since it is managed automatically: check=serfHealth +48:41.897 [INFO] test-client: Synced node info +48:41.897 [DEBUG] test-client: Node info in sync +48:44.327 [INFO] test-client: Requesting shutdown +48:44.328 [INFO] test-client.client: shutting down client +48:44.328 [WARN] test-client.client.serf.lan: serf: Shutdown without a Leave +48:44.332 [INFO] test-client: consul client down +48:44.332 [INFO] test-client: shutdown complete +48:44.332 [INFO] test-client: Stopping server: protocol=DNS address=127.0.0.1:24058 network=tcp +48:44.332 [INFO] test-client: Stopping server: protocol=DNS address=127.0.0.1:24058 network=udp +48:44.332 [INFO] test-client: Stopping server: protocol=HTTP address=127.0.0.1:24059 network=tcp +48:44.332 [ERROR] test-client.client: RPC failed to server: method=ConnectCA.Roots server=127.0.0.1:24056 error="rpc error making call: EOF" +48:44.836 [INFO] test-client: Waiting for endpoints to shut down +48:44.836 [INFO] test-client: Endpoints down +2022-12-04T05:48:40.414Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:40.430Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:40.433 [INFO] TestAgent-Server.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b0aee4fd-1c89-0e92-5629-f30e1a75558c Address:127.0.0.1:24056}]" +48:40.433 [INFO] TestAgent-Server.server.serf.wan: serf: EventMemberJoin: Node-b0aee4fd-1c89-0e92-5629-f30e1a75558c.dc1 127.0.0.1 +48:40.433 [INFO] TestAgent-Server.server.raft: entering follower state: follower="Node at 127.0.0.1:24056 [Follower]" leader= +48:40.434 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberJoin: Node-b0aee4fd-1c89-0e92-5629-f30e1a75558c 127.0.0.1 +2022-12-04T05:48:40.434Z [INFO] agent.router: Initializing LAN area manager +48:40.434 [INFO] TestAgent-Server: Started DNS server: address=127.0.0.1:24051 network=udp +48:40.434 [INFO] TestAgent-Server.server: Adding LAN server: server="Node-b0aee4fd-1c89-0e92-5629-f30e1a75558c (Addr: tcp/127.0.0.1:24056) (DC: dc1)" +48:40.434 [INFO] TestAgent-Server: Started DNS server: address=127.0.0.1:24051 network=tcp +48:40.434 [INFO] TestAgent-Server.server: Handled event for server in area: event=member-join server=Node-b0aee4fd-1c89-0e92-5629-f30e1a75558c.dc1 area=wan +48:40.435 [INFO] TestAgent-Server: Started HTTP server: address=127.0.0.1:24052 network=tcp +48:40.435 [INFO] TestAgent-Server: started state syncer +48:40.435 [INFO] TestAgent-Server: Started gRPC server: address=127.0.0.1:24057 network=tcp +48:40.485 [DEBUG] TestAgent-Server.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:40.496 [WARN] TestAgent-Server.server.raft: heartbeat timeout reached, starting election: last-leader= +48:40.496 [INFO] TestAgent-Server.server.raft: entering candidate state: node="Node at 127.0.0.1:24056 [Candidate]" term=2 +48:40.496 [DEBUG] TestAgent-Server.server.raft: votes: needed=1 +48:40.496 [DEBUG] TestAgent-Server.server.raft: vote granted: from=b0aee4fd-1c89-0e92-5629-f30e1a75558c term=2 tally=1 +48:40.496 [INFO] TestAgent-Server.server.raft: election won: tally=1 +48:40.496 [INFO] TestAgent-Server.server.raft: entering leader state: leader="Node at 127.0.0.1:24056 [Leader]" +48:40.496 [INFO] TestAgent-Server.server: cluster leadership acquired +48:40.496 [INFO] TestAgent-Server.server: initializing acls +48:40.497 [INFO] TestAgent-Server.server: Created ACL 'global-management' policy +48:40.497 [INFO] TestAgent-Server.server: New leader elected: payload=Node-b0aee4fd-1c89-0e92-5629-f30e1a75558c +48:40.497 [INFO] TestAgent-Server.server: Bootstrapped ACL master token from configuration +48:40.498 [INFO] TestAgent-Server.server: Created ACL anonymous token from configuration +48:40.498 [INFO] TestAgent-Server.leader: started routine: routine="legacy ACL token upgrade" +48:40.498 [INFO] TestAgent-Server.leader: started routine: routine="acl token reaping" +48:40.498 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberUpdate: Node-b0aee4fd-1c89-0e92-5629-f30e1a75558c +48:40.498 [INFO] TestAgent-Server.server.serf.wan: serf: EventMemberUpdate: Node-b0aee4fd-1c89-0e92-5629-f30e1a75558c.dc1 +48:40.498 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:40.500 [INFO] TestAgent-Server.server.connect: initialized primary datacenter CA with provider: provider=consul +48:40.500 [INFO] TestAgent-Server.leader: started routine: routine="federation state anti-entropy" +48:40.500 [INFO] TestAgent-Server.leader: started routine: routine="federation state pruning" +48:40.500 [INFO] TestAgent-Server.leader: started routine: routine="intermediate cert renew watch" +48:40.500 [INFO] TestAgent-Server.leader: started routine: routine="CA root pruning" +48:40.500 [INFO] TestAgent-Server.server: Handled event for server in area: event=member-update server=Node-b0aee4fd-1c89-0e92-5629-f30e1a75558c.dc1 area=wan +48:40.500 [INFO] TestAgent-Server.server: Updating LAN server: server="Node-b0aee4fd-1c89-0e92-5629-f30e1a75558c (Addr: tcp/127.0.0.1:24056) (DC: dc1)" +48:40.501 [INFO] TestAgent-Server: Synced node info +48:40.501 [DEBUG] TestAgent-Server: Node info in sync +48:40.501 [INFO] TestAgent-Server.server: federation state anti-entropy synced +48:40.503 [DEBUG] TestAgent-Server.server: Skipping self join check for node since the cluster is too small: node=Node-b0aee4fd-1c89-0e92-5629-f30e1a75558c +48:40.503 [INFO] TestAgent-Server.server: member joined, marking health alive: member=Node-b0aee4fd-1c89-0e92-5629-f30e1a75558c +48:40.503 [DEBUG] TestAgent-Server.server: Skipping self join check for node since the cluster is too small: node=Node-b0aee4fd-1c89-0e92-5629-f30e1a75558c +48:40.556 [DEBUG] TestAgent-Server.acl: dropping node from result due to ACLs: node=Node-b0aee4fd-1c89-0e92-5629-f30e1a75558c +48:40.685 [DEBUG] TestAgent-Server.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:42824 +48:40.685 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 +48:40.685 [INFO] TestAgent-Server.server: member joined, marking health alive: member=test-client +48:40.721 [DEBUG] connect.ca.consul: consul CA provider configured: id=6e:5a:f4:d3:60:ea:bf:d8:5a:9a:b2:a2:65:99:52:1a:bd:c8:2b:60:10:2f:36:d2:12:89:92:3d:aa:a5:93:ce is_primary=true +48:40.723 [INFO] TestAgent-Server.server.connect: CA rotated to new root under provider: provider=consul +48:40.884 [INFO] TestAgent-Server.server.serf.lan: serf: EventMemberUpdate: test-client +48:40.885 [DEBUG] TestAgent-Server.server.serf.lan: serf: messageJoinType: test-client +48:41.084 [DEBUG] TestAgent-Server.server.serf.lan: serf: messageJoinType: test-client +48:41.281 [DEBUG] TestAgent-Server.server.serf.lan: serf: messageJoinType: test-client +48:41.435 [DEBUG] TestAgent-Server.server.serf.lan: serf: messageJoinType: test-client +48:43.002 [DEBUG] TestAgent-Server: Skipping remote check since it is managed automatically: check=serfHealth +48:43.002 [DEBUG] TestAgent-Server: Node info in sync +48:44.836 [INFO] TestAgent-Server: Requesting shutdown +48:44.836 [INFO] TestAgent-Server.server: shutting down server +48:44.836 [DEBUG] TestAgent-Server.leader: stopping routine: routine="CA root pruning" +48:44.836 [DEBUG] TestAgent-Server.leader: stopping routine: routine="legacy ACL token upgrade" +48:44.836 [DEBUG] TestAgent-Server.leader: stopping routine: routine="acl token reaping" +48:44.836 [DEBUG] TestAgent-Server.leader: stopping routine: routine="federation state anti-entropy" +48:44.836 [DEBUG] TestAgent-Server.leader: stopping routine: routine="federation state pruning" +48:44.836 [DEBUG] TestAgent-Server.leader: stopping routine: routine="intermediate cert renew watch" +48:44.836 [WARN] TestAgent-Server.server.serf.lan: serf: Shutdown without a Leave +48:44.836 [INFO] TestAgent-Server.server: shutting down Connect CA roots monitor +48:44.836 [ERROR] TestAgent-Server.server: error performing anti-entropy sync of federation state: error="context canceled" +48:44.836 [DEBUG] TestAgent-Server.leader: stopped routine: routine="federation state anti-entropy" +48:44.836 [WARN] TestAgent-Server.server.serf.wan: serf: Shutdown without a Leave +48:44.836 [DEBUG] TestAgent-Server.leader: stopped routine: routine="federation state pruning" +48:44.836 [DEBUG] TestAgent-Server.leader: stopped routine: routine="CA root pruning" +48:44.836 [DEBUG] TestAgent-Server.leader: stopped routine: routine="legacy ACL token upgrade" +48:44.836 [DEBUG] TestAgent-Server.leader: stopped routine: routine="acl token reaping" +48:44.836 [DEBUG] TestAgent-Server.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:48:44.836Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:44.836Z [INFO] agent.router.manager: shutting down +48:44.837 [INFO] TestAgent-Server: consul server down +48:44.837 [INFO] TestAgent-Server: shutdown complete +48:44.837 [INFO] TestAgent-Server: Stopping server: protocol=DNS address=127.0.0.1:24051 network=tcp +48:44.837 [INFO] TestAgent-Server: Stopping server: protocol=DNS address=127.0.0.1:24051 network=udp +48:44.837 [INFO] TestAgent-Server: Stopping server: protocol=HTTP address=127.0.0.1:24052 network=tcp +48:45.337 [INFO] TestAgent-Server: Waiting for endpoints to shut down +48:45.337 [INFO] TestAgent-Server: Endpoints down +--- PASS: TestAutoConfig_Integration (4.96s) === RUN TestAgent_AutoEncrypt [WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! @@ -13394,226 +13439,231 @@ [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:30.718Z [INFO] agent.auto_config: automatically upgraded to TLS -12:30.720 [INFO] test-client.client.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 -2021-10-31T23:12:30.720Z [INFO] agent.router: Initializing LAN area manager -2021-10-31T23:12:30.720Z [INFO] agent.auto_config: auto-config started -12:30.720 [INFO] test-client: Started DNS server: address=127.0.0.1:16925 network=udp -2021-10-31T23:12:30.720Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.720Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.720Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.720Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.720Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.720Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.720Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.720Z [WARN] agent.router.manager: No servers available -12:30.722 [INFO] test-client: Started DNS server: address=127.0.0.1:16925 network=tcp -2021-10-31T23:12:30.722Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.722Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.722Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.722Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.722Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.722Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.722Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.722Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.722Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.722Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.722Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.722Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.722Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.722Z [WARN] agent.router.manager: No servers available -12:30.722 [INFO] test-client: Started HTTPS server: address=127.0.0.1:16927 network=tcp -12:30.722 [INFO] test-client: started state syncer -2021-10-31T23:12:30.722Z [WARN] agent.router.manager: No servers available -2021-10-31T23:12:30.722Z [WARN] agent.router.manager: No servers available -12:30.722 [INFO] test-client: Started gRPC server: address=127.0.0.1:16931 network=tcp -12:30.722 [INFO] test-client: Retry join is supported for the following discovery methods: cluster=LAN discovery_methods="aliyun aws gce mdns os packet" -12:30.722 [INFO] test-client: Joining cluster...: cluster=LAN -12:30.723 [INFO] test-client: (LAN) joining: lan_addresses=[127.0.0.1:16921] -2021-10-31T23:12:30.723Z [WARN] agent.router.manager: No servers available -12:30.723 [ERROR] test-client.anti_entropy: failed to sync remote state: error="No known Consul servers" -12:30.725 [DEBUG] test-client.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:16921 -12:30.730 [INFO] test-client.client.serf.lan: serf: EventMemberJoin: Node-c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a 127.0.0.1 -12:30.730 [INFO] test-client: (LAN) joined: number_of_nodes=1 -12:30.730 [DEBUG] test-client: systemd notify failed: error="No socket" -12:30.730 [INFO] test-client: Join cluster completed. Synced with initial agents: cluster=LAN num_agents=1 -12:30.730 [INFO] test-client.client: adding server: server="Node-c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a (Addr: tcp/127.0.0.1:16923) (DC: dc1)" -12:30.766 [INFO] test-client: Requesting shutdown -12:30.766 [INFO] test-client.client: shutting down client -12:30.766 [WARN] test-client.client.serf.lan: serf: Shutdown without a Leave -12:30.767 [INFO] test-client: consul client down -12:30.767 [INFO] test-client: shutdown complete -12:30.767 [INFO] test-client: Stopping server: protocol=DNS address=127.0.0.1:16925 network=tcp -12:30.767 [INFO] test-client: Stopping server: protocol=DNS address=127.0.0.1:16925 network=udp -12:30.767 [INFO] test-client: Stopping server: protocol=HTTPS address=127.0.0.1:16927 network=tcp -12:31.269 [INFO] test-client: Waiting for endpoints to shut down -12:31.269 [INFO] test-client: Endpoints down -2021-10-31T23:12:30.142Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:30.203Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:30.210 [INFO] test-server.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a Address:127.0.0.1:16923}]" -12:30.211 [INFO] test-server.server.serf.wan: serf: EventMemberJoin: Node-c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a.dc1 127.0.0.1 -12:30.211 [INFO] test-server.server.serf.lan: serf: EventMemberJoin: Node-c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a 127.0.0.1 -2021-10-31T23:12:30.211Z [INFO] agent.router: Initializing LAN area manager -12:30.211 [INFO] test-server: Started DNS server: address=127.0.0.1:16918 network=udp -12:30.211 [INFO] test-server.server.raft: entering follower state: follower="Node at 127.0.0.1:16923 [Follower]" leader= -12:30.212 [INFO] test-server.server: Adding LAN server: server="Node-c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a (Addr: tcp/127.0.0.1:16923) (DC: dc1)" -12:30.212 [INFO] test-server.server: Handled event for server in area: event=member-join server=Node-c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a.dc1 area=wan -12:30.212 [INFO] test-server: Started DNS server: address=127.0.0.1:16918 network=tcp -12:30.213 [INFO] test-server: Started HTTP server: address=127.0.0.1:16919 network=tcp -12:30.213 [INFO] test-server: started state syncer -12:30.213 [INFO] test-server: Started gRPC server: address=127.0.0.1:16924 network=tcp -12:30.265 [DEBUG] test-server.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -12:30.281 [WARN] test-server.server.raft: heartbeat timeout reached, starting election: last-leader= -12:30.281 [INFO] test-server.server.raft: entering candidate state: node="Node at 127.0.0.1:16923 [Candidate]" term=2 -12:30.282 [DEBUG] test-server.server.raft: votes: needed=1 -12:30.282 [DEBUG] test-server.server.raft: vote granted: from=c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a term=2 tally=1 -12:30.282 [INFO] test-server.server.raft: election won: tally=1 -12:30.282 [INFO] test-server.server.raft: entering leader state: leader="Node at 127.0.0.1:16923 [Leader]" -12:30.282 [INFO] test-server.server: cluster leadership acquired -12:30.282 [INFO] test-server.server: initializing acls -12:30.282 [INFO] test-server.server: Created ACL 'global-management' policy -12:30.283 [INFO] test-server.server: Bootstrapped ACL master token from configuration -12:30.283 [INFO] test-server.server: Created ACL anonymous token from configuration -12:30.283 [INFO] test-server.leader: started routine: routine="legacy ACL token upgrade" -12:30.283 [INFO] test-server.leader: started routine: routine="acl token reaping" -12:30.283 [INFO] test-server.server.serf.lan: serf: EventMemberUpdate: Node-c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a -12:30.283 [INFO] test-server.server.serf.wan: serf: EventMemberUpdate: Node-c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a.dc1 -12:30.297 [INFO] test-server.server: Handled event for server in area: event=member-update server=Node-c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a.dc1 area=wan -12:30.301 [INFO] test-server.server: New leader elected: payload=Node-c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a -12:30.301 [INFO] test-server.server: Updating LAN server: server="Node-c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a (Addr: tcp/127.0.0.1:16923) (DC: dc1)" -12:30.302 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:30.304 [INFO] test-server.server.connect: initialized primary datacenter CA with provider: provider=consul -12:30.304 [INFO] test-server.leader: started routine: routine="federation state anti-entropy" -12:30.304 [INFO] test-server.leader: started routine: routine="federation state pruning" -12:30.304 [INFO] test-server.leader: started routine: routine="intermediate cert renew watch" -12:30.304 [INFO] test-server.leader: started routine: routine="CA root pruning" -12:30.304 [DEBUG] test-server.server: Skipping self join check for node since the cluster is too small: node=Node-c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a -12:30.304 [INFO] test-server.server: member joined, marking health alive: member=Node-c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a -12:30.314 [INFO] test-server.server: federation state anti-entropy synced -12:30.314 [DEBUG] test-server.server: Skipping self join check for node since the cluster is too small: node=Node-c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a -12:30.350 [DEBUG] test-server: Skipping remote check since it is managed automatically: check=serfHealth -12:30.351 [INFO] test-server: Synced node info -12:30.351 [DEBUG] test-server: Node info in sync -12:30.601 [DEBUG] test-server.acl: dropping node from result due to ACLs: node=Node-c8b49dbc-e0b1-135f-4fcd-6b05cb9dc79a -12:30.726 [DEBUG] test-server.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:32828 -12:30.726 [INFO] test-server.server.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 -12:30.726 [INFO] test-server.server: member joined, marking health alive: member=test-client -12:31.269 [INFO] test-server: Requesting shutdown -12:31.270 [INFO] test-server.server: shutting down server -12:31.270 [DEBUG] test-server.leader: stopping routine: routine="intermediate cert renew watch" -12:31.270 [DEBUG] test-server.leader: stopping routine: routine="CA root pruning" -12:31.270 [DEBUG] test-server.leader: stopping routine: routine="legacy ACL token upgrade" -12:31.270 [DEBUG] test-server.leader: stopping routine: routine="acl token reaping" -12:31.270 [DEBUG] test-server.leader: stopping routine: routine="federation state anti-entropy" -12:31.270 [DEBUG] test-server.leader: stopping routine: routine="federation state pruning" -12:31.270 [WARN] test-server.server.serf.lan: serf: Shutdown without a Leave -12:31.270 [ERROR] test-server.server: error performing anti-entropy sync of federation state: error="context canceled" -12:31.270 [DEBUG] test-server.leader: stopped routine: routine="federation state anti-entropy" -12:31.270 [INFO] test-server.server: shutting down Connect CA roots monitor -12:31.270 [DEBUG] test-server.leader: stopped routine: routine="intermediate cert renew watch" -12:31.270 [DEBUG] test-server.leader: stopped routine: routine="CA root pruning" -12:31.270 [DEBUG] test-server.leader: stopped routine: routine="legacy ACL token upgrade" -12:31.270 [DEBUG] test-server.leader: stopped routine: routine="acl token reaping" -12:31.270 [DEBUG] test-server.leader: stopped routine: routine="federation state pruning" -12:31.270 [WARN] test-server.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:31.270Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:31.270Z [INFO] agent.router.manager: shutting down -12:31.270 [INFO] test-server: consul server down -12:31.270 [INFO] test-server: shutdown complete -12:31.270 [INFO] test-server: Stopping server: protocol=DNS address=127.0.0.1:16918 network=tcp -12:31.270 [INFO] test-server: Stopping server: protocol=DNS address=127.0.0.1:16918 network=udp -12:31.270 [INFO] test-server: Stopping server: protocol=HTTP address=127.0.0.1:16919 network=tcp -12:31.717 [DEBUG] test-server.server.memberlist.lan: memberlist: Failed ping: test-client (timeout reached) -12:31.773 [INFO] test-server: Waiting for endpoints to shut down -12:31.773 [INFO] test-server: Endpoints down ---- PASS: TestAgent_AutoEncrypt (1.68s) +2022-12-04T05:48:45.897Z [INFO] agent.auto_config: automatically upgraded to TLS +48:45.898 [INFO] test-client.client.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 +2022-12-04T05:48:45.898Z [INFO] agent.router: Initializing LAN area manager +2022-12-04T05:48:45.898Z [INFO] agent.auto_config: auto-config started +48:45.898 [INFO] test-client: Started DNS server: address=127.0.0.1:24072 network=udp +2022-12-04T05:48:45.899Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.899Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.899Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.899Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.899Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.899Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.899Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.899Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.904Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.904Z [WARN] agent.router.manager: No servers available +48:45.904 [INFO] test-client: Started DNS server: address=127.0.0.1:24072 network=tcp +2022-12-04T05:48:45.904Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.904Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.904Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.904Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.904Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.904Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.904Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.904Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.904Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.904Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.904Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.904Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.904Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.904Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.905Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.905Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.905Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.905Z [WARN] agent.router.manager: No servers available +48:45.905 [INFO] test-client: Started HTTPS server: address=127.0.0.1:24074 network=tcp +48:45.905 [INFO] test-client: started state syncer +2022-12-04T05:48:45.905Z [WARN] agent.router.manager: No servers available +2022-12-04T05:48:45.905Z [WARN] agent.router.manager: No servers available +48:45.905 [INFO] test-client: Started gRPC server: address=127.0.0.1:24078 network=tcp +48:45.905 [INFO] test-client: Retry join is supported for the following discovery methods: cluster=LAN discovery_methods="aliyun aws gce mdns os packet" +48:45.905 [INFO] test-client: Joining cluster...: cluster=LAN +48:45.905 [INFO] test-client: (LAN) joining: lan_addresses=[127.0.0.1:24068] +2022-12-04T05:48:45.905Z [WARN] agent.router.manager: No servers available +48:45.905 [ERROR] test-client.anti_entropy: failed to sync remote state: error="No known Consul servers" +48:45.909 [DEBUG] test-client.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:24068 +48:45.911 [INFO] test-client.client.serf.lan: serf: EventMemberJoin: Node-b6332ccb-6413-cc3b-647b-646840024a20 127.0.0.1 +48:45.911 [INFO] test-client: (LAN) joined: number_of_nodes=1 +48:45.911 [DEBUG] test-client: systemd notify failed: error="No socket" +48:45.911 [INFO] test-client: Join cluster completed. Synced with initial agents: cluster=LAN num_agents=1 +48:45.911 [INFO] test-client.client: adding server: server="Node-b6332ccb-6413-cc3b-647b-646840024a20 (Addr: tcp/127.0.0.1:24070) (DC: dc1)" +48:45.952 [DEBUG] test-client.client: transitioned out of legacy ACL mode +48:45.952 [INFO] test-client.client.serf.lan: serf: EventMemberUpdate: test-client +48:45.952 [INFO] test-client: Requesting shutdown +48:45.952 [INFO] test-client.client: shutting down client +48:45.952 [WARN] test-client.client.serf.lan: serf: Shutdown without a Leave +48:45.953 [INFO] test-client: consul client down +48:45.953 [INFO] test-client: shutdown complete +48:45.953 [INFO] test-client: Stopping server: protocol=DNS address=127.0.0.1:24072 network=tcp +48:45.953 [INFO] test-client: Stopping server: protocol=DNS address=127.0.0.1:24072 network=udp +48:45.953 [INFO] test-client: Stopping server: protocol=HTTPS address=127.0.0.1:24074 network=tcp +48:46.456 [INFO] test-client: Waiting for endpoints to shut down +48:46.456 [INFO] test-client: Endpoints down +2022-12-04T05:48:45.355Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:45.375Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:45.378 [INFO] test-server.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b6332ccb-6413-cc3b-647b-646840024a20 Address:127.0.0.1:24070}]" +48:45.384 [INFO] test-server.server.serf.wan: serf: EventMemberJoin: Node-b6332ccb-6413-cc3b-647b-646840024a20.dc1 127.0.0.1 +48:45.385 [INFO] test-server.server.raft: entering follower state: follower="Node at 127.0.0.1:24070 [Follower]" leader= +48:45.393 [INFO] test-server.server.serf.lan: serf: EventMemberJoin: Node-b6332ccb-6413-cc3b-647b-646840024a20 127.0.0.1 +2022-12-04T05:48:45.393Z [INFO] agent.router: Initializing LAN area manager +48:45.393 [INFO] test-server.server: Handled event for server in area: event=member-join server=Node-b6332ccb-6413-cc3b-647b-646840024a20.dc1 area=wan +48:45.394 [INFO] test-server.server: Adding LAN server: server="Node-b6332ccb-6413-cc3b-647b-646840024a20 (Addr: tcp/127.0.0.1:24070) (DC: dc1)" +48:45.394 [INFO] test-server: Started DNS server: address=127.0.0.1:24065 network=udp +48:45.394 [INFO] test-server: Started DNS server: address=127.0.0.1:24065 network=tcp +48:45.396 [INFO] test-server: Started HTTP server: address=127.0.0.1:24066 network=tcp +48:45.398 [INFO] test-server: started state syncer +48:45.398 [INFO] test-server: Started gRPC server: address=127.0.0.1:24071 network=tcp +48:45.439 [WARN] test-server.server.raft: heartbeat timeout reached, starting election: last-leader= +48:45.439 [INFO] test-server.server.raft: entering candidate state: node="Node at 127.0.0.1:24070 [Candidate]" term=2 +48:45.439 [DEBUG] test-server.server.raft: votes: needed=1 +48:45.439 [DEBUG] test-server.server.raft: vote granted: from=b6332ccb-6413-cc3b-647b-646840024a20 term=2 tally=1 +48:45.439 [INFO] test-server.server.raft: election won: tally=1 +48:45.439 [INFO] test-server.server.raft: entering leader state: leader="Node at 127.0.0.1:24070 [Leader]" +48:45.439 [INFO] test-server.server: cluster leadership acquired +48:45.439 [INFO] test-server.server: New leader elected: payload=Node-b6332ccb-6413-cc3b-647b-646840024a20 +48:45.439 [INFO] test-server.server: initializing acls +48:45.439 [INFO] test-server.server: Created ACL 'global-management' policy +48:45.440 [INFO] test-server.server: Bootstrapped ACL master token from configuration +48:45.440 [INFO] test-server.server: Created ACL anonymous token from configuration +48:45.440 [INFO] test-server.leader: started routine: routine="legacy ACL token upgrade" +48:45.440 [INFO] test-server.leader: started routine: routine="acl token reaping" +48:45.440 [INFO] test-server.server.serf.lan: serf: EventMemberUpdate: Node-b6332ccb-6413-cc3b-647b-646840024a20 +48:45.441 [INFO] test-server.server.serf.wan: serf: EventMemberUpdate: Node-b6332ccb-6413-cc3b-647b-646840024a20.dc1 +48:45.441 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:45.441 [INFO] test-server.server: Updating LAN server: server="Node-b6332ccb-6413-cc3b-647b-646840024a20 (Addr: tcp/127.0.0.1:24070) (DC: dc1)" +48:45.441 [INFO] test-server.server: Handled event for server in area: event=member-update server=Node-b6332ccb-6413-cc3b-647b-646840024a20.dc1 area=wan +48:45.443 [INFO] test-server.server.connect: initialized primary datacenter CA with provider: provider=consul +48:45.443 [INFO] test-server.leader: started routine: routine="federation state anti-entropy" +48:45.443 [INFO] test-server.leader: started routine: routine="federation state pruning" +48:45.443 [INFO] test-server.leader: started routine: routine="intermediate cert renew watch" +48:45.443 [INFO] test-server.leader: started routine: routine="CA root pruning" +48:45.443 [DEBUG] test-server.server: Skipping self join check for node since the cluster is too small: node=Node-b6332ccb-6413-cc3b-647b-646840024a20 +48:45.443 [INFO] test-server.server: member joined, marking health alive: member=Node-b6332ccb-6413-cc3b-647b-646840024a20 +48:45.443 [INFO] test-server.server: federation state anti-entropy synced +48:45.444 [DEBUG] test-server.server: Skipping self join check for node since the cluster is too small: node=Node-b6332ccb-6413-cc3b-647b-646840024a20 +48:45.701 [DEBUG] test-server: Skipping remote check since it is managed automatically: check=serfHealth +48:45.701 [INFO] test-server: Synced node info +48:45.820 [DEBUG] test-server.acl: dropping node from result due to ACLs: node=Node-b6332ccb-6413-cc3b-647b-646840024a20 +48:45.909 [DEBUG] test-server.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:54544 +48:45.910 [INFO] test-server.server.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 +48:45.910 [INFO] test-server.server: member joined, marking health alive: member=test-client +48:46.456 [INFO] test-server: Requesting shutdown +48:46.456 [INFO] test-server.server: shutting down server +48:46.456 [DEBUG] test-server.leader: stopping routine: routine="legacy ACL token upgrade" +48:46.456 [DEBUG] test-server.leader: stopping routine: routine="acl token reaping" +48:46.456 [DEBUG] test-server.leader: stopping routine: routine="federation state anti-entropy" +48:46.456 [DEBUG] test-server.leader: stopping routine: routine="federation state pruning" +48:46.456 [DEBUG] test-server.leader: stopping routine: routine="intermediate cert renew watch" +48:46.456 [DEBUG] test-server.leader: stopping routine: routine="CA root pruning" +48:46.456 [WARN] test-server.server.serf.lan: serf: Shutdown without a Leave +48:46.456 [ERROR] test-server.server: error performing anti-entropy sync of federation state: error="context canceled" +48:46.456 [DEBUG] test-server.leader: stopped routine: routine="federation state anti-entropy" +48:46.456 [INFO] test-server.server: shutting down Connect CA roots monitor +48:46.456 [DEBUG] test-server.leader: stopped routine: routine="legacy ACL token upgrade" +48:46.456 [DEBUG] test-server.leader: stopped routine: routine="acl token reaping" +48:46.456 [DEBUG] test-server.leader: stopped routine: routine="federation state pruning" +48:46.456 [DEBUG] test-server.leader: stopped routine: routine="intermediate cert renew watch" +48:46.456 [DEBUG] test-server.leader: stopped routine: routine="CA root pruning" +48:46.456 [WARN] test-server.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:46.456Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:46.456Z [INFO] agent.router.manager: shutting down +48:46.457 [INFO] test-server: consul server down +48:46.457 [INFO] test-server: shutdown complete +48:46.457 [INFO] test-server: Stopping server: protocol=DNS address=127.0.0.1:24065 network=tcp +48:46.457 [INFO] test-server: Stopping server: protocol=DNS address=127.0.0.1:24065 network=udp +48:46.457 [INFO] test-server: Stopping server: protocol=HTTP address=127.0.0.1:24066 network=tcp +48:46.897 [DEBUG] test-server.server.memberlist.lan: memberlist: Failed ping: test-client (timeout reached) +48:46.960 [INFO] test-server: Waiting for endpoints to shut down +48:46.960 [INFO] test-server: Endpoints down +--- PASS: TestAgent_AutoEncrypt (1.62s) === RUN TestSharedRPCRouter [WARN] freeport: 6 out of 20 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 6 out of 6 pending ports are still in use; something probably didn't wait around for the port to be closed! -12:32.087 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-6ca5f779-ae04-f672-8c04-0bbced4a9b12 127.0.0.1 -2021-10-31T23:12:32.087Z [INFO] agent.router: Initializing LAN area manager -12:32.088 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16939 network=udp -12:32.088 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16939 network=tcp -12:32.088 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16940 network=tcp -12:32.088 [INFO] TestAgent: started state syncer -2021-10-31T23:12:32.089Z [WARN] agent.router.manager: No servers available -12:32.089 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16945 network=tcp -12:32.089 [INFO] TestAgent: Retry join is supported for the following discovery methods: cluster=LAN discovery_methods="aliyun aws gce mdns os packet" -12:32.089 [INFO] TestAgent: Joining cluster...: cluster=LAN -12:32.089 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:16935] -2021-10-31T23:12:32.089Z [WARN] agent.router.manager: No servers available -12:32.089 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -12:32.089 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:16935 -12:32.094 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-b79b3f05-d2c5-3b14-dfd2-da2f9d949f05 127.0.0.1 -12:32.094 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -12:32.094 [DEBUG] TestAgent: systemd notify failed: error="No socket" -12:32.094 [INFO] TestAgent: Join cluster completed. Synced with initial agents: cluster=LAN num_agents=1 -12:32.094 [INFO] TestAgent.client: adding server: server="Node-b79b3f05-d2c5-3b14-dfd2-da2f9d949f05 (Addr: tcp/127.0.0.1:16937) (DC: dc1)" -12:32.130 [ERROR] TestAgent.client: RPC failed to server: method=Health.NodeChecks server=127.0.0.1:16937 error="rpc error making call: EOF" -12:32.130 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="rpc error making call: EOF" -2021-10-31T23:12:31.806Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:31.840Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:31.852 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b79b3f05-d2c5-3b14-dfd2-da2f9d949f05 Address:127.0.0.1:16937}]" -12:31.852 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16937 [Follower]" leader= -12:31.852 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b79b3f05-d2c5-3b14-dfd2-da2f9d949f05.dc1 127.0.0.1 -12:31.853 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b79b3f05-d2c5-3b14-dfd2-da2f9d949f05 127.0.0.1 -2021-10-31T23:12:31.853Z [INFO] agent.router: Initializing LAN area manager -12:31.853 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16932 network=udp -12:31.853 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b79b3f05-d2c5-3b14-dfd2-da2f9d949f05.dc1 area=wan -12:31.854 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16932 network=tcp -12:31.854 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16933 network=tcp -12:31.854 [INFO] TestAgent: started state syncer -12:31.854 [INFO] TestAgent.server: Adding LAN server: server="Node-b79b3f05-d2c5-3b14-dfd2-da2f9d949f05 (Addr: tcp/127.0.0.1:16937) (DC: dc1)" -12:31.854 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16938 network=tcp -12:31.901 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:31.901 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16937 [Candidate]" term=2 -12:31.902 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:31.902 [DEBUG] TestAgent.server.raft: vote granted: from=b79b3f05-d2c5-3b14-dfd2-da2f9d949f05 term=2 tally=1 -12:31.902 [INFO] TestAgent.server.raft: election won: tally=1 -12:31.902 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16937 [Leader]" -12:31.902 [INFO] TestAgent.server: cluster leadership acquired -12:31.902 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16937 -12:31.903 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:31.903 [INFO] TestAgent.server: New leader elected: payload=Node-b79b3f05-d2c5-3b14-dfd2-da2f9d949f05 -12:31.904 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:31.904 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:31.904 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:31.904 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:31.904 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:31.904 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b79b3f05-d2c5-3b14-dfd2-da2f9d949f05 -12:31.904 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b79b3f05-d2c5-3b14-dfd2-da2f9d949f05 -12:31.905 [INFO] TestAgent.server: federation state anti-entropy synced -12:32.018 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:32.018 [INFO] TestAgent: Synced node info -12:32.090 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:46534 -12:32.090 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6ca5f779-ae04-f672-8c04-0bbced4a9b12 127.0.0.1 -12:32.090 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6ca5f779-ae04-f672-8c04-0bbced4a9b12 -12:32.119 [INFO] TestAgent: Requesting shutdown -12:32.119 [INFO] TestAgent.server: shutting down server -12:32.119 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:32.119 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:32.119 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:32.119 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:32.119 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:32.119 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:32.119 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:32.119 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:32.119 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:32.119 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:32.120 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:32.120Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:32.120Z [INFO] agent.router.manager: shutting down -12:32.120 [INFO] TestAgent: consul server down -12:32.120 [INFO] TestAgent: shutdown complete -12:32.120 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16932 network=tcp -12:32.120 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16932 network=udp -12:32.120 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16933 network=tcp -12:32.625 [INFO] TestAgent: Waiting for endpoints to shut down -12:32.626 [INFO] TestAgent: Endpoints down ---- PASS: TestSharedRPCRouter (0.85s) +48:47.298 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-da89a363-5144-9ce3-8ee6-0377b0ba797a 127.0.0.1 +2022-12-04T05:48:47.298Z [INFO] agent.router: Initializing LAN area manager +48:47.298 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24086 network=udp +48:47.299 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24086 network=tcp +48:47.299 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24087 network=tcp +48:47.299 [INFO] TestAgent: started state syncer +2022-12-04T05:48:47.300Z [WARN] agent.router.manager: No servers available +48:47.300 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +48:47.311 [INFO] TestAgent: Retry join is supported for the following discovery methods: cluster=LAN discovery_methods="aliyun aws gce mdns os packet" +48:47.311 [INFO] TestAgent: Joining cluster...: cluster=LAN +48:47.311 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:24082] +48:47.312 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24092 network=tcp +2022-12-04T05:48:47.316Z [WARN] agent.router.manager: No servers available +48:47.316 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:24082 +48:47.323 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-4637b6f9-a882-5586-2411-fecaf25cbcdd 127.0.0.1 +48:47.323 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +48:47.323 [DEBUG] TestAgent: systemd notify failed: error="No socket" +48:47.323 [INFO] TestAgent: Join cluster completed. Synced with initial agents: cluster=LAN num_agents=1 +48:47.323 [INFO] TestAgent.client: adding server: server="Node-4637b6f9-a882-5586-2411-fecaf25cbcdd (Addr: tcp/127.0.0.1:24084) (DC: dc1)" +48:47.433 [ERROR] TestAgent.client: RPC failed to server: method=Health.NodeChecks server=127.0.0.1:24084 error="rpc error making call: EOF" +48:47.433 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="rpc error making call: EOF" +2022-12-04T05:48:46.991Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:47.016Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:47.025 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4637b6f9-a882-5586-2411-fecaf25cbcdd Address:127.0.0.1:24084}]" +48:47.026 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4637b6f9-a882-5586-2411-fecaf25cbcdd.dc1 127.0.0.1 +48:47.027 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4637b6f9-a882-5586-2411-fecaf25cbcdd 127.0.0.1 +2022-12-04T05:48:47.027Z [INFO] agent.router: Initializing LAN area manager +48:47.027 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24079 network=udp +48:47.027 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24084 [Follower]" leader= +48:47.027 [INFO] TestAgent.server: Adding LAN server: server="Node-4637b6f9-a882-5586-2411-fecaf25cbcdd (Addr: tcp/127.0.0.1:24084) (DC: dc1)" +48:47.027 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4637b6f9-a882-5586-2411-fecaf25cbcdd.dc1 area=wan +48:47.027 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24079 network=tcp +48:47.028 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24080 network=tcp +48:47.028 [INFO] TestAgent: started state syncer +48:47.028 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24085 network=tcp +48:47.088 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:47.088 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24084 [Candidate]" term=2 +48:47.088 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:47.088 [DEBUG] TestAgent.server.raft: vote granted: from=4637b6f9-a882-5586-2411-fecaf25cbcdd term=2 tally=1 +48:47.088 [INFO] TestAgent.server.raft: election won: tally=1 +48:47.088 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24084 [Leader]" +48:47.089 [INFO] TestAgent.server: cluster leadership acquired +48:47.089 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24084 +48:47.089 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:47.091 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:47.091 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:47.091 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:47.091 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:47.091 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:47.091 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4637b6f9-a882-5586-2411-fecaf25cbcdd +48:47.091 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4637b6f9-a882-5586-2411-fecaf25cbcdd +48:47.092 [INFO] TestAgent.server: New leader elected: payload=Node-4637b6f9-a882-5586-2411-fecaf25cbcdd +48:47.093 [INFO] TestAgent.server: federation state anti-entropy synced +48:47.230 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:47.231 [INFO] TestAgent: Synced node info +48:47.312 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:47352 +48:47.322 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-da89a363-5144-9ce3-8ee6-0377b0ba797a 127.0.0.1 +48:47.322 [INFO] TestAgent.server: member joined, marking health alive: member=Node-da89a363-5144-9ce3-8ee6-0377b0ba797a +48:47.343 [INFO] TestAgent: Requesting shutdown +48:47.343 [INFO] TestAgent.server: shutting down server +48:47.343 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:47.343 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:47.343 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:47.343 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:47.343 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:47.343 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:47.343 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:47.343 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:47.343 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:47.343 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:47.343 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:47.343Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:47.343Z [INFO] agent.router.manager: shutting down +48:47.343 [INFO] TestAgent: consul server down +48:47.343 [INFO] TestAgent: shutdown complete +48:47.343 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24079 network=tcp +48:47.343 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24079 network=udp +48:47.343 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24080 network=tcp +48:47.844 [INFO] TestAgent: Waiting for endpoints to shut down +48:47.844 [INFO] TestAgent: Endpoints down +--- PASS: TestSharedRPCRouter (0.88s) === RUN TestCatalogRegister_Service_InvalidAddress === PAUSE TestCatalogRegister_Service_InvalidAddress === RUN TestCatalogDeregister @@ -13700,171 +13750,180 @@ [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:33.063Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:33.109Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:33.115 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c9442937-e4e9-8e08-5b64-42a2a18007c3 Address:127.0.0.1:16958}]" -12:33.116 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c9442937-e4e9-8e08-5b64-42a2a18007c3.dc2 127.0.0.1 -12:33.116 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c9442937-e4e9-8e08-5b64-42a2a18007c3 127.0.0.1 -2021-10-31T23:12:33.116Z [INFO] agent.router: Initializing LAN area manager -12:33.117 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16953 network=udp -12:33.117 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16958 [Follower]" leader= -12:33.117 [INFO] TestAgent.server: Adding LAN server: server="Node-c9442937-e4e9-8e08-5b64-42a2a18007c3 (Addr: tcp/127.0.0.1:16958) (DC: dc2)" -12:33.117 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c9442937-e4e9-8e08-5b64-42a2a18007c3.dc2 area=wan -12:33.117 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16953 network=tcp -12:33.130 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16954 network=tcp -12:33.130 [INFO] TestAgent: started state syncer -12:33.130 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16959 network=tcp -12:33.130 [INFO] TestAgent: Retry join is supported for the following discovery methods: cluster=WAN discovery_methods="aliyun aws gce mdns os packet" -12:33.130 [INFO] TestAgent: Joining cluster...: cluster=WAN -12:33.130 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:16950] -12:33.134 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:16950 -12:33.136 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dd1923bc-52b1-2627-3ee5-0307cf7d17c5.dc1 127.0.0.1 -12:33.136 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -12:33.136 [INFO] TestAgent: Join cluster completed. Synced with initial agents: cluster=WAN num_agents=1 -12:33.136 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dd1923bc-52b1-2627-3ee5-0307cf7d17c5.dc1 area=wan -12:33.180 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:33.180 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16958 [Candidate]" term=2 -12:33.180 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:33.180 [DEBUG] TestAgent.server.raft: vote granted: from=c9442937-e4e9-8e08-5b64-42a2a18007c3 term=2 tally=1 -12:33.180 [INFO] TestAgent.server.raft: election won: tally=1 -12:33.180 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16958 [Leader]" -12:33.180 [INFO] TestAgent.server: cluster leadership acquired -12:33.181 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=0 found=true -12:33.181 [INFO] TestAgent.server: New leader elected: payload=Node-c9442937-e4e9-8e08-5b64-42a2a18007c3 -12:33.187 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:4a:c6:ab:ef:63:c9:60:1a:51:7f:19:62:e3:e9:d9:0e:76:55:10:6e:74:24:69:28:a1:6c:b8:b9:8f:fd:89 is_primary=false -12:33.194 [INFO] TestAgent.server.connect: received new intermediate certificate from primary datacenter -12:33.194 [INFO] TestAgent.server.connect: updated root certificates from primary datacenter -12:33.194 [INFO] TestAgent.server.connect: initialized secondary datacenter CA with provider: provider=consul -12:33.194 [INFO] TestAgent.leader: started routine: routine="config entry replication" -12:33.194 [INFO] TestAgent.leader: started routine: routine="federation state replication" -12:33.194 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:33.194 [INFO] TestAgent.leader: started routine: routine="secondary CA roots watch" -12:33.194 [INFO] TestAgent.leader: started routine: routine="intention replication" -12:33.194 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:33.194 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:33.194 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c9442937-e4e9-8e08-5b64-42a2a18007c3 -12:33.194 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c9442937-e4e9-8e08-5b64-42a2a18007c3 -12:33.195 [DEBUG] TestAgent.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 -12:33.195 [DEBUG] TestAgent.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 -12:33.196 [DEBUG] TestAgent.server: finished fetching remote objects: amount=1 -12:33.196 [DEBUG] TestAgent.server: diffing replication state: local_amount=0 remote_amount=1 -12:33.196 [DEBUG] TestAgent.server: diffed replication state: deletions=0 updates=1 -12:33.196 [DEBUG] TestAgent.server: performing updates: updates=1 -12:33.196 [DEBUG] TestAgent.server: finished updates -12:33.196 [DEBUG] TestAgent.replication.federation_state: replication completed through remote index: index=11 -12:33.196 [DEBUG] TestAgent.server.replication.config_entry: finished fetching config entries: amount=0 -12:33.196 [DEBUG] TestAgent.server.replication.config_entry: Config Entry replication: local=0 remote=0 -12:33.196 [DEBUG] TestAgent.server.replication.config_entry: Config Entry replication: deletions=0 updates=0 -12:33.196 [DEBUG] TestAgent.server.replication.config_entry: replication completed through remote index: index=1 -12:33.196 [DEBUG] TestAgent.server: finished fetching remote objects: amount=2 -12:33.196 [DEBUG] TestAgent.server: diffing replication state: local_amount=1 remote_amount=2 -12:33.196 [DEBUG] TestAgent.server: diffed replication state: deletions=0 updates=1 -12:33.196 [DEBUG] TestAgent.server: performing updates: updates=1 -12:33.196 [DEBUG] TestAgent.server: finished updates -12:33.196 [DEBUG] TestAgent.replication.federation_state: replication completed through remote index: index=14 -12:33.197 [INFO] TestAgent.server: federation state anti-entropy synced -12:33.197 [INFO] TestAgent.server.connect: updated root certificates from primary datacenter -12:33.256 [INFO] TestAgent: Requesting shutdown -12:33.256 [INFO] TestAgent.server: shutting down server -12:33.256 [DEBUG] TestAgent.leader: stopping routine: routine="config entry replication" -12:33.256 [DEBUG] TestAgent.leader: stopping routine: routine="federation state replication" -12:33.256 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:33.256 [DEBUG] TestAgent.leader: stopping routine: routine="secondary CA roots watch" -12:33.256 [DEBUG] TestAgent.leader: stopping routine: routine="intention replication" -12:33.256 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:33.256 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:33.256 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:33.256 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:33.256 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:33.256 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:33.256 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:33.257 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:33.257 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:33.257Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:33.257Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:33.257Z [INFO] agent.router.manager: shutting down -12:33.257 [INFO] TestAgent: consul server down -12:33.257 [INFO] TestAgent: shutdown complete -12:33.257 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16953 network=tcp -12:33.257 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16953 network=udp -12:33.257 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16954 network=tcp -12:33.257 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:16951 datacenter=dc1 method=ConfigEntry.ListAll error="rpc error making call: EOF" -12:33.257 [INFO] TestAgent.server.replication.config_entry: stopped replication -12:33.257 [DEBUG] TestAgent.leader: stopped routine: routine="config entry replication" -12:33.257 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:16951 datacenter=dc1 method=FederationState.List error="rpc error making call: EOF" -12:33.257 [WARN] TestAgent.replication.federation_state: replication error (will retry if still leader): error="failed to retrieve federation states: rpc error making call: EOF" -12:33.257 [INFO] TestAgent.replication.federation_state: stopped replication -12:33.257 [DEBUG] TestAgent.leader: stopped routine: routine="federation state replication" -12:33.257 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:16951 datacenter=dc1 method=ConnectCA.Roots error="rpc error making call: EOF" -12:33.257 [ERROR] TestAgent.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: rpc error making call: EOF" -12:33.257 [DEBUG] TestAgent.leader: stopped routine: routine="secondary CA roots watch" -12:33.257 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:16951 datacenter=dc1 method=Intention.List error="rpc error making call: EOF" -12:33.257 [ERROR] TestAgent.server.connect: error replicating intentions: routine="intention replication" error="rpc error making call: EOF" -12:33.257 [DEBUG] TestAgent.leader: stopped routine: routine="intention replication" -12:33.757 [INFO] TestAgent: Waiting for endpoints to shut down -12:33.757 [INFO] TestAgent: Endpoints down -2021-10-31T23:12:32.669Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:32.719Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:32.730 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dd1923bc-52b1-2627-3ee5-0307cf7d17c5 Address:127.0.0.1:16951}]" -12:32.730 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dd1923bc-52b1-2627-3ee5-0307cf7d17c5.dc1 127.0.0.1 -12:32.731 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dd1923bc-52b1-2627-3ee5-0307cf7d17c5 127.0.0.1 -2021-10-31T23:12:32.731Z [INFO] agent.router: Initializing LAN area manager -12:32.731 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16946 network=udp -12:32.731 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16951 [Follower]" leader= -12:32.732 [INFO] TestAgent.server: Adding LAN server: server="Node-dd1923bc-52b1-2627-3ee5-0307cf7d17c5 (Addr: tcp/127.0.0.1:16951) (DC: dc1)" -12:32.732 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dd1923bc-52b1-2627-3ee5-0307cf7d17c5.dc1 area=wan -12:32.732 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16946 network=tcp -12:32.732 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16947 network=tcp -12:32.732 [INFO] TestAgent: started state syncer -12:32.733 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16952 network=tcp -12:32.789 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:32.789 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16951 [Candidate]" term=2 -12:32.790 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:32.790 [DEBUG] TestAgent.server.raft: vote granted: from=dd1923bc-52b1-2627-3ee5-0307cf7d17c5 term=2 tally=1 -12:32.790 [INFO] TestAgent.server.raft: election won: tally=1 -12:32.790 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16951 [Leader]" -12:32.790 [INFO] TestAgent.server: cluster leadership acquired -12:32.790 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16951 -12:32.791 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:32.792 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:32.792 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:32.792 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:32.792 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:32.792 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:32.792 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dd1923bc-52b1-2627-3ee5-0307cf7d17c5 -12:32.792 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dd1923bc-52b1-2627-3ee5-0307cf7d17c5 -12:32.793 [INFO] TestAgent.server: New leader elected: payload=Node-dd1923bc-52b1-2627-3ee5-0307cf7d17c5 -12:32.804 [INFO] TestAgent.server: federation state anti-entropy synced -12:33.002 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:33.002 [INFO] TestAgent: Synced node info -12:33.002 [DEBUG] TestAgent: Node info in sync -12:33.134 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:45334 -12:33.135 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c9442937-e4e9-8e08-5b64-42a2a18007c3.dc2 127.0.0.1 -12:33.135 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c9442937-e4e9-8e08-5b64-42a2a18007c3.dc2 area=wan -12:33.757 [INFO] TestAgent: Requesting shutdown -12:33.757 [INFO] TestAgent.server: shutting down server -12:33.758 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:33.758 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:33.758 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:33.758 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:33.758 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:33.758 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:33.758 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:33.758 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:33.758 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:33.758 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:33.758 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:33.758Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:33.758Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:33.758Z [INFO] agent.router.manager: shutting down -12:33.758 [INFO] TestAgent: consul server down -12:33.758 [INFO] TestAgent: shutdown complete -12:33.758 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16946 network=tcp -12:33.758 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16946 network=udp -12:33.758 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16947 network=tcp -12:34.260 [INFO] TestAgent: Waiting for endpoints to shut down -12:34.260 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectCARoots_PEMEncoding (1.63s) +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:48:48.204Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:48.233Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:48.235 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d755788a-b699-8ff8-4db8-572b4cb4c7f0 Address:127.0.0.1:24105}]" +48:48.235 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d755788a-b699-8ff8-4db8-572b4cb4c7f0.dc2 127.0.0.1 +48:48.235 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24105 [Follower]" leader= +48:48.236 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d755788a-b699-8ff8-4db8-572b4cb4c7f0 127.0.0.1 +2022-12-04T05:48:48.236Z [INFO] agent.router: Initializing LAN area manager +48:48.236 [INFO] TestAgent.server: Adding LAN server: server="Node-d755788a-b699-8ff8-4db8-572b4cb4c7f0 (Addr: tcp/127.0.0.1:24105) (DC: dc2)" +48:48.236 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d755788a-b699-8ff8-4db8-572b4cb4c7f0.dc2 area=wan +48:48.236 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24100 network=udp +48:48.236 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24100 network=tcp +48:48.237 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24101 network=tcp +48:48.237 [INFO] TestAgent: started state syncer +48:48.237 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24106 network=tcp +48:48.237 [INFO] TestAgent: Retry join is supported for the following discovery methods: cluster=WAN discovery_methods="aliyun aws gce mdns os packet" +48:48.237 [INFO] TestAgent: Joining cluster...: cluster=WAN +48:48.237 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:24097] +48:48.238 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:24097 +48:48.239 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3f96d635-d9e3-3605-7d64-c01af93930b4.dc1 127.0.0.1 +48:48.239 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +48:48.239 [INFO] TestAgent: Join cluster completed. Synced with initial agents: cluster=WAN num_agents=1 +48:48.239 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3f96d635-d9e3-3605-7d64-c01af93930b4.dc1 area=wan +48:48.287 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:48.287 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24105 [Candidate]" term=2 +48:48.287 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:48.287 [DEBUG] TestAgent.server.raft: vote granted: from=d755788a-b699-8ff8-4db8-572b4cb4c7f0 term=2 tally=1 +48:48.287 [INFO] TestAgent.server.raft: election won: tally=1 +48:48.287 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24105 [Leader]" +48:48.287 [INFO] TestAgent.server: cluster leadership acquired +48:48.287 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=0 found=true +48:48.288 [INFO] TestAgent.server: New leader elected: payload=Node-d755788a-b699-8ff8-4db8-572b4cb4c7f0 +48:48.289 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:4a:c6:ab:ef:63:c9:60:1a:51:7f:19:62:e3:e9:d9:0e:76:55:10:6e:74:24:69:28:a1:6c:b8:b9:8f:fd:89 is_primary=false +48:48.297 [INFO] TestAgent.server.connect: received new intermediate certificate from primary datacenter +48:48.298 [INFO] TestAgent.server.connect: updated root certificates from primary datacenter +48:48.298 [INFO] TestAgent.server.connect: initialized secondary datacenter CA with provider: provider=consul +48:48.298 [INFO] TestAgent.leader: started routine: routine="config entry replication" +48:48.298 [INFO] TestAgent.leader: started routine: routine="federation state replication" +48:48.298 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:48.298 [INFO] TestAgent.leader: started routine: routine="secondary CA roots watch" +48:48.298 [INFO] TestAgent.leader: started routine: routine="intention replication" +48:48.298 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:48.298 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:48.298 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d755788a-b699-8ff8-4db8-572b4cb4c7f0 +48:48.298 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d755788a-b699-8ff8-4db8-572b4cb4c7f0 +48:48.298 [DEBUG] TestAgent.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 +48:48.298 [DEBUG] TestAgent.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 +48:48.304 [INFO] TestAgent.server: federation state anti-entropy synced +48:48.305 [INFO] TestAgent.server.connect: updated root certificates from primary datacenter +48:48.305 [DEBUG] TestAgent.server.replication.config_entry: finished fetching config entries: amount=0 +48:48.305 [DEBUG] TestAgent.server.replication.config_entry: Config Entry replication: local=0 remote=0 +48:48.305 [DEBUG] TestAgent.server.replication.config_entry: Config Entry replication: deletions=0 updates=0 +48:48.305 [DEBUG] TestAgent.server.replication.config_entry: replication completed through remote index: index=1 +48:48.305 [DEBUG] TestAgent.server: finished fetching remote objects: amount=1 +48:48.305 [DEBUG] TestAgent.server: diffing replication state: local_amount=0 remote_amount=1 +48:48.305 [DEBUG] TestAgent.server: diffed replication state: deletions=0 updates=1 +48:48.305 [DEBUG] TestAgent.server: performing updates: updates=1 +48:48.305 [DEBUG] TestAgent.server: finished updates +48:48.305 [DEBUG] TestAgent.replication.federation_state: replication completed through remote index: index=11 +48:48.305 [DEBUG] TestAgent.server: finished fetching remote objects: amount=2 +48:48.305 [DEBUG] TestAgent.server: diffing replication state: local_amount=1 remote_amount=2 +48:48.305 [DEBUG] TestAgent.server: diffed replication state: deletions=0 updates=1 +48:48.305 [DEBUG] TestAgent.server: performing updates: updates=1 +48:48.306 [DEBUG] TestAgent.server: finished updates +48:48.306 [DEBUG] TestAgent.replication.federation_state: replication completed through remote index: index=13 +48:48.306 [INFO] TestAgent.server: federation state anti-entropy synced +48:48.313 [DEBUG] TestAgent.server: finished fetching remote objects: amount=2 +48:48.313 [DEBUG] TestAgent.server: diffing replication state: local_amount=2 remote_amount=2 +48:48.313 [DEBUG] TestAgent.server: diffed replication state: deletions=0 updates=1 +48:48.313 [DEBUG] TestAgent.server: performing updates: updates=1 +48:48.313 [DEBUG] TestAgent.server: finished updates +48:48.313 [DEBUG] TestAgent.replication.federation_state: replication completed through remote index: index=15 +48:48.473 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:48.474 [INFO] TestAgent: Synced node info +48:48.651 [INFO] TestAgent: Requesting shutdown +48:48.651 [INFO] TestAgent.server: shutting down server +48:48.651 [DEBUG] TestAgent.leader: stopping routine: routine="secondary CA roots watch" +48:48.651 [DEBUG] TestAgent.leader: stopping routine: routine="intention replication" +48:48.651 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:48.651 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:48.651 [DEBUG] TestAgent.leader: stopping routine: routine="config entry replication" +48:48.651 [DEBUG] TestAgent.leader: stopping routine: routine="federation state replication" +48:48.651 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:48.651 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:48.651 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:48.651 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:48.651 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:48.651 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:48.652 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:48.652Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:48.652Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:48.652Z [INFO] agent.router.manager: shutting down +48:48.652 [INFO] TestAgent: consul server down +48:48.652 [INFO] TestAgent: shutdown complete +48:48.652 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24100 network=tcp +48:48.652 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24100 network=udp +48:48.652 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24101 network=tcp +48:48.652 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:24098 datacenter=dc1 method=FederationState.List error="rpc error making call: EOF" +48:48.652 [WARN] TestAgent.replication.federation_state: replication error (will retry if still leader): error="failed to retrieve federation states: rpc error making call: EOF" +48:48.652 [INFO] TestAgent.replication.federation_state: stopped replication +48:48.652 [DEBUG] TestAgent.leader: stopped routine: routine="federation state replication" +48:48.652 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:24098 datacenter=dc1 method=ConnectCA.Roots error="rpc error making call: EOF" +48:48.652 [ERROR] TestAgent.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: rpc error making call: EOF" +48:48.652 [DEBUG] TestAgent.leader: stopped routine: routine="secondary CA roots watch" +48:48.652 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:24098 datacenter=dc1 method=ConfigEntry.ListAll error="rpc error making call: EOF" +48:48.652 [INFO] TestAgent.server.replication.config_entry: stopped replication +48:48.652 [DEBUG] TestAgent.leader: stopped routine: routine="config entry replication" +48:48.652 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:24098 datacenter=dc1 method=Intention.List error="rpc error making call: EOF" +48:48.652 [ERROR] TestAgent.server.connect: error replicating intentions: routine="intention replication" error="rpc error making call: EOF" +48:48.652 [DEBUG] TestAgent.leader: stopped routine: routine="intention replication" +48:49.152 [INFO] TestAgent: Waiting for endpoints to shut down +48:49.152 [INFO] TestAgent: Endpoints down +2022-12-04T05:48:47.875Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:47.905Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:47.907 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3f96d635-d9e3-3605-7d64-c01af93930b4 Address:127.0.0.1:24098}]" +48:47.907 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3f96d635-d9e3-3605-7d64-c01af93930b4.dc1 127.0.0.1 +48:47.908 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24098 [Follower]" leader= +48:47.916 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3f96d635-d9e3-3605-7d64-c01af93930b4 127.0.0.1 +2022-12-04T05:48:47.916Z [INFO] agent.router: Initializing LAN area manager +48:47.916 [INFO] TestAgent.server: Adding LAN server: server="Node-3f96d635-d9e3-3605-7d64-c01af93930b4 (Addr: tcp/127.0.0.1:24098) (DC: dc1)" +48:47.916 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3f96d635-d9e3-3605-7d64-c01af93930b4.dc1 area=wan +48:47.916 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24093 network=udp +48:47.917 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24093 network=tcp +48:47.917 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24094 network=tcp +48:47.917 [INFO] TestAgent: started state syncer +48:47.917 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24099 network=tcp +48:47.963 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:47.963 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24098 [Candidate]" term=2 +48:47.963 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:47.963 [DEBUG] TestAgent.server.raft: vote granted: from=3f96d635-d9e3-3605-7d64-c01af93930b4 term=2 tally=1 +48:47.963 [INFO] TestAgent.server.raft: election won: tally=1 +48:47.963 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24098 [Leader]" +48:47.964 [INFO] TestAgent.server: cluster leadership acquired +48:47.964 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24098 +48:47.964 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:47.966 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:47.966 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:47.966 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:47.966 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:47.966 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:47.966 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3f96d635-d9e3-3605-7d64-c01af93930b4 +48:47.966 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3f96d635-d9e3-3605-7d64-c01af93930b4 +48:47.967 [INFO] TestAgent.server: New leader elected: payload=Node-3f96d635-d9e3-3605-7d64-c01af93930b4 +48:47.967 [INFO] TestAgent.server: federation state anti-entropy synced +48:48.238 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:59168 +48:48.238 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d755788a-b699-8ff8-4db8-572b4cb4c7f0.dc2 127.0.0.1 +48:48.239 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d755788a-b699-8ff8-4db8-572b4cb4c7f0.dc2 area=wan +48:48.300 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:48.301 [INFO] TestAgent: Synced node info +48:48.301 [DEBUG] TestAgent: Node info in sync +48:49.152 [INFO] TestAgent: Requesting shutdown +48:49.152 [INFO] TestAgent.server: shutting down server +48:49.152 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:49.152 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:49.152 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:49.152 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:49.152 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:49.153 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:49.153 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:49.153 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:49.153 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:49.153 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:49.153 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:48:49.153Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:49.153Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:49.153Z [INFO] agent.router.manager: shutting down +48:49.153 [INFO] TestAgent: consul server down +48:49.153 [INFO] TestAgent: shutdown complete +48:49.153 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24093 network=tcp +48:49.153 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24093 network=udp +48:49.153 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24094 network=tcp +48:49.653 [INFO] TestAgent: Waiting for endpoints to shut down +48:49.653 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectCARoots_PEMEncoding (1.81s) === RUN TestCoordinate_Disabled_Response === PAUSE TestCoordinate_Disabled_Response === RUN TestCoordinate_Datacenters @@ -13909,315 +13968,309 @@ [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:34.314Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:34.354Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:34.356 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c4e2cd6f-7ad2-e527-7d0e-a2150f5b7704 Address:127.0.0.1:16965}]" -12:34.356 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16965 [Follower]" leader= -12:34.362 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c4e2cd6f-7ad2-e527-7d0e-a2150f5b7704.dc1 127.0.0.1 -12:34.362 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c4e2cd6f-7ad2-e527-7d0e-a2150f5b7704 127.0.0.1 -2021-10-31T23:12:34.362Z [INFO] agent.router: Initializing LAN area manager -12:34.363 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16960 network=udp -12:34.363 [INFO] TestAgent.server: Adding LAN server: server="Node-c4e2cd6f-7ad2-e527-7d0e-a2150f5b7704 (Addr: tcp/127.0.0.1:16965) (DC: dc1)" -12:34.363 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c4e2cd6f-7ad2-e527-7d0e-a2150f5b7704.dc1 area=wan -12:34.363 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16960 network=tcp -12:34.364 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16961 network=tcp -12:34.364 [INFO] TestAgent: started state syncer -12:34.364 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16966 network=tcp -12:34.429 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:34.429 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16965 [Candidate]" term=2 -12:34.430 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:34.430 [DEBUG] TestAgent.server.raft: vote granted: from=c4e2cd6f-7ad2-e527-7d0e-a2150f5b7704 term=2 tally=1 -12:34.430 [INFO] TestAgent.server.raft: election won: tally=1 -12:34.430 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16965 [Leader]" -12:34.430 [INFO] TestAgent.server: cluster leadership acquired -12:34.430 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16965 -12:34.430 [INFO] TestAgent.server: New leader elected: payload=Node-c4e2cd6f-7ad2-e527-7d0e-a2150f5b7704 -12:34.431 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:34.432 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:34.432 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:34.432 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:34.432 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:34.432 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:34.432 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c4e2cd6f-7ad2-e527-7d0e-a2150f5b7704 -12:34.432 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c4e2cd6f-7ad2-e527-7d0e-a2150f5b7704 -12:34.433 [INFO] TestAgent.server: federation state anti-entropy synced -12:34.574 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:34.574 [INFO] TestAgent: Synced node info -12:34.771 [DEBUG] TestAgent.dns: request served from client: name=google.node.consul. type=TXT class=IN latency=114.854µs client=127.0.0.1:33647 client_network=udp -12:34.771 [INFO] TestAgent: Requesting shutdown -12:34.771 [INFO] TestAgent.server: shutting down server -12:34.771 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:34.771 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:34.771 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:34.771 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:34.771 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:34.772 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:34.772 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:34.772 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:34.772 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:34.772 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:34.772 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:34.772Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:34.772Z [INFO] agent.router.manager: shutting down -12:34.772 [INFO] TestAgent: consul server down -12:34.772 [INFO] TestAgent: shutdown complete -12:34.772 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16960 network=tcp -12:34.772 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16960 network=udp -12:34.772 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16961 network=tcp -12:35.273 [INFO] TestAgent: Waiting for endpoints to shut down -12:35.273 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_TXT (1.01s) +2022-12-04T05:48:49.667Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:49.679Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:49.681 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c0171455-459c-4ba9-c44c-788844439264 Address:127.0.0.1:24112}]" +48:49.681 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24112 [Follower]" leader= +48:49.682 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c0171455-459c-4ba9-c44c-788844439264.dc1 127.0.0.1 +48:49.688 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c0171455-459c-4ba9-c44c-788844439264 127.0.0.1 +2022-12-04T05:48:49.688Z [INFO] agent.router: Initializing LAN area manager +48:49.690 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c0171455-459c-4ba9-c44c-788844439264.dc1 area=wan +48:49.690 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24107 network=udp +48:49.690 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24107 network=tcp +48:49.690 [INFO] TestAgent.server: Adding LAN server: server="Node-c0171455-459c-4ba9-c44c-788844439264 (Addr: tcp/127.0.0.1:24112) (DC: dc1)" +48:49.691 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24108 network=tcp +48:49.691 [INFO] TestAgent: started state syncer +48:49.691 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24113 network=tcp +48:49.729 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:49.729 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24112 [Candidate]" term=2 +48:49.729 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:49.729 [DEBUG] TestAgent.server.raft: vote granted: from=c0171455-459c-4ba9-c44c-788844439264 term=2 tally=1 +48:49.729 [INFO] TestAgent.server.raft: election won: tally=1 +48:49.729 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24112 [Leader]" +48:49.729 [INFO] TestAgent.server: cluster leadership acquired +48:49.729 [INFO] TestAgent.server: New leader elected: payload=Node-c0171455-459c-4ba9-c44c-788844439264 +48:49.729 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24112 +48:49.730 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:49.731 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:49.731 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:49.731 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:49.731 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:49.731 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:49.732 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c0171455-459c-4ba9-c44c-788844439264 +48:49.732 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c0171455-459c-4ba9-c44c-788844439264 +48:49.732 [INFO] TestAgent.server: federation state anti-entropy synced +48:49.926 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:49.927 [INFO] TestAgent: Synced node info +48:50.109 [DEBUG] TestAgent.dns: request served from client: name=google.node.consul. type=TXT class=IN latency=254.921µs client=127.0.0.1:37047 client_network=udp +48:50.109 [INFO] TestAgent: Requesting shutdown +48:50.109 [INFO] TestAgent.server: shutting down server +48:50.110 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:50.110 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:50.110 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:50.110 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:50.110 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:50.110 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:50.110 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:50.110 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:50.110 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:50.110 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:50.110 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:50.110Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:50.110Z [INFO] agent.router.manager: shutting down +48:50.110 [INFO] TestAgent: consul server down +48:50.110 [INFO] TestAgent: shutdown complete +48:50.110 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24107 network=tcp +48:50.110 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24107 network=udp +48:50.110 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24108 network=tcp +48:50.610 [INFO] TestAgent: Waiting for endpoints to shut down +48:50.610 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_TXT (0.96s) === RUN TestDNS_NodeLookup_TXT_DontSuppress [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:35.321Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:35.369Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:35.371 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bce3b758-79d4-ade3-4448-a978fd61796d Address:127.0.0.1:16972}]" -12:35.372 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16972 [Follower]" leader= -12:35.372 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bce3b758-79d4-ade3-4448-a978fd61796d.dc1 127.0.0.1 -12:35.373 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bce3b758-79d4-ade3-4448-a978fd61796d 127.0.0.1 -2021-10-31T23:12:35.373Z [INFO] agent.router: Initializing LAN area manager -12:35.373 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16967 network=udp -12:35.373 [INFO] TestAgent.server: Adding LAN server: server="Node-bce3b758-79d4-ade3-4448-a978fd61796d (Addr: tcp/127.0.0.1:16972) (DC: dc1)" -12:35.373 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bce3b758-79d4-ade3-4448-a978fd61796d.dc1 area=wan -12:35.374 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16967 network=tcp -12:35.374 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16968 network=tcp -12:35.374 [INFO] TestAgent: started state syncer -12:35.374 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16973 network=tcp -12:35.413 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:35.414 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16972 [Candidate]" term=2 -12:35.414 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:35.414 [DEBUG] TestAgent.server.raft: vote granted: from=bce3b758-79d4-ade3-4448-a978fd61796d term=2 tally=1 -12:35.414 [INFO] TestAgent.server.raft: election won: tally=1 -12:35.414 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16972 [Leader]" -12:35.414 [INFO] TestAgent.server: cluster leadership acquired -12:35.414 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16972 -12:35.415 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:35.416 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:35.416 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:35.416 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:35.416 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:35.416 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:35.416 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bce3b758-79d4-ade3-4448-a978fd61796d -12:35.416 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bce3b758-79d4-ade3-4448-a978fd61796d -12:35.417 [INFO] TestAgent.server: New leader elected: payload=Node-bce3b758-79d4-ade3-4448-a978fd61796d -12:35.423 [INFO] TestAgent.server: federation state anti-entropy synced -12:35.514 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:35.514 [INFO] TestAgent: Synced node info -12:35.787 [DEBUG] TestAgent.dns: request served from client: name=google.node.consul. type=TXT class=IN latency=103.343µs client=127.0.0.1:47584 client_network=udp -12:35.788 [INFO] TestAgent: Requesting shutdown -12:35.788 [INFO] TestAgent.server: shutting down server -12:35.788 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:35.788 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:35.788 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:35.788 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:35.788 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:35.788 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:35.788 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:35.788 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:35.788 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:35.788 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:35.788 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:35.788Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:35.788Z [INFO] agent.router.manager: shutting down -12:35.788 [INFO] TestAgent: consul server down -12:35.788 [INFO] TestAgent: shutdown complete -12:35.788 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16967 network=tcp -12:35.788 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16967 network=udp -12:35.788 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16968 network=tcp -12:36.289 [INFO] TestAgent: Waiting for endpoints to shut down -12:36.289 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_TXT_DontSuppress (1.02s) +2022-12-04T05:48:50.622Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:50.636Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:50.638 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:557439bc-64a7-fb1b-53be-2218bbd50a39 Address:127.0.0.1:24119}]" +48:50.638 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24119 [Follower]" leader= +48:50.638 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-557439bc-64a7-fb1b-53be-2218bbd50a39.dc1 127.0.0.1 +48:50.639 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-557439bc-64a7-fb1b-53be-2218bbd50a39 127.0.0.1 +2022-12-04T05:48:50.639Z [INFO] agent.router: Initializing LAN area manager +48:50.639 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24114 network=udp +48:50.639 [INFO] TestAgent.server: Adding LAN server: server="Node-557439bc-64a7-fb1b-53be-2218bbd50a39 (Addr: tcp/127.0.0.1:24119) (DC: dc1)" +48:50.639 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-557439bc-64a7-fb1b-53be-2218bbd50a39.dc1 area=wan +48:50.639 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24114 network=tcp +48:50.639 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24115 network=tcp +48:50.639 [INFO] TestAgent: started state syncer +48:50.639 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24120 network=tcp +48:50.674 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:50.674 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24119 [Candidate]" term=2 +48:50.675 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:50.675 [DEBUG] TestAgent.server.raft: vote granted: from=557439bc-64a7-fb1b-53be-2218bbd50a39 term=2 tally=1 +48:50.675 [INFO] TestAgent.server.raft: election won: tally=1 +48:50.675 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24119 [Leader]" +48:50.675 [INFO] TestAgent.server: cluster leadership acquired +48:50.675 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24119 +48:50.676 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:50.676 [INFO] TestAgent.server: New leader elected: payload=Node-557439bc-64a7-fb1b-53be-2218bbd50a39 +48:50.677 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:50.677 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:50.677 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:50.677 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:50.677 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:50.677 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-557439bc-64a7-fb1b-53be-2218bbd50a39 +48:50.677 [INFO] TestAgent.server: member joined, marking health alive: member=Node-557439bc-64a7-fb1b-53be-2218bbd50a39 +48:50.678 [INFO] TestAgent.server: federation state anti-entropy synced +48:50.798 [DEBUG] TestAgent.dns: request served from client: name=google.node.consul. type=TXT class=IN latency=220.839µs client=127.0.0.1:45353 client_network=udp +48:50.798 [INFO] TestAgent: Requesting shutdown +48:50.798 [INFO] TestAgent.server: shutting down server +48:50.798 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:50.798 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:50.798 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:50.798 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:50.798 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:50.798 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:50.798 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:50.798 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:50.798 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:50.798 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:50.798 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:50.798 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:50.798Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:50.798Z [INFO] agent.router.manager: shutting down +48:50.799 [INFO] TestAgent: consul server down +48:50.799 [INFO] TestAgent: shutdown complete +48:50.799 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24114 network=tcp +48:50.799 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24114 network=udp +48:50.799 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24115 network=tcp +48:51.299 [INFO] TestAgent: Waiting for endpoints to shut down +48:51.299 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_TXT_DontSuppress (0.69s) === RUN TestDNS_NodeLookup_ANY [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:36.338Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:36.380Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:36.397 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f7429caa-e208-f0d9-32ad-05fdd054456e Address:127.0.0.1:16979}]" -12:36.397 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16979 [Follower]" leader= -12:36.398 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f7429caa-e208-f0d9-32ad-05fdd054456e.dc1 127.0.0.1 -12:36.398 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f7429caa-e208-f0d9-32ad-05fdd054456e 127.0.0.1 -2021-10-31T23:12:36.398Z [INFO] agent.router: Initializing LAN area manager -12:36.399 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16974 network=udp -12:36.399 [INFO] TestAgent.server: Adding LAN server: server="Node-f7429caa-e208-f0d9-32ad-05fdd054456e (Addr: tcp/127.0.0.1:16979) (DC: dc1)" -12:36.399 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f7429caa-e208-f0d9-32ad-05fdd054456e.dc1 area=wan -12:36.399 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16974 network=tcp -12:36.399 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16975 network=tcp -12:36.399 [INFO] TestAgent: started state syncer -12:36.400 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16980 network=tcp -12:36.461 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:36.461 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16979 [Candidate]" term=2 -12:36.462 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:36.462 [DEBUG] TestAgent.server.raft: vote granted: from=f7429caa-e208-f0d9-32ad-05fdd054456e term=2 tally=1 -12:36.462 [INFO] TestAgent.server.raft: election won: tally=1 -12:36.462 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16979 [Leader]" -12:36.462 [INFO] TestAgent.server: cluster leadership acquired -12:36.462 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16979 -12:36.463 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:36.464 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:36.464 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:36.464 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:36.464 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:36.464 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:36.464 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f7429caa-e208-f0d9-32ad-05fdd054456e -12:36.464 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f7429caa-e208-f0d9-32ad-05fdd054456e -12:36.465 [INFO] TestAgent.server: New leader elected: payload=Node-f7429caa-e208-f0d9-32ad-05fdd054456e -12:36.465 [INFO] TestAgent.server: federation state anti-entropy synced -12:36.498 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:36.498 [INFO] TestAgent: Synced node info -12:36.498 [DEBUG] TestAgent: Node info in sync -12:36.758 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=ANY class=IN latency=269.417µs client=127.0.0.1:54135 client_network=udp -12:36.761 [INFO] TestAgent: Requesting shutdown -12:36.762 [INFO] TestAgent.server: shutting down server -12:36.762 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:36.762 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:36.762 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:36.762 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:36.762 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:36.762 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:36.762 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:36.762 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:36.762 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:36.762 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:36.762 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:36.762Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:36.762Z [INFO] agent.router.manager: shutting down -12:36.762 [INFO] TestAgent: consul server down -12:36.762 [INFO] TestAgent: shutdown complete -12:36.762 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16974 network=tcp -12:36.762 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16974 network=udp -12:36.762 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16975 network=tcp -12:37.265 [INFO] TestAgent: Waiting for endpoints to shut down -12:37.265 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_ANY (0.98s) +2022-12-04T05:48:51.309Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:51.318Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:51.319 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:25e22c92-5ceb-369f-b366-6959acd723f9 Address:127.0.0.1:24126}]" +48:51.320 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24126 [Follower]" leader= +48:51.320 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-25e22c92-5ceb-369f-b366-6959acd723f9.dc1 127.0.0.1 +48:51.320 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-25e22c92-5ceb-369f-b366-6959acd723f9 127.0.0.1 +2022-12-04T05:48:51.321Z [INFO] agent.router: Initializing LAN area manager +48:51.321 [INFO] TestAgent.server: Adding LAN server: server="Node-25e22c92-5ceb-369f-b366-6959acd723f9 (Addr: tcp/127.0.0.1:24126) (DC: dc1)" +48:51.321 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24121 network=udp +48:51.321 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-25e22c92-5ceb-369f-b366-6959acd723f9.dc1 area=wan +48:51.321 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24121 network=tcp +48:51.321 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24122 network=tcp +48:51.321 [INFO] TestAgent: started state syncer +48:51.321 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24127 network=tcp +48:51.387 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:51.387 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24126 [Candidate]" term=2 +48:51.387 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:51.387 [DEBUG] TestAgent.server.raft: vote granted: from=25e22c92-5ceb-369f-b366-6959acd723f9 term=2 tally=1 +48:51.387 [INFO] TestAgent.server.raft: election won: tally=1 +48:51.387 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24126 [Leader]" +48:51.387 [INFO] TestAgent.server: cluster leadership acquired +48:51.387 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24126 +48:51.388 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:51.390 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:51.390 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:51.390 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:51.390 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:51.390 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:51.390 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-25e22c92-5ceb-369f-b366-6959acd723f9 +48:51.390 [INFO] TestAgent.server: member joined, marking health alive: member=Node-25e22c92-5ceb-369f-b366-6959acd723f9 +48:51.390 [INFO] TestAgent.server: New leader elected: payload=Node-25e22c92-5ceb-369f-b366-6959acd723f9 +48:51.390 [INFO] TestAgent.server: federation state anti-entropy synced +48:51.524 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:51.525 [INFO] TestAgent: Synced node info +48:51.644 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=ANY class=IN latency=199.005µs client=127.0.0.1:44982 client_network=udp +48:51.644 [INFO] TestAgent: Requesting shutdown +48:51.645 [INFO] TestAgent.server: shutting down server +48:51.645 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:51.645 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:51.645 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:51.645 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:51.645 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:51.645 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:51.645 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:51.645 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:51.645 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:51.645 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:51.645 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:51.645Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:51.645Z [INFO] agent.router.manager: shutting down +48:51.645 [INFO] TestAgent: consul server down +48:51.645 [INFO] TestAgent: shutdown complete +48:51.645 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24121 network=tcp +48:51.645 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24121 network=udp +48:51.645 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24122 network=tcp +48:52.148 [INFO] TestAgent: Waiting for endpoints to shut down +48:52.148 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_ANY (0.85s) === RUN TestDNS_NodeLookup_ANY_DontSuppressTXT [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:37.327Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:37.391Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:37.393 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a1b2c7cb-46ff-b6dd-7380-0f5930ac5b0c Address:127.0.0.1:16986}]" -12:37.393 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16986 [Follower]" leader= -12:37.402 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a1b2c7cb-46ff-b6dd-7380-0f5930ac5b0c.dc1 127.0.0.1 -12:37.403 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a1b2c7cb-46ff-b6dd-7380-0f5930ac5b0c 127.0.0.1 -2021-10-31T23:12:37.403Z [INFO] agent.router: Initializing LAN area manager -12:37.410 [INFO] TestAgent.server: Adding LAN server: server="Node-a1b2c7cb-46ff-b6dd-7380-0f5930ac5b0c (Addr: tcp/127.0.0.1:16986) (DC: dc1)" -12:37.410 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a1b2c7cb-46ff-b6dd-7380-0f5930ac5b0c.dc1 area=wan -12:37.414 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16981 network=udp -12:37.414 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16981 network=tcp -12:37.415 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16982 network=tcp -12:37.415 [INFO] TestAgent: started state syncer -12:37.415 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16987 network=tcp -12:37.457 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:37.457 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16986 [Candidate]" term=2 -12:37.457 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:37.457 [DEBUG] TestAgent.server.raft: vote granted: from=a1b2c7cb-46ff-b6dd-7380-0f5930ac5b0c term=2 tally=1 -12:37.457 [INFO] TestAgent.server.raft: election won: tally=1 -12:37.457 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16986 [Leader]" -12:37.457 [INFO] TestAgent.server: cluster leadership acquired -12:37.457 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16986 -12:37.457 [INFO] TestAgent.server: New leader elected: payload=Node-a1b2c7cb-46ff-b6dd-7380-0f5930ac5b0c -12:37.458 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:37.460 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:37.460 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:37.460 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:37.460 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:37.460 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:37.460 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1b2c7cb-46ff-b6dd-7380-0f5930ac5b0c -12:37.460 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a1b2c7cb-46ff-b6dd-7380-0f5930ac5b0c -12:37.461 [INFO] TestAgent.server: federation state anti-entropy synced -12:37.754 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=ANY class=IN latency=227.006µs client=127.0.0.1:44040 client_network=udp -12:37.757 [INFO] TestAgent: Requesting shutdown -12:37.758 [INFO] TestAgent.server: shutting down server -12:37.758 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:37.758 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:37.758 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:37.758 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:37.758 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:37.758 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:37.758 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:37.758 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:37.758 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:37.758 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:37.758 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:37.758 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:37.758Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:37.758Z [INFO] agent.router.manager: shutting down -12:37.758 [INFO] TestAgent: consul server down -12:37.758 [INFO] TestAgent: shutdown complete -12:37.758 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16981 network=tcp -12:37.758 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16981 network=udp -12:37.758 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16982 network=tcp -12:38.261 [INFO] TestAgent: Waiting for endpoints to shut down -12:38.261 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_ANY_DontSuppressTXT (1.00s) +2022-12-04T05:48:52.175Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:52.205Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:52.207 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ad228865-016b-bd9e-4aba-34974ce74fb3 Address:127.0.0.1:24133}]" +48:52.208 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ad228865-016b-bd9e-4aba-34974ce74fb3.dc1 127.0.0.1 +48:52.208 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24133 [Follower]" leader= +48:52.208 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ad228865-016b-bd9e-4aba-34974ce74fb3 127.0.0.1 +2022-12-04T05:48:52.208Z [INFO] agent.router: Initializing LAN area manager +48:52.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24128 network=udp +48:52.208 [INFO] TestAgent.server: Adding LAN server: server="Node-ad228865-016b-bd9e-4aba-34974ce74fb3 (Addr: tcp/127.0.0.1:24133) (DC: dc1)" +48:52.209 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ad228865-016b-bd9e-4aba-34974ce74fb3.dc1 area=wan +48:52.209 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24128 network=tcp +48:52.209 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24129 network=tcp +48:52.209 [INFO] TestAgent: started state syncer +48:52.209 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24134 network=tcp +48:52.256 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:52.256 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24133 [Candidate]" term=2 +48:52.256 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:52.256 [DEBUG] TestAgent.server.raft: vote granted: from=ad228865-016b-bd9e-4aba-34974ce74fb3 term=2 tally=1 +48:52.256 [INFO] TestAgent.server.raft: election won: tally=1 +48:52.256 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24133 [Leader]" +48:52.256 [INFO] TestAgent.server: cluster leadership acquired +48:52.256 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24133 +48:52.257 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:52.258 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:52.259 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:52.259 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:52.259 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:52.259 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:52.259 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ad228865-016b-bd9e-4aba-34974ce74fb3 +48:52.259 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ad228865-016b-bd9e-4aba-34974ce74fb3 +48:52.259 [INFO] TestAgent.server: New leader elected: payload=Node-ad228865-016b-bd9e-4aba-34974ce74fb3 +48:52.259 [INFO] TestAgent.server: federation state anti-entropy synced +48:52.344 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=ANY class=IN latency=221.907µs client=127.0.0.1:50319 client_network=udp +48:52.345 [INFO] TestAgent: Requesting shutdown +48:52.345 [INFO] TestAgent.server: shutting down server +48:52.345 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:52.345 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:52.345 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:52.345 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:52.345 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:52.345 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:52.345 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:52.345 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:52.345 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:52.345 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:52.345 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:52.345 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:52.345Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:52.345Z [INFO] agent.router.manager: shutting down +48:52.345 [INFO] TestAgent: consul server down +48:52.345 [INFO] TestAgent: shutdown complete +48:52.345 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24128 network=tcp +48:52.345 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24128 network=udp +48:52.345 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24129 network=tcp +48:52.845 [INFO] TestAgent: Waiting for endpoints to shut down +48:52.845 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_ANY_DontSuppressTXT (0.70s) === RUN TestDNS_NodeLookup_A_SuppressTXT [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:38.292Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:38.334Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:38.336 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:42abfa2f-4eab-a664-4fe7-de2b112d04a2 Address:127.0.0.1:16993}]" -12:38.337 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-42abfa2f-4eab-a664-4fe7-de2b112d04a2.dc1 127.0.0.1 -12:38.337 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16993 [Follower]" leader= -12:38.346 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-42abfa2f-4eab-a664-4fe7-de2b112d04a2 127.0.0.1 -2021-10-31T23:12:38.346Z [INFO] agent.router: Initializing LAN area manager -12:38.346 [INFO] TestAgent.server: Adding LAN server: server="Node-42abfa2f-4eab-a664-4fe7-de2b112d04a2 (Addr: tcp/127.0.0.1:16993) (DC: dc1)" -12:38.346 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-42abfa2f-4eab-a664-4fe7-de2b112d04a2.dc1 area=wan -12:38.346 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16988 network=udp -12:38.346 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16988 network=tcp -12:38.347 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16989 network=tcp -12:38.347 [INFO] TestAgent: started state syncer -12:38.347 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16994 network=tcp -12:38.404 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:38.404 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16993 [Candidate]" term=2 -12:38.404 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:38.404 [DEBUG] TestAgent.server.raft: vote granted: from=42abfa2f-4eab-a664-4fe7-de2b112d04a2 term=2 tally=1 -12:38.404 [INFO] TestAgent.server.raft: election won: tally=1 -12:38.404 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16993 [Leader]" -12:38.404 [INFO] TestAgent.server: cluster leadership acquired -12:38.404 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16993 -12:38.405 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:38.405 [INFO] TestAgent.server: New leader elected: payload=Node-42abfa2f-4eab-a664-4fe7-de2b112d04a2 -12:38.406 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:38.406 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:38.406 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:38.406 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:38.406 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:38.406 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-42abfa2f-4eab-a664-4fe7-de2b112d04a2 -12:38.406 [INFO] TestAgent.server: member joined, marking health alive: member=Node-42abfa2f-4eab-a664-4fe7-de2b112d04a2 -12:38.407 [INFO] TestAgent.server: federation state anti-entropy synced -12:38.613 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=A class=IN latency=72.931µs client=127.0.0.1:58115 client_network=udp -12:38.613 [INFO] TestAgent: Requesting shutdown -12:38.613 [INFO] TestAgent.server: shutting down server -12:38.613 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:38.613 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:38.613 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:38.613 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:38.613 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:38.613 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:38.613 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:38.614 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:38.613 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:38.613 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:38.613 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:38.614 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:38.614Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:38.614Z [INFO] agent.router.manager: shutting down -12:38.614 [INFO] TestAgent: consul server down -12:38.614 [INFO] TestAgent: shutdown complete -12:38.614 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16988 network=tcp -12:38.614 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16988 network=udp -12:38.614 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16989 network=tcp -12:39.115 [INFO] TestAgent: Waiting for endpoints to shut down -12:39.115 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_A_SuppressTXT (0.85s) +2022-12-04T05:48:52.869Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:52.879Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:52.880 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aa4468d2-2f45-0a74-b42f-1079e2b1b1a2 Address:127.0.0.1:24140}]" +48:52.880 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24140 [Follower]" leader= +48:52.881 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aa4468d2-2f45-0a74-b42f-1079e2b1b1a2.dc1 127.0.0.1 +48:52.881 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aa4468d2-2f45-0a74-b42f-1079e2b1b1a2 127.0.0.1 +2022-12-04T05:48:52.881Z [INFO] agent.router: Initializing LAN area manager +48:52.881 [INFO] TestAgent.server: Adding LAN server: server="Node-aa4468d2-2f45-0a74-b42f-1079e2b1b1a2 (Addr: tcp/127.0.0.1:24140) (DC: dc1)" +48:52.881 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aa4468d2-2f45-0a74-b42f-1079e2b1b1a2.dc1 area=wan +48:52.881 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24135 network=udp +48:52.881 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24135 network=tcp +48:52.882 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24136 network=tcp +48:52.882 [INFO] TestAgent: started state syncer +48:52.882 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24141 network=tcp +48:52.946 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:52.946 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24140 [Candidate]" term=2 +48:52.946 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:52.946 [DEBUG] TestAgent.server.raft: vote granted: from=aa4468d2-2f45-0a74-b42f-1079e2b1b1a2 term=2 tally=1 +48:52.946 [INFO] TestAgent.server.raft: election won: tally=1 +48:52.946 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24140 [Leader]" +48:52.946 [INFO] TestAgent.server: cluster leadership acquired +48:52.946 [INFO] TestAgent.server: New leader elected: payload=Node-aa4468d2-2f45-0a74-b42f-1079e2b1b1a2 +48:52.946 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24140 +48:52.947 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:52.948 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:52.948 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:52.948 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:52.948 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:52.948 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:52.948 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aa4468d2-2f45-0a74-b42f-1079e2b1b1a2 +48:52.948 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aa4468d2-2f45-0a74-b42f-1079e2b1b1a2 +48:52.948 [INFO] TestAgent.server: federation state anti-entropy synced +48:53.111 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=A class=IN latency=177.025µs client=127.0.0.1:59600 client_network=udp +48:53.111 [INFO] TestAgent: Requesting shutdown +48:53.111 [INFO] TestAgent.server: shutting down server +48:53.111 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:53.111 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:53.111 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:53.111 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:53.111 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:53.111 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:53.111 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:53.111 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:53.111 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:53.111 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:53.111 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:53.111 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:48:53.111Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:53.111Z [INFO] agent.router.manager: shutting down +48:53.112 [INFO] TestAgent: consul server down +48:53.112 [INFO] TestAgent: shutdown complete +48:53.112 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24135 network=tcp +48:53.112 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24135 network=udp +48:53.112 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24136 network=tcp +48:53.612 [INFO] TestAgent: Waiting for endpoints to shut down +48:53.612 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_A_SuppressTXT (0.77s) === RUN TestDNS_EDNS0 === PAUSE TestDNS_EDNS0 === RUN TestDNS_EDNS0_ECS @@ -14337,128 +14390,125 @@ === RUN TestDNS_ServiceLookup_MetaTXT [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:39.149Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:39.224Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:39.226 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dee00557-1d2d-63fe-a3e5-9f7b52544c11 Address:127.0.0.1:17000}]" -12:39.227 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dee00557-1d2d-63fe-a3e5-9f7b52544c11.dc1 127.0.0.1 -12:39.227 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dee00557-1d2d-63fe-a3e5-9f7b52544c11 127.0.0.1 -2021-10-31T23:12:39.227Z [INFO] agent.router: Initializing LAN area manager -12:39.227 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16995 network=udp -12:39.227 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17000 [Follower]" leader= -12:39.228 [INFO] TestAgent.server: Adding LAN server: server="Node-dee00557-1d2d-63fe-a3e5-9f7b52544c11 (Addr: tcp/127.0.0.1:17000) (DC: dc1)" -12:39.228 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dee00557-1d2d-63fe-a3e5-9f7b52544c11.dc1 area=wan -12:39.228 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16995 network=tcp -12:39.228 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16996 network=tcp -12:39.228 [INFO] TestAgent: started state syncer -12:39.228 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17001 network=tcp -12:39.277 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:39.277 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17000 [Candidate]" term=2 -12:39.278 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:39.278 [DEBUG] TestAgent.server.raft: vote granted: from=dee00557-1d2d-63fe-a3e5-9f7b52544c11 term=2 tally=1 -12:39.278 [INFO] TestAgent.server.raft: election won: tally=1 -12:39.278 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17000 [Leader]" -12:39.278 [INFO] TestAgent.server: cluster leadership acquired -12:39.278 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17000 -12:39.278 [INFO] TestAgent.server: New leader elected: payload=Node-dee00557-1d2d-63fe-a3e5-9f7b52544c11 -12:39.279 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:39.280 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:39.280 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:39.280 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:39.280 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:39.280 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:39.280 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dee00557-1d2d-63fe-a3e5-9f7b52544c11 -12:39.280 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dee00557-1d2d-63fe-a3e5-9f7b52544c11 -12:39.281 [INFO] TestAgent.server: federation state anti-entropy synced -12:39.434 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:39.434 [INFO] TestAgent: Synced node info -12:39.434 [DEBUG] TestAgent: Node info in sync -12:39.619 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=4.022386ms client=127.0.0.1:60211 client_network=udp -12:39.622 [INFO] TestAgent: Requesting shutdown -12:39.629 [INFO] TestAgent.server: shutting down server -12:39.630 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:39.630 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:39.630 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:39.630 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:39.630 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:39.630 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:39.630 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:39.630 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:39.630 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:39.630 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:39.630 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:39.630Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:39.630Z [INFO] agent.router.manager: shutting down -12:39.630 [INFO] TestAgent: consul server down -12:39.630 [INFO] TestAgent: shutdown complete -12:39.630 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16995 network=tcp -12:39.630 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16995 network=udp -12:39.630 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16996 network=tcp -12:40.133 [INFO] TestAgent: Waiting for endpoints to shut down -12:40.133 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_MetaTXT (1.02s) +2022-12-04T05:48:53.626Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:53.635Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:53.637 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:775718b8-6017-8db8-8d6d-5d9c474e4daa Address:127.0.0.1:24147}]" +48:53.637 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24147 [Follower]" leader= +48:53.637 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-775718b8-6017-8db8-8d6d-5d9c474e4daa.dc1 127.0.0.1 +48:53.637 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-775718b8-6017-8db8-8d6d-5d9c474e4daa 127.0.0.1 +2022-12-04T05:48:53.637Z [INFO] agent.router: Initializing LAN area manager +48:53.638 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24142 network=udp +48:53.638 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-775718b8-6017-8db8-8d6d-5d9c474e4daa.dc1 area=wan +48:53.638 [INFO] TestAgent.server: Adding LAN server: server="Node-775718b8-6017-8db8-8d6d-5d9c474e4daa (Addr: tcp/127.0.0.1:24147) (DC: dc1)" +48:53.638 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24142 network=tcp +48:53.638 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24143 network=tcp +48:53.638 [INFO] TestAgent: started state syncer +48:53.638 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24148 network=tcp +48:53.695 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:53.695 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24147 [Candidate]" term=2 +48:53.695 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:53.695 [DEBUG] TestAgent.server.raft: vote granted: from=775718b8-6017-8db8-8d6d-5d9c474e4daa term=2 tally=1 +48:53.695 [INFO] TestAgent.server.raft: election won: tally=1 +48:53.695 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24147 [Leader]" +48:53.696 [INFO] TestAgent.server: cluster leadership acquired +48:53.696 [INFO] TestAgent.server: New leader elected: payload=Node-775718b8-6017-8db8-8d6d-5d9c474e4daa +48:53.696 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24147 +48:53.697 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:53.698 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:53.698 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:53.698 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:53.698 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:53.698 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:53.698 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-775718b8-6017-8db8-8d6d-5d9c474e4daa +48:53.698 [INFO] TestAgent.server: member joined, marking health alive: member=Node-775718b8-6017-8db8-8d6d-5d9c474e4daa +48:53.699 [INFO] TestAgent.server: federation state anti-entropy synced +48:53.792 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=216.043µs client=127.0.0.1:46616 client_network=udp +48:53.792 [INFO] TestAgent: Requesting shutdown +48:53.792 [INFO] TestAgent.server: shutting down server +48:53.792 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:53.792 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:53.792 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:53.792 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:53.792 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:53.792 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:53.792 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:53.792 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:53.792 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:53.792 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:53.792 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:53.792 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:53.792Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:53.792Z [INFO] agent.router.manager: shutting down +48:53.792 [INFO] TestAgent: consul server down +48:53.792 [INFO] TestAgent: shutdown complete +48:53.792 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24142 network=tcp +48:53.792 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24142 network=udp +48:53.792 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24143 network=tcp +48:54.293 [INFO] TestAgent: Waiting for endpoints to shut down +48:54.293 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_MetaTXT (0.68s) === RUN TestDNS_ServiceLookup_SuppressTXT [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:40.182Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:40.227Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:40.229 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d11b3193-d1c3-4d52-b3c0-512720117942 Address:127.0.0.1:17007}]" -12:40.229 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17007 [Follower]" leader= -12:40.250 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d11b3193-d1c3-4d52-b3c0-512720117942.dc1 127.0.0.1 -12:40.251 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d11b3193-d1c3-4d52-b3c0-512720117942 127.0.0.1 -2021-10-31T23:12:40.251Z [INFO] agent.router: Initializing LAN area manager -12:40.251 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17002 network=udp -12:40.254 [INFO] TestAgent.server: Adding LAN server: server="Node-d11b3193-d1c3-4d52-b3c0-512720117942 (Addr: tcp/127.0.0.1:17007) (DC: dc1)" -12:40.254 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17002 network=tcp -12:40.254 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17003 network=tcp -12:40.254 [INFO] TestAgent: started state syncer -12:40.255 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17008 network=tcp -12:40.266 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d11b3193-d1c3-4d52-b3c0-512720117942.dc1 area=wan -12:40.277 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:40.277 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17007 [Candidate]" term=2 -12:40.278 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:40.278 [DEBUG] TestAgent.server.raft: vote granted: from=d11b3193-d1c3-4d52-b3c0-512720117942 term=2 tally=1 -12:40.278 [INFO] TestAgent.server.raft: election won: tally=1 -12:40.278 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17007 [Leader]" -12:40.278 [INFO] TestAgent.server: cluster leadership acquired -12:40.278 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17007 -12:40.279 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:40.280 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:40.280 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:40.280 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:40.280 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:40.280 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:40.280 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d11b3193-d1c3-4d52-b3c0-512720117942 -12:40.280 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d11b3193-d1c3-4d52-b3c0-512720117942 -12:40.281 [INFO] TestAgent.server: New leader elected: payload=Node-d11b3193-d1c3-4d52-b3c0-512720117942 -12:40.281 [INFO] TestAgent.server: federation state anti-entropy synced -12:40.458 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=275.122µs client=127.0.0.1:59010 client_network=udp -12:40.461 [INFO] TestAgent: Requesting shutdown -12:40.462 [INFO] TestAgent.server: shutting down server -12:40.462 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:40.462 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:40.462 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:40.462 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:40.462 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:40.462 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:40.462 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:40.462 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -12:40.462 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:40.462 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:40.462 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:40.462 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:12:40.462Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:40.462Z [INFO] agent.router.manager: shutting down -12:40.462 [INFO] TestAgent: consul server down -12:40.462 [INFO] TestAgent: shutdown complete -12:40.462 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17002 network=tcp -12:40.462 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17002 network=udp -12:40.462 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17003 network=tcp -12:40.965 [INFO] TestAgent: Waiting for endpoints to shut down -12:40.965 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_SuppressTXT (0.83s) +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:48:54.303Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:54.312Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:54.314 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f3e76cb5-83ad-8639-45e2-1a93c78db65d Address:127.0.0.1:24154}]" +48:54.314 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f3e76cb5-83ad-8639-45e2-1a93c78db65d.dc1 127.0.0.1 +48:54.315 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f3e76cb5-83ad-8639-45e2-1a93c78db65d 127.0.0.1 +2022-12-04T05:48:54.315Z [INFO] agent.router: Initializing LAN area manager +48:54.315 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24149 network=udp +48:54.315 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24154 [Follower]" leader= +48:54.315 [INFO] TestAgent.server: Adding LAN server: server="Node-f3e76cb5-83ad-8639-45e2-1a93c78db65d (Addr: tcp/127.0.0.1:24154) (DC: dc1)" +48:54.315 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f3e76cb5-83ad-8639-45e2-1a93c78db65d.dc1 area=wan +48:54.315 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24149 network=tcp +48:54.315 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24150 network=tcp +48:54.316 [INFO] TestAgent: started state syncer +48:54.316 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24155 network=tcp +48:54.381 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:54.381 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24154 [Candidate]" term=2 +48:54.382 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:54.382 [DEBUG] TestAgent.server.raft: vote granted: from=f3e76cb5-83ad-8639-45e2-1a93c78db65d term=2 tally=1 +48:54.382 [INFO] TestAgent.server.raft: election won: tally=1 +48:54.382 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24154 [Leader]" +48:54.382 [INFO] TestAgent.server: cluster leadership acquired +48:54.382 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24154 +48:54.382 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:54.383 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:54.383 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:54.383 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:54.383 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:54.383 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:54.384 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f3e76cb5-83ad-8639-45e2-1a93c78db65d +48:54.384 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f3e76cb5-83ad-8639-45e2-1a93c78db65d +48:54.384 [INFO] TestAgent.server: New leader elected: payload=Node-f3e76cb5-83ad-8639-45e2-1a93c78db65d +48:54.384 [INFO] TestAgent.server: federation state anti-entropy synced +48:54.576 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=86.646µs client=127.0.0.1:36265 client_network=udp +48:54.577 [INFO] TestAgent: Requesting shutdown +48:54.577 [INFO] TestAgent.server: shutting down server +48:54.577 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:54.577 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:54.577 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:54.577 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:54.577 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:54.577 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:54.577 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:54.577 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:54.577 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:54.577 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:54.577 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:54.577 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:54.577Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:54.577Z [INFO] agent.router.manager: shutting down +48:54.577 [INFO] TestAgent: consul server down +48:54.577 [INFO] TestAgent: shutdown complete +48:54.577 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24149 network=tcp +48:54.577 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24149 network=udp +48:54.577 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24150 network=tcp +48:55.078 [INFO] TestAgent: Waiting for endpoints to shut down +48:55.078 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_SuppressTXT (0.79s) === RUN TestDNS_AddressLookup === PAUSE TestDNS_AddressLookup === RUN TestDNS_AddressLookupIPV6 @@ -14815,7 +14865,7 @@ === RUN TestDecodeAgentRegisterService/serviceID:_first_set === RUN TestDecodeAgentRegisterService/serviceID:_second_set === RUN TestDecodeAgentRegisterService/serviceID:_neither_set ---- PASS: TestDecodeAgentRegisterService (0.02s) +--- PASS: TestDecodeAgentRegisterService (0.01s) --- PASS: TestDecodeAgentRegisterService/translateEnableTagTCs:_both_set (0.00s) --- PASS: TestDecodeAgentRegisterService/translateEnableTagTCs:_first_set (0.00s) --- PASS: TestDecodeAgentRegisterService/translateEnableTagTCs:_second_set (0.00s) @@ -15021,19 +15071,6 @@ --- PASS: TestDecodeTxnConvertOps/durations_negative (0.00s) --- PASS: TestDecodeTxnConvertOps/durations_numeric_and_negative (0.00s) === RUN TestHTTPAPI_MethodNotAllowed_OSS -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query/xxx/execute === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query/xxx/execute === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query/xxx/execute @@ -15046,224 +15083,415 @@ === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/xxx/explain === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query/xxx/explain === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query/xxx/explain -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rules -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rules -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rules -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rules -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rules -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rules -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/force-leave/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/force-leave/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/force-leave/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/force-leave/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/force-leave/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/force-leave/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/autopilot/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/autopilot/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/autopilot/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/autopilot/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/autopilot/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/autopilot/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/check -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/check -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/check -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/check -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/check -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/check -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/checks/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/checks/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/checks/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/checks/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/checks/ -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/checks/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-methods -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-methods -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-methods -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-methods -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-methods -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-methods -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/members -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/members -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/members -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/members -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/members -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/members -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/connect/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/config/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/config/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/config/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/config/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/config/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/config/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/tokens +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/tokens +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/tokens +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/tokens +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/tokens +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/tokens +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/services === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/maintenance/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/maintenance/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/maintenance/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/maintenance/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/maintenance/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/maintenance/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rule/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rule/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rule/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rule/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rule/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rule/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/warn/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/warn/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/warn/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/warn/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/warn/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/warn/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rule -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rule -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rule -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rule -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rule -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rule -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/deregister/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/renew/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/renew/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/renew/ -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/renew/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/renew/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/renew/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/info/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/info/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/info/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/info/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/info/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/info/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/metrics -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/metrics -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/metrics -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/metrics -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/metrics -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/metrics -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/deregister -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/deregister -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/deregister -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/deregister -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/deregister -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/deregister -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-states -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-states -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-states -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-states -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-states -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-states +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/rules/translate +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/rules/translate +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/rules/translate +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/rules/translate +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/rules/translate +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/rules/translate +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rule +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rule +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rule +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rule +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rule +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rule +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/ca/roots +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/ca/roots +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/ca/roots +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/ca/roots +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/ca/roots +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/ca/roots +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/force-leave/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/force-leave/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/force-leave/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/force-leave/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/force-leave/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/force-leave/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/leave +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/leave +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/leave +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/leave +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/leave +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/leave +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/destroy/ +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/update/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/update/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/update/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/update/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/update/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/update/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/acl/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/acl/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/acl/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/acl/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/acl/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/acl/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/raft/peer +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/raft/peer +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/raft/peer +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/raft/peer +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/raft/peer +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/raft/peer === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/keyring === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/keyring === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/keyring === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/keyring === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/keyring === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/keyring -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/snapshot -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/snapshot -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/snapshot -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/snapshot -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/snapshot -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/snapshot +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/login +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/login +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/login +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/login +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/login +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/login +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/checks +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/checks +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/checks +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/checks +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/checks +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/checks +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/gateway-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/gateway-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/gateway-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/gateway-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/gateway-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/gateway-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/clone/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/clone/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/clone/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/clone/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/clone/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/clone/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/authorize +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policies +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policies +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policies +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policies +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policies +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policies +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/info/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/info/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/info/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/info/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/info/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/info/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-state/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/create +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/service/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/destroy/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/destroy/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/destroy/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/destroy/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/destroy/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/destroy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/autopilot/health +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/autopilot/health +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/autopilot/health +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/autopilot/health +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/autopilot/health +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/autopilot/health +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/gateway-services-nodes/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/gateway-services-nodes/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/gateway-services-nodes/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/gateway-services-nodes/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/gateway-services-nodes/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/gateway-services-nodes/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/node/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/config +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/config +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/config +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/config +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/config +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/config +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/kv/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/kv/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/kv/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/kv/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/kv/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/kv/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rule/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rule/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rule/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rule/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rule/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rule/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/discovery-chain/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/discovery-chain/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/discovery-chain/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/discovery-chain/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/discovery-chain/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/discovery-chain/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/event/fire/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/event/fire/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/event/fire/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/event/fire/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/event/fire/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/event/fire/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/ca/leaf/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/ca/leaf/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/ca/leaf/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/ca/leaf/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/ca/leaf/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/ca/leaf/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/check +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/check +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/check +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/check +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/check +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/check === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/register === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/register === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/register === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/register === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/register === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/event/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/event/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/event/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/event/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/event/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/event/list === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/connect/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/connect/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/connect/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/connect/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/connect/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/raft/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/raft/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/raft/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/raft/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/raft/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/raft/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/autopilot/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/autopilot/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/autopilot/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/autopilot/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/autopilot/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/autopilot/configuration +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/metrics +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/metrics +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/metrics +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/metrics +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/metrics +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/metrics +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/members +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/members +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/members +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/members +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/members +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/members +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/deregister/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-states/mesh-gateways +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-states/mesh-gateways +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-states/mesh-gateways +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-states/mesh-gateways +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-states/mesh-gateways +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-states/mesh-gateways +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/logout +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/logout +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/logout +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/logout +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/logout +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/logout === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/health/service/id/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/health/service/id/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/health/service/id/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/health/service/id/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/health/service/id/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/health/service/id/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/pass/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/pass/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/pass/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/pass/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/pass/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/pass/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/replication +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/replication +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/replication +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/replication +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/replication +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/replication === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/roles === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/roles === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/roles @@ -15276,1733 +15504,1536 @@ === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/node/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/node/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/txn -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/txn -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/txn -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/txn -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/txn -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/txn -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/datacenters -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-state/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/logout -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/logout -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/logout -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/logout -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/logout -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/logout -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/pass/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/pass/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/pass/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/pass/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/pass/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/pass/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/gateway-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/gateway-services/ -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/gateway-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/gateway-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/gateway-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/gateway-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/leave -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/leave -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/leave -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/leave -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/leave -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/leave -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/login -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/login -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/login -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/login -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/login -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/login -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/self -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/gateway-services-nodes/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/gateway-services-nodes/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/gateway-services-nodes/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/gateway-services-nodes/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/gateway-services-nodes/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/gateway-services-nodes/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/raft/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/raft/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/raft/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/raft/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/raft/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/raft/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/clone/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/clone/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/clone/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/clone/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/clone/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/clone/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/maintenance -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/maintenance -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/maintenance -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/maintenance -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/maintenance -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/maintenance -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/create -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/fail/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/fail/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/fail/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/fail/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/fail/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/fail/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/update/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/update/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/update/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/update/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/update/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/update/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/node/ -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/services -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-states +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-states +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-states +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-states +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-states +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-states +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/checks/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/checks/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/checks/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/checks/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/checks/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/checks/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/health/service/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/health/service/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/health/service/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/health/service/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/health/service/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/health/service/name/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/nodes === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/ca/configuration === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/ca/configuration === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/ca/configuration === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/ca/configuration === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/ca/configuration === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/ca/configuration -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/info/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/info/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/info/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/info/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/info/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/info/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-states/mesh-gateways -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-states/mesh-gateways -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-states/mesh-gateways -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-states/mesh-gateways -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-states/mesh-gateways -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-states/mesh-gateways === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/node/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/node/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/node/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/node/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/node/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policies -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policies -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policies -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policies -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policies -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policies -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/join/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/join/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/join/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/join/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/join/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/join/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/update -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/checks -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/checks -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/checks -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/checks -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/checks -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/checks -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/kv/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/kv/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/kv/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/kv/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/kv/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/kv/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/status/leader +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/status/leader +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/status/leader +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/status/leader +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/status/leader +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/status/leader +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/snapshot +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/snapshot +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/snapshot +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/snapshot +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/snapshot +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/snapshot +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-methods +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-methods +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-methods +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-methods +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-methods +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-methods +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/warn/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/warn/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/warn/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/warn/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/warn/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/warn/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token/self +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/maintenance +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/maintenance +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/maintenance +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/maintenance +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/maintenance +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/maintenance +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/match +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/match +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/match +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/match +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/match +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/match +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/ingress/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/ingress/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/ingress/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/ingress/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/ingress/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/ingress/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/services +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/node-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/node-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/node-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/node-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/node-services/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/node-services/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/ca/roots === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/ca/roots === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/ca/roots === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/ca/roots === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/ca/roots === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/ca/roots +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/update +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/update === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/bootstrap === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/bootstrap === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/bootstrap === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/bootstrap === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/bootstrap === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/bootstrap -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/list -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/list -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/event/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/event/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/event/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/event/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/event/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/event/list +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/nodes +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/txn +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/txn +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/txn +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/txn +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/txn +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/txn +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/config/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/config/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/config/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/config/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/config/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/config/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/rules/translate/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/rules/translate/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/rules/translate/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/rules/translate/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/rules/translate/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/rules/translate/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/nodes -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/ca/leaf/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/ca/leaf/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/ca/leaf/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/ca/leaf/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/ca/leaf/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/ca/leaf/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/event/fire/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/event/fire/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/event/fire/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/event/fire/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/event/fire/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/event/fire/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/service/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/join/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/join/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/join/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/join/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/join/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/join/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/datacenters +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/datacenters === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-method === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-method === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-method === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-method === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-method === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-method -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/config -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/config -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/config -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/config -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/config -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/config -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/match -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/match -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/match -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/match -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/match -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/match -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/rules/translate -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/rules/translate -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/rules/translate -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/rules/translate -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/rules/translate -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/rules/translate -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token/ === RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/host === RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/host === RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/host === RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/host === RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/host === RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/host -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/health/service/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/health/service/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/health/service/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/health/service/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/health/service/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/health/service/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/register -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy/name/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/raft/peer -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/raft/peer -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/raft/peer -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/raft/peer -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/raft/peer -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/raft/peer -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/replication -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/replication -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/replication -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/replication -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/replication -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/replication -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/acl/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/acl/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/acl/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/acl/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/acl/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/acl/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/destroy/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/deregister/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/ingress/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/ingress/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/ingress/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/ingress/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/ingress/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/ingress/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/autopilot/health -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/autopilot/health -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/autopilot/health -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/autopilot/health -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/autopilot/health -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/autopilot/health -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/status/leader -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/status/leader -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/status/leader -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/status/leader -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/status/leader -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/status/leader -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/node/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/tokens -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/tokens -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/tokens -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/tokens -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/tokens -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/tokens -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/authorize -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/ca/roots -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/ca/roots -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/ca/roots -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/ca/roots -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/ca/roots -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/ca/roots -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/node-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/node-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/node-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/node-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/node-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/node-services/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/discovery-chain/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/discovery-chain/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/discovery-chain/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/discovery-chain/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/discovery-chain/ -=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/discovery-chain/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/register +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/connect/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/maintenance/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/maintenance/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/maintenance/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/maintenance/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/maintenance/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/maintenance/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/deregister +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/deregister +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/deregister +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/deregister +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/deregister +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/deregister +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/info/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/info/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/info/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/info/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/info/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/info/ +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rules +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rules +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rules +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rules +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rules +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rules +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token +=== RUN TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:41.022Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:41.045Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:41.051 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:eb2bf899-e5f7-814f-a40d-2f73de031663 Address:127.0.0.1:17014}]" -12:41.052 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-eb2bf899-e5f7-814f-a40d-2f73de031663.dc1 127.0.0.1 -12:41.052 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-eb2bf899-e5f7-814f-a40d-2f73de031663 127.0.0.1 -2021-10-31T23:12:41.052Z [INFO] agent.router: Initializing LAN area manager -12:41.052 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17009 network=udp -12:41.052 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17014 [Follower]" leader= -12:41.053 [INFO] TestAgent.server: Adding LAN server: server="Node-eb2bf899-e5f7-814f-a40d-2f73de031663 (Addr: tcp/127.0.0.1:17014) (DC: dc1)" -12:41.053 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-eb2bf899-e5f7-814f-a40d-2f73de031663.dc1 area=wan -12:41.053 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17009 network=tcp -12:41.058 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17010 network=tcp -12:41.058 [INFO] TestAgent: started state syncer -12:41.058 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17015 network=tcp -12:41.103 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -12:41.117 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:41.117 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17014 [Candidate]" term=2 -12:41.118 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:41.118 [DEBUG] TestAgent.server.raft: vote granted: from=eb2bf899-e5f7-814f-a40d-2f73de031663 term=2 tally=1 -12:41.118 [INFO] TestAgent.server.raft: election won: tally=1 -12:41.118 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17014 [Leader]" -12:41.118 [INFO] TestAgent.server: cluster leadership acquired -12:41.118 [INFO] TestAgent.server: initializing acls -12:41.118 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:41.118 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:41.119 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:41.119 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:41.119 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:41.119 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:41.119 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-eb2bf899-e5f7-814f-a40d-2f73de031663 -12:41.119 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-eb2bf899-e5f7-814f-a40d-2f73de031663.dc1 -12:41.120 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:41.122 [INFO] TestAgent.server: New leader elected: payload=Node-eb2bf899-e5f7-814f-a40d-2f73de031663 -12:41.122 [INFO] TestAgent.server: Updating LAN server: server="Node-eb2bf899-e5f7-814f-a40d-2f73de031663 (Addr: tcp/127.0.0.1:17014) (DC: dc1)" -12:41.122 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-eb2bf899-e5f7-814f-a40d-2f73de031663.dc1 area=wan -12:41.126 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:41.126 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:41.126 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:41.126 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:41.126 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:41.126 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eb2bf899-e5f7-814f-a40d-2f73de031663 -12:41.126 [INFO] TestAgent.server: member joined, marking health alive: member=Node-eb2bf899-e5f7-814f-a40d-2f73de031663 -12:41.126 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eb2bf899-e5f7-814f-a40d-2f73de031663 -12:41.127 [INFO] TestAgent.server: federation state anti-entropy synced -12:41.142 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-eb2bf899-e5f7-814f-a40d-2f73de031663 -12:41.150 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/query from=127.0.0.1:50936 latency=711.365µs -12:41.158 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/query from=127.0.0.1:50938 error="method PUT not allowed" -12:41.171 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/query from=127.0.0.1:50938 latency=13.650753ms -12:41.173 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/query from=127.0.0.1:50940 latency=260.602µs -12:41.174 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/query from=127.0.0.1:50942 error="method DELETE not allowed" -12:41.174 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/query from=127.0.0.1:50942 latency=164.65µs -12:41.175 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/query from=127.0.0.1:50944 error="method HEAD not allowed" -12:41.175 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/query from=127.0.0.1:50944 latency=21.32µs -12:41.175 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/query from=127.0.0.1:50944 latency=1.895µs -12:41.175 [ERROR] TestAgent.http: Request error: method=GET url=/v1/query/ from=127.0.0.1:50944 error="failed prepared query lookup: index error: UUID must be 36 characters" -12:41.175 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/query/ from=127.0.0.1:50944 latency=287.795µs -12:41.177 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/query/ from=127.0.0.1:50946 error="Prepared Query lookup failed: failed prepared query lookup: index error: UUID must be 36 characters" -12:41.179 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/query/ from=127.0.0.1:50946 latency=2.571023ms -12:41.188 [ERROR] TestAgent.http: Request error: method=POST url=/v1/query/ from=127.0.0.1:50948 error="method POST not allowed" -12:41.198 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/query/ from=127.0.0.1:50948 latency=10.017518ms -12:41.201 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/query/ from=127.0.0.1:50950 error="Prepared Query lookup failed: failed prepared query lookup: index error: UUID must be 36 characters" -12:41.202 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/query/ from=127.0.0.1:50950 latency=390.967µs -12:41.202 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/query/ from=127.0.0.1:50952 error="method HEAD not allowed" -12:41.202 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/query/ from=127.0.0.1:50952 latency=68.515µs -12:41.214 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/query/ from=127.0.0.1:50952 latency=495.84µs -12:41.215 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/query/xxx/execute from=127.0.0.1:50954 latency=486.513µs -12:41.221 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/query/xxx/execute from=127.0.0.1:50956 error="method PUT not allowed" -12:41.222 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/query/xxx/execute from=127.0.0.1:50956 latency=211.395µs -12:41.225 [ERROR] TestAgent.http: Request error: method=POST url=/v1/query/xxx/execute from=127.0.0.1:50958 error="method POST not allowed" -12:41.225 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/query/xxx/execute from=127.0.0.1:50958 latency=338.878µs -12:41.226 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/query/xxx/execute from=127.0.0.1:50960 error="method DELETE not allowed" -12:41.226 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/query/xxx/execute from=127.0.0.1:50960 latency=130.199µs -12:41.227 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/query/xxx/execute from=127.0.0.1:50962 error="method HEAD not allowed" -12:41.227 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/query/xxx/execute from=127.0.0.1:50962 latency=17.655µs -12:41.227 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/query/xxx/execute from=127.0.0.1:50962 latency=121.75µs -12:41.228 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/query/xxx/explain from=127.0.0.1:50964 latency=193.487µs -12:41.228 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/query/xxx/explain from=127.0.0.1:50966 error="method PUT not allowed" -12:41.228 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/query/xxx/explain from=127.0.0.1:50966 latency=143.654µs -12:41.229 [ERROR] TestAgent.http: Request error: method=POST url=/v1/query/xxx/explain from=127.0.0.1:50968 error="method POST not allowed" -12:41.229 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/query/xxx/explain from=127.0.0.1:50968 latency=132.249µs -12:41.234 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/query/xxx/explain from=127.0.0.1:50970 error="method DELETE not allowed" -12:41.235 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/query/xxx/explain from=127.0.0.1:50970 latency=1.349588ms -12:41.243 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/query/xxx/explain from=127.0.0.1:50972 error="method HEAD not allowed" -12:41.243 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/query/xxx/explain from=127.0.0.1:50972 latency=44.337µs -12:41.243 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/query/xxx/explain from=127.0.0.1:50972 latency=208.306µs -12:41.244 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/binding-rules from=127.0.0.1:50974 error="Permission denied" -12:41.244 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:50974 latency=203.002µs -12:41.245 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rules from=127.0.0.1:50976 error="method PUT not allowed" -12:41.245 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rules from=127.0.0.1:50976 latency=288.483µs -12:41.254 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:41.254 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/binding-rules from=127.0.0.1:50978 error="method POST not allowed" -12:41.254 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/binding-rules from=127.0.0.1:50978 latency=259.449µs -12:41.254 [INFO] TestAgent: Synced node info -12:41.254 [DEBUG] TestAgent: Node info in sync -12:41.262 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/binding-rules from=127.0.0.1:50980 error="method DELETE not allowed" -12:41.263 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rules from=127.0.0.1:50980 latency=889.136µs -12:41.264 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/binding-rules from=127.0.0.1:50982 error="method HEAD not allowed" -12:41.264 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/binding-rules from=127.0.0.1:50982 latency=23.986µs -12:41.264 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/binding-rules from=127.0.0.1:50982 latency=1.888µs -12:41.264 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/force-leave/ from=127.0.0.1:50982 error="method GET not allowed" -12:41.264 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/force-leave/ from=127.0.0.1:50982 latency=146.293µs -12:41.265 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/force-leave/ from=127.0.0.1:50984 error="Permission denied" -12:41.265 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/force-leave/ from=127.0.0.1:50984 latency=133.29µs -12:41.266 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/force-leave/ from=127.0.0.1:50986 error="method POST not allowed" -12:41.266 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/force-leave/ from=127.0.0.1:50986 latency=137.608µs -12:41.277 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/force-leave/ from=127.0.0.1:50988 error="method DELETE not allowed" -12:41.278 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/force-leave/ from=127.0.0.1:50988 latency=209.597µs -12:41.282 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/force-leave/ from=127.0.0.1:50990 error="method HEAD not allowed" -12:41.282 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/force-leave/ from=127.0.0.1:50990 latency=38.814µs -12:41.283 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/force-leave/ from=127.0.0.1:50990 latency=1.882µs -12:41.283 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/datacenters from=127.0.0.1:50990 latency=333.519µs -12:41.284 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/datacenters from=127.0.0.1:50992 error="method PUT not allowed" -12:41.284 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/datacenters from=127.0.0.1:50992 latency=205.655µs -12:41.284 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/datacenters from=127.0.0.1:50994 error="method POST not allowed" -12:41.285 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/datacenters from=127.0.0.1:50994 latency=107.541µs -12:41.285 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/datacenters from=127.0.0.1:50996 error="method DELETE not allowed" -12:41.286 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/datacenters from=127.0.0.1:50996 latency=334.685µs -12:41.290 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/datacenters from=127.0.0.1:50998 error="method HEAD not allowed" -12:41.290 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/datacenters from=127.0.0.1:50998 latency=39.287µs -12:41.291 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/datacenters from=127.0.0.1:50998 latency=2.546µs -12:41.291 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/autopilot/configuration from=127.0.0.1:50998 error="Permission denied" -12:41.291 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/autopilot/configuration from=127.0.0.1:50998 latency=252.568µs -12:41.292 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/autopilot/configuration from=127.0.0.1:51000 error="Bad request: Error parsing autopilot config: EOF" -12:41.292 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/autopilot/configuration from=127.0.0.1:51000 latency=264.076µs -12:41.298 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/autopilot/configuration from=127.0.0.1:51002 error="method POST not allowed" -12:41.298 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/autopilot/configuration from=127.0.0.1:51002 latency=222.622µs -12:41.299 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/autopilot/configuration from=127.0.0.1:51004 error="method DELETE not allowed" -12:41.299 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/autopilot/configuration from=127.0.0.1:51004 latency=93.922µs -12:41.300 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/autopilot/configuration from=127.0.0.1:51006 error="method HEAD not allowed" -12:41.300 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/autopilot/configuration from=127.0.0.1:51006 latency=45.699µs -12:41.306 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/autopilot/configuration from=127.0.0.1:51006 latency=4.347µs -12:41.306 [ERROR] TestAgent.http: Request error: method=GET url=/v1/session/create from=127.0.0.1:51006 error="method GET not allowed" -12:41.306 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/create from=127.0.0.1:51006 latency=251.454µs -12:41.307 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/session/create from=127.0.0.1:51008 error="Permission denied" -12:41.307 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:51008 latency=225.572µs -12:41.308 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/create from=127.0.0.1:51010 error="method POST not allowed" -12:41.308 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/create from=127.0.0.1:51010 latency=194.197µs -12:41.314 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/create from=127.0.0.1:51012 error="method DELETE not allowed" -12:41.314 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/create from=127.0.0.1:51012 latency=410.285µs -12:41.315 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/create from=127.0.0.1:51014 error="method HEAD not allowed" -12:41.315 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/create from=127.0.0.1:51014 latency=47.748µs -12:41.322 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/create from=127.0.0.1:51014 latency=3.451µs -12:41.322 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-eb2bf899-e5f7-814f-a40d-2f73de031663 -12:41.323 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:51014 latency=419.915µs -12:41.323 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/nodes from=127.0.0.1:51016 error="method PUT not allowed" -12:41.323 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/nodes from=127.0.0.1:51016 latency=130.162µs -12:41.324 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/nodes from=127.0.0.1:51018 error="method POST not allowed" -12:41.324 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/nodes from=127.0.0.1:51018 latency=118.444µs -12:41.325 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/nodes from=127.0.0.1:51020 error="method DELETE not allowed" -12:41.325 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/nodes from=127.0.0.1:51020 latency=196.118µs -12:41.334 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/nodes from=127.0.0.1:51022 error="method HEAD not allowed" -12:41.334 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/nodes from=127.0.0.1:51022 latency=39.389µs -12:41.334 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/nodes from=127.0.0.1:51022 latency=1.945µs -12:41.334 [ERROR] TestAgent.http: Request error: method=GET url=/v1/connect/intentions/check from=127.0.0.1:51022 error="required query parameter 'source' not set" -12:41.335 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/check from=127.0.0.1:51022 latency=229.032µs -12:41.335 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/intentions/check from=127.0.0.1:51024 error="method PUT not allowed" -12:41.335 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions/check from=127.0.0.1:51024 latency=109.276µs -12:41.336 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions/check from=127.0.0.1:51026 error="method POST not allowed" -12:41.336 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions/check from=127.0.0.1:51026 latency=168.483µs -12:41.337 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/intentions/check from=127.0.0.1:51028 error="method DELETE not allowed" -12:41.337 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions/check from=127.0.0.1:51028 latency=210.611µs -12:41.344 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/intentions/check from=127.0.0.1:51030 error="method HEAD not allowed" -12:41.344 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/intentions/check from=127.0.0.1:51030 latency=46.496µs -12:41.346 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/intentions/check from=127.0.0.1:51030 latency=4.006µs -12:41.346 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/checks/ from=127.0.0.1:51030 latency=257.569µs -12:41.347 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/checks/ from=127.0.0.1:51032 error="method PUT not allowed" -12:41.348 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/checks/ from=127.0.0.1:51032 latency=238.431µs -12:41.355 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/checks/ from=127.0.0.1:51034 error="method POST not allowed" -12:41.355 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/checks/ from=127.0.0.1:51034 latency=193.013µs -12:41.355 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/checks/ from=127.0.0.1:51036 error="method DELETE not allowed" -12:41.356 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/checks/ from=127.0.0.1:51036 latency=112.636µs -12:41.356 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/checks/ from=127.0.0.1:51038 error="method HEAD not allowed" -12:41.356 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/checks/ from=127.0.0.1:51038 latency=59.564µs -12:41.362 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/checks/ from=127.0.0.1:51038 latency=4.938µs -12:41.363 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/state/ from=127.0.0.1:51038 latency=347.383µs -12:41.363 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/state/ from=127.0.0.1:51040 error="method PUT not allowed" -12:41.363 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/state/ from=127.0.0.1:51040 latency=142.121µs -12:41.364 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/state/ from=127.0.0.1:51042 error="method POST not allowed" -12:41.364 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/state/ from=127.0.0.1:51042 latency=128.51µs -12:41.365 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/state/ from=127.0.0.1:51044 error="method DELETE not allowed" -12:41.365 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/state/ from=127.0.0.1:51044 latency=135.987µs -12:41.366 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/state/ from=127.0.0.1:51046 error="method HEAD not allowed" -12:41.366 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/state/ from=127.0.0.1:51046 latency=34.362µs -12:41.366 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/state/ from=127.0.0.1:51046 latency=2.199µs -12:41.366 [ERROR] TestAgent.http: Request error: method=GET url=/v1/coordinate/update from=127.0.0.1:51046 error="method GET not allowed" -12:41.366 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/update from=127.0.0.1:51046 latency=237.318µs -12:41.367 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/coordinate/update from=127.0.0.1:51048 latency=125.534µs -12:41.368 [ERROR] TestAgent.http: Request error: method=POST url=/v1/coordinate/update from=127.0.0.1:51050 error="method POST not allowed" -12:41.368 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/coordinate/update from=127.0.0.1:51050 latency=118.466µs -12:41.368 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/coordinate/update from=127.0.0.1:51052 error="method DELETE not allowed" -12:41.368 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/coordinate/update from=127.0.0.1:51052 latency=111.342µs -12:41.369 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/coordinate/update from=127.0.0.1:51054 error="method HEAD not allowed" -12:41.369 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/coordinate/update from=127.0.0.1:51054 latency=48.374µs -12:41.378 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/coordinate/update from=127.0.0.1:51054 latency=7.823µs -12:41.383 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/auth-methods from=127.0.0.1:51054 error="Permission denied" -12:41.383 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-methods from=127.0.0.1:51054 latency=290.073µs -12:41.384 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/auth-methods from=127.0.0.1:51056 error="method PUT not allowed" -12:41.384 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-methods from=127.0.0.1:51056 latency=133.016µs -12:41.384 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/auth-methods from=127.0.0.1:51058 error="method POST not allowed" -12:41.385 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/auth-methods from=127.0.0.1:51058 latency=284.194µs -12:41.387 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/auth-methods from=127.0.0.1:51060 error="method DELETE not allowed" -12:41.388 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-methods from=127.0.0.1:51060 latency=220.346µs -12:41.388 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/auth-methods from=127.0.0.1:51062 error="method HEAD not allowed" -12:41.388 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/auth-methods from=127.0.0.1:51062 latency=70.088µs -12:41.395 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/auth-methods from=127.0.0.1:51062 latency=4.637µs -12:41.395 [DEBUG] TestAgent: dropping node from result due to ACLs: node=Node-eb2bf899-e5f7-814f-a40d-2f73de031663 accessorID=00000000-0000-0000-0000-000000000002 -12:41.396 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members from=127.0.0.1:51062 latency=357.251µs -12:41.402 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/members from=127.0.0.1:51064 error="method PUT not allowed" -12:41.403 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/members from=127.0.0.1:51064 latency=316.108µs -12:41.403 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/members from=127.0.0.1:51066 error="method POST not allowed" -12:41.404 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/members from=127.0.0.1:51066 latency=149.85µs -12:41.404 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/members from=127.0.0.1:51068 error="method DELETE not allowed" -12:41.404 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/members from=127.0.0.1:51068 latency=115.114µs -12:41.405 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/members from=127.0.0.1:51070 error="method HEAD not allowed" -12:41.405 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/members from=127.0.0.1:51070 latency=34.731µs -12:41.410 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/members from=127.0.0.1:51070 latency=3.856µs -12:41.410 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/connect/ from=127.0.0.1:51070 latency=355.255µs -12:41.411 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/connect/ from=127.0.0.1:51072 error="method PUT not allowed" -12:41.411 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/connect/ from=127.0.0.1:51072 latency=123.374µs -12:41.412 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/connect/ from=127.0.0.1:51074 error="method POST not allowed" -12:41.412 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/connect/ from=127.0.0.1:51074 latency=97.644µs -12:41.412 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/connect/ from=127.0.0.1:51076 error="method DELETE not allowed" -12:41.412 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/connect/ from=127.0.0.1:51076 latency=97.352µs -12:41.413 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/connect/ from=127.0.0.1:51078 error="method HEAD not allowed" -12:41.413 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/connect/ from=127.0.0.1:51078 latency=18.449µs -12:41.413 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/connect/ from=127.0.0.1:51078 latency=1.808µs -12:41.414 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ from=127.0.0.1:51078 latency=671.743µs -12:41.422 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/config/ from=127.0.0.1:51080 error="method PUT not allowed" -12:41.422 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config/ from=127.0.0.1:51080 latency=237.4µs -12:41.434 [ERROR] TestAgent.http: Request error: method=POST url=/v1/config/ from=127.0.0.1:51082 error="method POST not allowed" -12:41.434 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/config/ from=127.0.0.1:51082 latency=297.063µs -12:41.438 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/config/ from=127.0.0.1:51084 latency=163.371µs -12:41.439 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/config/ from=127.0.0.1:51086 error="method HEAD not allowed" -12:41.439 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/config/ from=127.0.0.1:51086 latency=61.014µs -12:41.445 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/config/ from=127.0.0.1:51086 latency=4.618µs -12:41.454 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:51086 latency=589.023µs -12:41.462 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/intentions from=127.0.0.1:51088 error="method PUT not allowed" -12:41.462 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions from=127.0.0.1:51088 latency=228.717µs -12:41.464 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions from=127.0.0.1:51090 error="Failed to decode request body: EOF" -12:41.464 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:51090 latency=274.199µs -12:41.470 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/intentions from=127.0.0.1:51092 error="method DELETE not allowed" -12:41.471 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions from=127.0.0.1:51092 latency=209.838µs -12:41.471 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/intentions from=127.0.0.1:51094 error="method HEAD not allowed" -12:41.471 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/intentions from=127.0.0.1:51094 latency=20.696µs -12:41.471 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/intentions from=127.0.0.1:51094 latency=1.859µs -12:41.472 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/token/self from=127.0.0.1:51094 error="ACL not found" -12:41.472 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:51094 latency=197.802µs -12:41.472 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/token/self from=127.0.0.1:51096 error="method PUT not allowed" -12:41.473 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/self from=127.0.0.1:51096 latency=200.92µs -12:41.478 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/token/self from=127.0.0.1:51098 error="method POST not allowed" -12:41.478 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/token/self from=127.0.0.1:51098 latency=277.281µs -12:41.486 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/token/self from=127.0.0.1:51100 error="method DELETE not allowed" -12:41.486 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/token/self from=127.0.0.1:51100 latency=243.021µs -12:41.491 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/token/self from=127.0.0.1:51102 error="method HEAD not allowed" -12:41.491 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/token/self from=127.0.0.1:51102 latency=44.233µs -12:41.492 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/token/self from=127.0.0.1:51102 latency=2.142µs -12:41.498 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/service/maintenance/ from=127.0.0.1:51102 error="method GET not allowed" -12:41.499 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/maintenance/ from=127.0.0.1:51102 latency=1.619388ms -12:41.500 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/maintenance/ from=127.0.0.1:51104 latency=170.963µs -12:41.501 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/service/maintenance/ from=127.0.0.1:51106 error="method POST not allowed" -12:41.501 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/service/maintenance/ from=127.0.0.1:51106 latency=114.482µs -12:41.507 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/service/maintenance/ from=127.0.0.1:51108 error="method DELETE not allowed" -12:41.507 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/service/maintenance/ from=127.0.0.1:51108 latency=257.545µs -12:41.508 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/service/maintenance/ from=127.0.0.1:51110 error="method HEAD not allowed" -12:41.508 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/service/maintenance/ from=127.0.0.1:51110 latency=17.072µs -12:41.508 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/service/maintenance/ from=127.0.0.1:51110 latency=1.963µs -12:41.508 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/role/ from=127.0.0.1:51110 error="Bad request: Missing role ID" -12:41.508 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/ from=127.0.0.1:51110 latency=141.455µs -12:41.509 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/role/ from=127.0.0.1:51112 error="Bad request: Role decoding failed: EOF" -12:41.509 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/ from=127.0.0.1:51112 latency=251.171µs -12:41.515 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/role/ from=127.0.0.1:51114 error="method POST not allowed" -12:41.515 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/role/ from=127.0.0.1:51114 latency=203.246µs -12:41.517 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/role/ from=127.0.0.1:51116 error="Bad request: Missing role ID" -12:41.518 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role/ from=127.0.0.1:51116 latency=232.518µs -12:41.526 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/role/ from=127.0.0.1:51118 error="method HEAD not allowed" -12:41.526 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/role/ from=127.0.0.1:51118 latency=47.004µs -12:41.530 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/role/ from=127.0.0.1:51118 latency=4.68µs -12:41.530 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/binding-rule/ from=127.0.0.1:51118 error="Bad request: Missing binding rule ID" -12:41.530 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/ from=127.0.0.1:51118 latency=181.839µs -12:41.531 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule/ from=127.0.0.1:51120 error="Bad request: BindingRule decoding failed: EOF" -12:41.531 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/ from=127.0.0.1:51120 latency=119.414µs -12:41.531 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/binding-rule/ from=127.0.0.1:51122 error="method POST not allowed" -12:41.532 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/binding-rule/ from=127.0.0.1:51122 latency=112.89µs -12:41.532 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/binding-rule/ from=127.0.0.1:51124 error="Bad request: Missing binding rule ID" -12:41.532 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/ from=127.0.0.1:51124 latency=124.983µs -12:41.533 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/binding-rule/ from=127.0.0.1:51126 error="method HEAD not allowed" -12:41.533 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/binding-rule/ from=127.0.0.1:51126 latency=86.669µs -12:41.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/binding-rule/ from=127.0.0.1:51126 latency=5.668µs -12:41.545 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/warn/ from=127.0.0.1:51126 error="method GET not allowed" -12:41.546 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/warn/ from=127.0.0.1:51126 latency=318.724µs -12:41.558 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/check/warn/ from=127.0.0.1:51128 error="Unknown check """ -12:41.558 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/warn/ from=127.0.0.1:51128 latency=271.461µs -12:41.566 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/warn/ from=127.0.0.1:51130 error="method POST not allowed" -12:41.566 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/warn/ from=127.0.0.1:51130 latency=282.79µs -12:41.567 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/warn/ from=127.0.0.1:51132 error="method DELETE not allowed" -12:41.567 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/warn/ from=127.0.0.1:51132 latency=151.495µs -12:41.568 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/warn/ from=127.0.0.1:51134 error="method HEAD not allowed" -12:41.568 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/warn/ from=127.0.0.1:51134 latency=23.36µs -12:41.568 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/warn/ from=127.0.0.1:51134 latency=1.854µs -12:41.568 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/binding-rule from=127.0.0.1:51134 error="method GET not allowed" -12:41.568 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule from=127.0.0.1:51134 latency=161.814µs -12:41.569 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:51136 error="Bad request: BindingRule decoding failed: EOF" -12:41.569 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:51136 latency=180.593µs -12:41.576 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/binding-rule from=127.0.0.1:51138 error="method POST not allowed" -12:41.576 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/binding-rule from=127.0.0.1:51138 latency=149.706µs -12:41.576 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/binding-rule from=127.0.0.1:51140 error="method DELETE not allowed" -12:41.576 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule from=127.0.0.1:51140 latency=103.368µs -12:41.577 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/binding-rule from=127.0.0.1:51142 error="method HEAD not allowed" -12:41.577 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/binding-rule from=127.0.0.1:51142 latency=15.954µs -12:41.577 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/binding-rule from=127.0.0.1:51142 latency=1.541µs -12:41.577 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/service/deregister/ from=127.0.0.1:51142 error="method GET not allowed" -12:41.577 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/deregister/ from=127.0.0.1:51142 latency=149.501µs -12:41.586 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/service/deregister/ from=127.0.0.1:51144 error="Unknown service {"" {}}" -12:41.586 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/ from=127.0.0.1:51144 latency=310.67µs -12:41.587 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/service/deregister/ from=127.0.0.1:51146 error="method POST not allowed" -12:41.587 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/service/deregister/ from=127.0.0.1:51146 latency=129.732µs -12:41.588 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/service/deregister/ from=127.0.0.1:51148 error="method DELETE not allowed" -12:41.588 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/service/deregister/ from=127.0.0.1:51148 latency=115.388µs -12:41.589 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/service/deregister/ from=127.0.0.1:51150 error="method HEAD not allowed" -12:41.589 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/service/deregister/ from=127.0.0.1:51150 latency=38.172µs -12:41.603 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/service/deregister/ from=127.0.0.1:51150 latency=3.049µs -12:41.603 [ERROR] TestAgent.http: Request error: method=GET url=/v1/session/renew/ from=127.0.0.1:51150 error="method GET not allowed" -12:41.603 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/renew/ from=127.0.0.1:51150 latency=163.018µs -12:41.604 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/renew/ from=127.0.0.1:51152 latency=98.467µs -12:41.605 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/renew/ from=127.0.0.1:51154 error="method POST not allowed" -12:41.605 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/renew/ from=127.0.0.1:51154 latency=239.151µs -12:41.610 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/renew/ from=127.0.0.1:51156 error="method DELETE not allowed" -12:41.611 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/renew/ from=127.0.0.1:51156 latency=235.087µs -12:41.611 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/renew/ from=127.0.0.1:51158 error="method HEAD not allowed" -12:41.611 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/renew/ from=127.0.0.1:51158 latency=21.873µs -12:41.611 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/renew/ from=127.0.0.1:51158 latency=1.643µs -12:41.612 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/info/ from=127.0.0.1:51158 latency=118.299µs -12:41.612 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/info/ from=127.0.0.1:51160 error="method PUT not allowed" -12:41.612 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/info/ from=127.0.0.1:51160 latency=127.43µs -12:41.613 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/info/ from=127.0.0.1:51162 error="method POST not allowed" -12:41.613 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/info/ from=127.0.0.1:51162 latency=127.724µs -12:41.618 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/info/ from=127.0.0.1:51164 error="method DELETE not allowed" -12:41.618 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/info/ from=127.0.0.1:51164 latency=267.774µs -12:41.619 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/info/ from=127.0.0.1:51166 error="method HEAD not allowed" -12:41.619 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/info/ from=127.0.0.1:51166 latency=17.778µs -12:41.619 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/info/ from=127.0.0.1:51166 latency=1.924µs -12:41.619 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/token/ from=127.0.0.1:51166 error="method GET not allowed" -12:41.619 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/token/ from=127.0.0.1:51166 latency=105.147µs -12:41.620 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/token/ from=127.0.0.1:51168 error="Permission denied" -12:41.620 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/ from=127.0.0.1:51168 latency=242.973µs -12:41.621 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/token/ from=127.0.0.1:51170 error="method POST not allowed" -12:41.621 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/token/ from=127.0.0.1:51170 latency=100.796µs -12:41.621 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/token/ from=127.0.0.1:51172 error="method DELETE not allowed" -12:41.621 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/token/ from=127.0.0.1:51172 latency=95.813µs -12:41.626 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/token/ from=127.0.0.1:51174 error="method HEAD not allowed" -12:41.626 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/token/ from=127.0.0.1:51174 latency=41.505µs -12:41.634 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/token/ from=127.0.0.1:51174 latency=3.329µs -12:41.635 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/metrics from=127.0.0.1:51174 error="Permission denied" -12:41.635 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/metrics from=127.0.0.1:51174 latency=255.482µs -12:41.636 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/metrics from=127.0.0.1:51176 error="method PUT not allowed" -12:41.636 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/metrics from=127.0.0.1:51176 latency=112.765µs -12:41.636 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/metrics from=127.0.0.1:51178 error="method POST not allowed" -12:41.636 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/metrics from=127.0.0.1:51178 latency=123.323µs -12:41.650 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/metrics from=127.0.0.1:51180 error="method DELETE not allowed" -12:41.650 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/metrics from=127.0.0.1:51180 latency=333.49µs -12:41.651 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/metrics from=127.0.0.1:51182 error="method HEAD not allowed" -12:41.651 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/metrics from=127.0.0.1:51182 latency=24.751µs -12:41.651 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/metrics from=127.0.0.1:51182 latency=2.281µs -12:41.651 [ERROR] TestAgent.http: Request error: method=GET url=/v1/catalog/deregister from=127.0.0.1:51182 error="method GET not allowed" -12:41.652 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/deregister from=127.0.0.1:51182 latency=134.439µs -12:41.652 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/deregister from=127.0.0.1:51184 latency=115.665µs -12:41.653 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/deregister from=127.0.0.1:51186 error="method POST not allowed" -12:41.653 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/deregister from=127.0.0.1:51186 latency=230.173µs -12:41.658 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/deregister from=127.0.0.1:51188 error="method DELETE not allowed" -12:41.659 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/deregister from=127.0.0.1:51188 latency=163.764µs -12:41.659 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/deregister from=127.0.0.1:51190 error="method HEAD not allowed" -12:41.659 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/deregister from=127.0.0.1:51190 latency=20.675µs -12:41.659 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/deregister from=127.0.0.1:51190 latency=1.706µs -12:41.660 [ERROR] TestAgent.http: Request error: method=GET url=/v1/internal/federation-states from=127.0.0.1:51190 error="Permission denied" -12:41.660 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/federation-states from=127.0.0.1:51190 latency=175.234µs -12:41.660 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/federation-states from=127.0.0.1:51192 error="method PUT not allowed" -12:41.660 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/federation-states from=127.0.0.1:51192 latency=231.601µs -12:41.666 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/federation-states from=127.0.0.1:51194 error="method POST not allowed" -12:41.666 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/federation-states from=127.0.0.1:51194 latency=202.726µs -12:41.667 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/federation-states from=127.0.0.1:51196 error="method DELETE not allowed" -12:41.667 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/federation-states from=127.0.0.1:51196 latency=130.655µs -12:41.667 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/federation-states from=127.0.0.1:51198 error="method HEAD not allowed" -12:41.667 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/federation-states from=127.0.0.1:51198 latency=22.004µs -12:41.668 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/federation-states from=127.0.0.1:51198 latency=2.184µs -12:41.668 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/keyring from=127.0.0.1:51198 error="Reading keyring denied by ACLs" -12:41.668 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:51198 latency=222.357µs -12:41.669 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/keyring from=127.0.0.1:51200 error="Bad request: Request decode failed: EOF" -12:41.669 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/keyring from=127.0.0.1:51200 latency=125.281µs -12:41.674 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/keyring from=127.0.0.1:51202 error="Bad request: Request decode failed: EOF" -12:41.675 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/keyring from=127.0.0.1:51202 latency=263.251µs -12:41.675 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/keyring from=127.0.0.1:51204 error="Bad request: Request decode failed: EOF" -12:41.675 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/keyring from=127.0.0.1:51204 latency=113.384µs -12:41.676 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/keyring from=127.0.0.1:51206 error="method HEAD not allowed" -12:41.676 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/keyring from=127.0.0.1:51206 latency=19.953µs -12:41.676 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/keyring from=127.0.0.1:51206 latency=2.348µs -12:41.676 [ERROR] TestAgent.http: Request error: method=GET url=/v1/snapshot from=127.0.0.1:51206 error="Permission denied" -12:41.676 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:51206 latency=148.839µs -12:41.677 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:51208 error="Permission denied" -12:41.677 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:51208 latency=246.418µs -12:41.686 [ERROR] TestAgent.http: Request error: method=POST url=/v1/snapshot from=127.0.0.1:51210 error="method POST not allowed" -12:41.686 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/snapshot from=127.0.0.1:51210 latency=288.914µs -12:41.690 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/snapshot from=127.0.0.1:51212 error="method DELETE not allowed" -12:41.690 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/snapshot from=127.0.0.1:51212 latency=246.95µs -12:41.691 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/snapshot from=127.0.0.1:51214 error="method HEAD not allowed" -12:41.691 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/snapshot from=127.0.0.1:51214 latency=22.144µs -12:41.691 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/snapshot from=127.0.0.1:51214 latency=2.048µs -12:41.692 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/destroy/ from=127.0.0.1:51214 error="method GET not allowed" -12:41.692 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/destroy/ from=127.0.0.1:51214 latency=249.126µs -12:41.693 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/destroy/ from=127.0.0.1:51216 latency=104.873µs -12:41.693 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/destroy/ from=127.0.0.1:51218 error="method POST not allowed" -12:41.693 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/destroy/ from=127.0.0.1:51218 latency=116.656µs -12:41.703 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/destroy/ from=127.0.0.1:51220 error="method DELETE not allowed" -12:41.703 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/destroy/ from=127.0.0.1:51220 latency=205.875µs -12:41.704 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/destroy/ from=127.0.0.1:51222 error="method HEAD not allowed" -12:41.704 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/destroy/ from=127.0.0.1:51222 latency=23.098µs -12:41.704 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/destroy/ from=127.0.0.1:51222 latency=1.99µs -12:41.704 [ERROR] TestAgent.http: Request error: method=GET url=/v1/catalog/register from=127.0.0.1:51222 error="method GET not allowed" -12:41.704 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/register from=127.0.0.1:51222 latency=132.719µs -12:41.705 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/register from=127.0.0.1:51224 latency=301.29µs -12:41.710 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/register from=127.0.0.1:51226 error="method POST not allowed" -12:41.711 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/register from=127.0.0.1:51226 latency=213.043µs -12:41.711 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/register from=127.0.0.1:51228 error="method DELETE not allowed" -12:41.711 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/register from=127.0.0.1:51228 latency=115.417µs -12:41.712 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/register from=127.0.0.1:51230 error="method HEAD not allowed" -12:41.712 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/register from=127.0.0.1:51230 latency=24.274µs -12:41.712 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/register from=127.0.0.1:51230 latency=2.041µs -12:41.713 [DEBUG] TestAgent.acl: dropping service from result due to ACLs: service=consul -12:41.713 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/services from=127.0.0.1:51230 latency=266.774µs -12:41.714 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/services from=127.0.0.1:51232 error="method PUT not allowed" -12:41.714 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/services from=127.0.0.1:51232 latency=257.465µs -12:41.722 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/services from=127.0.0.1:51234 error="method POST not allowed" -12:41.722 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/services from=127.0.0.1:51234 latency=241.451µs -12:41.723 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/services from=127.0.0.1:51236 error="method DELETE not allowed" -12:41.723 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/services from=127.0.0.1:51236 latency=108.427µs -12:41.723 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/services from=127.0.0.1:51238 error="method HEAD not allowed" -12:41.723 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/services from=127.0.0.1:51238 latency=26.314µs -12:41.724 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/services from=127.0.0.1:51238 latency=1.843µs -12:41.724 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/event/list from=127.0.0.1:51238 latency=294.831µs -12:41.725 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/event/list from=127.0.0.1:51240 error="method PUT not allowed" -12:41.725 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/list from=127.0.0.1:51240 latency=223.462µs -12:41.735 [ERROR] TestAgent.http: Request error: method=POST url=/v1/event/list from=127.0.0.1:51242 error="method POST not allowed" -12:41.735 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/event/list from=127.0.0.1:51242 latency=231.95µs -12:41.736 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/event/list from=127.0.0.1:51244 error="method DELETE not allowed" -12:41.736 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/event/list from=127.0.0.1:51244 latency=147.302µs -12:41.736 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/event/list from=127.0.0.1:51246 error="method HEAD not allowed" -12:41.737 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/event/list from=127.0.0.1:51246 latency=21.016µs -12:41.737 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/event/list from=127.0.0.1:51246 latency=2.147µs -12:41.737 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/connect/ from=127.0.0.1:51246 latency=121.026µs -12:41.746 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/connect/ from=127.0.0.1:51248 error="method PUT not allowed" -12:41.746 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/connect/ from=127.0.0.1:51248 latency=188.816µs -12:41.747 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/connect/ from=127.0.0.1:51250 error="method POST not allowed" -12:41.747 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/connect/ from=127.0.0.1:51250 latency=107.458µs -12:41.747 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/connect/ from=127.0.0.1:51252 error="method DELETE not allowed" -12:41.747 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/connect/ from=127.0.0.1:51252 latency=104.4µs -12:41.748 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/connect/ from=127.0.0.1:51254 error="method HEAD not allowed" -12:41.748 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/connect/ from=127.0.0.1:51254 latency=51.012µs -12:41.758 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/connect/ from=127.0.0.1:51254 latency=4.062µs -12:41.758 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/health/service/id/ from=127.0.0.1:51254 error="Bad request: Missing serviceID" -12:41.758 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/health/service/id/ from=127.0.0.1:51254 latency=209.335µs -12:41.759 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/health/service/id/ from=127.0.0.1:51256 error="method PUT not allowed" -12:41.759 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/health/service/id/ from=127.0.0.1:51256 latency=115.097µs -12:41.760 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/health/service/id/ from=127.0.0.1:51258 error="method POST not allowed" -12:41.760 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/health/service/id/ from=127.0.0.1:51258 latency=112.047µs -12:41.760 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/health/service/id/ from=127.0.0.1:51260 error="method DELETE not allowed" -12:41.760 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/health/service/id/ from=127.0.0.1:51260 latency=114.163µs -12:41.761 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/health/service/id/ from=127.0.0.1:51262 error="method HEAD not allowed" -12:41.761 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/health/service/id/ from=127.0.0.1:51262 latency=22.432µs -12:41.766 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/health/service/id/ from=127.0.0.1:51262 latency=3.617µs -12:41.766 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/roles from=127.0.0.1:51262 error="Permission denied" -12:41.766 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:51262 latency=241.709µs -12:41.767 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/roles from=127.0.0.1:51264 error="method PUT not allowed" -12:41.767 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/roles from=127.0.0.1:51264 latency=96.528µs -12:41.767 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/roles from=127.0.0.1:51266 error="method POST not allowed" -12:41.767 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/roles from=127.0.0.1:51266 latency=91.417µs -12:41.768 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/roles from=127.0.0.1:51268 error="method DELETE not allowed" -12:41.768 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/roles from=127.0.0.1:51268 latency=90.46µs -12:41.768 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/roles from=127.0.0.1:51270 error="method HEAD not allowed" -12:41.768 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/roles from=127.0.0.1:51270 latency=15.677µs -12:41.769 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/roles from=127.0.0.1:51270 latency=1.475µs -12:41.769 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/node/ from=127.0.0.1:51270 latency=87.108µs -12:41.770 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/node/ from=127.0.0.1:51272 error="method PUT not allowed" -12:41.770 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/node/ from=127.0.0.1:51272 latency=313.421µs -12:41.777 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/node/ from=127.0.0.1:51274 error="method POST not allowed" -12:41.777 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/node/ from=127.0.0.1:51274 latency=245.572µs -12:41.781 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/node/ from=127.0.0.1:51276 error="method DELETE not allowed" -12:41.781 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/node/ from=127.0.0.1:51276 latency=175.012µs -12:41.781 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/node/ from=127.0.0.1:51278 error="method HEAD not allowed" -12:41.781 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/node/ from=127.0.0.1:51278 latency=20.771µs -12:41.786 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/node/ from=127.0.0.1:51278 latency=4.207µs -12:41.786 [ERROR] TestAgent.http: Request error: method=GET url=/v1/txn from=127.0.0.1:51278 error="method GET not allowed" -12:41.786 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/txn from=127.0.0.1:51278 latency=186.632µs -12:41.787 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/txn from=127.0.0.1:51280 latency=111.901µs -12:41.787 [ERROR] TestAgent.http: Request error: method=POST url=/v1/txn from=127.0.0.1:51282 error="method POST not allowed" -12:41.788 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/txn from=127.0.0.1:51282 latency=111.507µs -12:41.788 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/txn from=127.0.0.1:51284 error="method DELETE not allowed" -12:41.788 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/txn from=127.0.0.1:51284 latency=108.029µs -12:41.789 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/txn from=127.0.0.1:51286 error="method HEAD not allowed" -12:41.789 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/txn from=127.0.0.1:51286 latency=18.185µs -12:41.789 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/txn from=127.0.0.1:51286 latency=1.436µs -12:41.794 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/datacenters from=127.0.0.1:51286 latency=4.486203ms -12:41.798 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/coordinate/datacenters from=127.0.0.1:51288 error="method PUT not allowed" -12:41.798 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/coordinate/datacenters from=127.0.0.1:51288 latency=205.939µs -12:41.806 [ERROR] TestAgent.http: Request error: method=POST url=/v1/coordinate/datacenters from=127.0.0.1:51290 error="method POST not allowed" -12:41.806 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/coordinate/datacenters from=127.0.0.1:51290 latency=248.795µs -12:41.807 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/coordinate/datacenters from=127.0.0.1:51292 error="method DELETE not allowed" -12:41.807 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/coordinate/datacenters from=127.0.0.1:51292 latency=114.224µs -12:41.808 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/coordinate/datacenters from=127.0.0.1:51294 error="method HEAD not allowed" -12:41.808 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/coordinate/datacenters from=127.0.0.1:51294 latency=18.048µs -12:41.808 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/coordinate/datacenters from=127.0.0.1:51294 latency=3.218µs -12:41.808 [ERROR] TestAgent.http: Request error: method=GET url=/v1/internal/federation-state/ from=127.0.0.1:51294 error="Bad request: Missing datacenter name" -12:41.808 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/federation-state/ from=127.0.0.1:51294 latency=112.323µs -12:41.809 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/federation-state/ from=127.0.0.1:51296 error="method PUT not allowed" -12:41.809 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/federation-state/ from=127.0.0.1:51296 latency=105.22µs -12:41.809 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/federation-state/ from=127.0.0.1:51298 error="method POST not allowed" -12:41.810 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/federation-state/ from=127.0.0.1:51298 latency=394.293µs -12:41.819 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/federation-state/ from=127.0.0.1:51300 error="method DELETE not allowed" -12:41.819 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/federation-state/ from=127.0.0.1:51300 latency=205.636µs -12:41.820 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/federation-state/ from=127.0.0.1:51302 error="method HEAD not allowed" -12:41.820 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/federation-state/ from=127.0.0.1:51302 latency=21.298µs -12:41.820 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/federation-state/ from=127.0.0.1:51302 latency=1.964µs -12:41.821 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/logout from=127.0.0.1:51302 error="method GET not allowed" -12:41.821 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/logout from=127.0.0.1:51302 latency=125.164µs -12:41.821 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/logout from=127.0.0.1:51304 error="method PUT not allowed" -12:41.821 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/logout from=127.0.0.1:51304 latency=268.993µs -12:41.823 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:51306 error="ACL not found" -12:41.824 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:51306 latency=196.645µs -12:41.824 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/logout from=127.0.0.1:51308 error="method DELETE not allowed" -12:41.824 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/logout from=127.0.0.1:51308 latency=170.702µs -12:41.827 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/logout from=127.0.0.1:51310 error="method HEAD not allowed" -12:41.827 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/logout from=127.0.0.1:51310 latency=32.232µs -12:41.827 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/logout from=127.0.0.1:51310 latency=1.946µs -12:41.827 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/register from=127.0.0.1:51310 error="method GET not allowed" -12:41.827 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/register from=127.0.0.1:51310 latency=180.743µs -12:41.828 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/register from=127.0.0.1:51312 latency=112.659µs -12:41.828 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/register from=127.0.0.1:51314 error="method POST not allowed" -12:41.829 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/register from=127.0.0.1:51314 latency=106.352µs -12:41.829 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/register from=127.0.0.1:51316 error="method DELETE not allowed" -12:41.829 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/register from=127.0.0.1:51316 latency=161.186µs -12:41.837 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/register from=127.0.0.1:51318 error="method HEAD not allowed" -12:41.837 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/register from=127.0.0.1:51318 latency=44.318µs -12:41.838 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/register from=127.0.0.1:51318 latency=2.049µs -12:41.838 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/pass/ from=127.0.0.1:51318 error="method GET not allowed" -12:41.838 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/pass/ from=127.0.0.1:51318 latency=181.883µs -12:41.839 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/check/pass/ from=127.0.0.1:51320 error="Unknown check """ -12:41.839 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/pass/ from=127.0.0.1:51320 latency=161.008µs -12:41.839 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/pass/ from=127.0.0.1:51322 error="method POST not allowed" -12:41.840 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/pass/ from=127.0.0.1:51322 latency=117.512µs -12:41.840 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/pass/ from=127.0.0.1:51324 error="method DELETE not allowed" -12:41.840 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/pass/ from=127.0.0.1:51324 latency=177.453µs -12:41.841 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/pass/ from=127.0.0.1:51326 error="method HEAD not allowed" -12:41.841 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/pass/ from=127.0.0.1:51326 latency=23.184µs -12:41.841 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/pass/ from=127.0.0.1:51326 latency=1.922µs -12:41.846 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/gateway-services/ from=127.0.0.1:51326 latency=277.096µs -12:41.854 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/gateway-services/ from=127.0.0.1:51328 error="method PUT not allowed" -12:41.854 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/gateway-services/ from=127.0.0.1:51328 latency=191.427µs -12:41.855 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/gateway-services/ from=127.0.0.1:51330 error="method POST not allowed" -12:41.855 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/gateway-services/ from=127.0.0.1:51330 latency=93.958µs -12:41.856 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/gateway-services/ from=127.0.0.1:51332 error="method DELETE not allowed" -12:41.856 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/gateway-services/ from=127.0.0.1:51332 latency=91.16µs -12:41.856 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/gateway-services/ from=127.0.0.1:51334 error="method HEAD not allowed" -12:41.856 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/gateway-services/ from=127.0.0.1:51334 latency=18.019µs -12:41.866 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/gateway-services/ from=127.0.0.1:51334 latency=5.381µs -12:41.866 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policy/ from=127.0.0.1:51334 error="Bad request: Missing policy ID" -12:41.866 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/ from=127.0.0.1:51334 latency=249.356µs -12:41.867 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/policy/ from=127.0.0.1:51344 error="Bad request: Policy decoding failed: EOF" -12:41.867 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy/ from=127.0.0.1:51344 latency=137.683µs -12:41.868 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/policy/ from=127.0.0.1:51346 error="method POST not allowed" -12:41.868 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/policy/ from=127.0.0.1:51346 latency=210.246µs -12:41.874 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/policy/ from=127.0.0.1:51348 error="Bad request: Missing policy ID" -12:41.875 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policy/ from=127.0.0.1:51348 latency=291.904µs -12:41.882 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/policy/ from=127.0.0.1:51350 error="method HEAD not allowed" -12:41.882 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/policy/ from=127.0.0.1:51350 latency=43.752µs -12:41.882 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/policy/ from=127.0.0.1:51350 latency=2.328µs -12:41.882 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/token from=127.0.0.1:51350 error="method GET not allowed" -12:41.883 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token from=127.0.0.1:51350 latency=229.439µs -12:41.883 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/token from=127.0.0.1:51352 error="Bad request: Token decoding failed: EOF" -12:41.883 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:51352 latency=166.244µs -12:41.884 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/token from=127.0.0.1:51354 error="method POST not allowed" -12:41.884 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/token from=127.0.0.1:51354 latency=176.072µs -12:41.895 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/token from=127.0.0.1:51356 error="method DELETE not allowed" -12:41.895 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/token from=127.0.0.1:51356 latency=224.978µs -12:41.895 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/token from=127.0.0.1:51358 error="method HEAD not allowed" -12:41.896 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/token from=127.0.0.1:51358 latency=18.297µs -12:41.896 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/token from=127.0.0.1:51358 latency=1.636µs -12:41.896 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/leave from=127.0.0.1:51358 error="method GET not allowed" -12:41.896 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/leave from=127.0.0.1:51358 latency=116.964µs -12:41.897 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/leave from=127.0.0.1:51360 error="Permission denied" -12:41.897 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/leave from=127.0.0.1:51360 latency=160.379µs -12:41.897 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/leave from=127.0.0.1:51362 error="method POST not allowed" -12:41.897 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/leave from=127.0.0.1:51362 latency=107.286µs -12:41.906 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/leave from=127.0.0.1:51364 error="method DELETE not allowed" -12:41.906 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/leave from=127.0.0.1:51364 latency=240.878µs -12:41.911 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/leave from=127.0.0.1:51366 error="method HEAD not allowed" -12:41.911 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/leave from=127.0.0.1:51366 latency=51.866µs -12:41.911 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/leave from=127.0.0.1:51366 latency=2.63µs -12:41.911 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-eb2bf899-e5f7-814f-a40d-2f73de031663 -12:41.912 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/ui/services from=127.0.0.1:51366 latency=572.363µs -12:41.912 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/ui/services from=127.0.0.1:51372 error="method PUT not allowed" -12:41.913 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/ui/services from=127.0.0.1:51372 latency=278.931µs -12:41.914 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/ui/services from=127.0.0.1:51374 error="method POST not allowed" -12:41.915 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/ui/services from=127.0.0.1:51374 latency=267.051µs -12:41.915 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/ui/services from=127.0.0.1:51376 error="method DELETE not allowed" -12:41.915 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/ui/services from=127.0.0.1:51376 latency=120.761µs -12:41.916 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/ui/services from=127.0.0.1:51378 error="method HEAD not allowed" -12:41.916 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/ui/services from=127.0.0.1:51378 latency=26.135µs -12:41.916 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/ui/services from=127.0.0.1:51378 latency=1.858µs -12:41.916 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/login from=127.0.0.1:51378 error="method GET not allowed" -12:41.917 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/login from=127.0.0.1:51378 latency=127.891µs -12:41.917 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/login from=127.0.0.1:51380 error="method PUT not allowed" -12:41.918 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/login from=127.0.0.1:51380 latency=266.121µs -12:41.918 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:41.918 [DEBUG] TestAgent: Node info in sync -12:41.924 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/login from=127.0.0.1:51386 error="Bad request: Failed to decode request body: EOF" -12:41.924 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:51386 latency=199.303µs -12:41.925 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/login from=127.0.0.1:51388 error="method DELETE not allowed" -12:41.925 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/login from=127.0.0.1:51388 latency=252.616µs -12:41.930 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/login from=127.0.0.1:51390 error="method HEAD not allowed" -12:41.930 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/login from=127.0.0.1:51390 latency=39.403µs -12:41.931 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/login from=127.0.0.1:51390 latency=2.066µs -12:41.931 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/auth-method/ from=127.0.0.1:51390 error="Bad request: Missing auth method name" -12:41.931 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/ from=127.0.0.1:51390 latency=174.399µs -12:41.932 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/auth-method/ from=127.0.0.1:51392 error="Bad request: AuthMethod decoding failed: EOF" -12:41.932 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/ from=127.0.0.1:51392 latency=191.791µs -12:41.938 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/auth-method/ from=127.0.0.1:51394 error="method POST not allowed" -12:41.938 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/auth-method/ from=127.0.0.1:51394 latency=239.423µs -12:41.946 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/auth-method/ from=127.0.0.1:51396 error="Bad request: Missing auth method name" -12:41.946 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-method/ from=127.0.0.1:51396 latency=217.933µs -12:41.947 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/auth-method/ from=127.0.0.1:51398 error="method HEAD not allowed" -12:41.947 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/auth-method/ from=127.0.0.1:51398 latency=48.877µs -12:41.954 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/auth-method/ from=127.0.0.1:51398 latency=3.528µs -12:41.954 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/self from=127.0.0.1:51398 error="Permission denied" -12:41.954 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:51398 latency=299.256µs -12:41.955 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/self from=127.0.0.1:51400 error="method PUT not allowed" -12:41.956 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/self from=127.0.0.1:51400 latency=243.256µs -12:41.958 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/self from=127.0.0.1:51402 error="method POST not allowed" -12:41.959 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/self from=127.0.0.1:51402 latency=245.53µs -12:41.959 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/self from=127.0.0.1:51404 error="method DELETE not allowed" -12:41.960 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/self from=127.0.0.1:51404 latency=121.499µs -12:41.960 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/self from=127.0.0.1:51406 error="method HEAD not allowed" -12:41.960 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/self from=127.0.0.1:51406 latency=44.939µs -12:41.966 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/self from=127.0.0.1:51406 latency=5.389µs -12:41.967 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:51406 latency=240.136µs -12:41.968 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:51408 error="method PUT not allowed" -12:41.968 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:51408 latency=213.203µs -12:41.979 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:51410 error="method POST not allowed" -12:41.980 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:51410 latency=269.647µs -12:41.980 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:51412 error="method DELETE not allowed" -12:41.980 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:51412 latency=235.609µs -12:41.981 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:51414 error="method HEAD not allowed" -12:41.981 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:51414 latency=18.119µs -12:41.982 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:51414 latency=5.231µs -12:41.986 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/raft/configuration from=127.0.0.1:51414 error="Permission denied" -12:41.986 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/raft/configuration from=127.0.0.1:51414 latency=306.099µs -12:41.994 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/raft/configuration from=127.0.0.1:51424 error="method PUT not allowed" -12:41.994 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/raft/configuration from=127.0.0.1:51424 latency=169.292µs -12:41.996 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/raft/configuration from=127.0.0.1:51426 error="method POST not allowed" -12:41.996 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/raft/configuration from=127.0.0.1:51426 latency=144.394µs -12:41.996 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/raft/configuration from=127.0.0.1:51428 error="method DELETE not allowed" -12:41.997 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/raft/configuration from=127.0.0.1:51428 latency=92.038µs -12:41.997 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/raft/configuration from=127.0.0.1:51430 error="method HEAD not allowed" -12:41.997 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/raft/configuration from=127.0.0.1:51430 latency=15.81µs -12:42.001 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/raft/configuration from=127.0.0.1:51430 latency=3.561µs -12:42.006 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/clone/ from=127.0.0.1:51430 error="method GET not allowed" -12:42.006 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/clone/ from=127.0.0.1:51430 latency=321.927µs -12:42.007 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/clone/ from=127.0.0.1:51432 latency=227.588µs -12:42.013 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/clone/ from=127.0.0.1:51434 error="method POST not allowed" -12:42.014 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/clone/ from=127.0.0.1:51434 latency=230.238µs -12:42.018 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/clone/ from=127.0.0.1:51436 error="method DELETE not allowed" -12:42.019 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/clone/ from=127.0.0.1:51436 latency=325.921µs -12:42.019 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/clone/ from=127.0.0.1:51438 error="method HEAD not allowed" -12:42.019 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/clone/ from=127.0.0.1:51438 latency=26.311µs -12:42.020 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/clone/ from=127.0.0.1:51438 latency=1.514µs -12:42.020 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/maintenance from=127.0.0.1:51438 error="method GET not allowed" -12:42.020 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/maintenance from=127.0.0.1:51438 latency=115.727µs -12:42.030 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/maintenance from=127.0.0.1:51440 latency=213.276µs -12:42.031 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/maintenance from=127.0.0.1:51442 error="method POST not allowed" -12:42.031 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/maintenance from=127.0.0.1:51442 latency=135.299µs -12:42.031 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/maintenance from=127.0.0.1:51444 error="method DELETE not allowed" -12:42.032 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/maintenance from=127.0.0.1:51444 latency=293.998µs -12:42.038 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/maintenance from=127.0.0.1:51446 error="method HEAD not allowed" -12:42.038 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/maintenance from=127.0.0.1:51446 latency=37.43µs -12:42.039 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/maintenance from=127.0.0.1:51446 latency=5.995µs -12:42.039 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/ from=127.0.0.1:51446 latency=219.01µs -12:42.040 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/service/ from=127.0.0.1:51448 error="method PUT not allowed" -12:42.040 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/service/ from=127.0.0.1:51448 latency=227.849µs -12:42.041 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/service/ from=127.0.0.1:51450 error="method POST not allowed" -12:42.041 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/service/ from=127.0.0.1:51450 latency=103.538µs -12:42.041 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/service/ from=127.0.0.1:51452 error="method DELETE not allowed" -12:42.041 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/service/ from=127.0.0.1:51452 latency=100.825µs -12:42.050 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/service/ from=127.0.0.1:51454 error="method HEAD not allowed" -12:42.050 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/service/ from=127.0.0.1:51454 latency=77.515µs -12:42.050 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/service/ from=127.0.0.1:51454 latency=2.215µs -12:42.050 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/create from=127.0.0.1:51454 error="method GET not allowed" -12:42.051 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/create from=127.0.0.1:51454 latency=157.75µs -12:42.051 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/create from=127.0.0.1:51456 error="Permission denied" -12:42.051 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/create from=127.0.0.1:51456 latency=213.475µs -12:42.052 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/create from=127.0.0.1:51458 error="method POST not allowed" -12:42.052 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/create from=127.0.0.1:51458 latency=104.59µs -12:42.053 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/create from=127.0.0.1:51460 error="method DELETE not allowed" -12:42.053 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/create from=127.0.0.1:51460 latency=340.052µs -12:42.058 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/create from=127.0.0.1:51462 error="method HEAD not allowed" -12:42.058 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/create from=127.0.0.1:51462 latency=48.722µs -12:42.059 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/create from=127.0.0.1:51462 latency=1.976µs -12:42.059 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/fail/ from=127.0.0.1:51462 error="method GET not allowed" -12:42.059 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/fail/ from=127.0.0.1:51462 latency=317.18µs -12:42.060 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/check/fail/ from=127.0.0.1:51464 error="Unknown check """ -12:42.060 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/fail/ from=127.0.0.1:51464 latency=173.14µs -12:42.061 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/fail/ from=127.0.0.1:51466 error="method POST not allowed" -12:42.061 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/fail/ from=127.0.0.1:51466 latency=304.049µs -12:42.066 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/fail/ from=127.0.0.1:51468 error="method DELETE not allowed" -12:42.066 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/fail/ from=127.0.0.1:51468 latency=217.973µs -12:42.067 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/fail/ from=127.0.0.1:51470 error="method HEAD not allowed" -12:42.067 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/fail/ from=127.0.0.1:51470 latency=19.19µs -12:42.067 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/fail/ from=127.0.0.1:51470 latency=1.83µs -12:42.067 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/update/ from=127.0.0.1:51470 error="method GET not allowed" -12:42.068 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/update/ from=127.0.0.1:51470 latency=118.432µs -12:42.068 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/update/ from=127.0.0.1:51472 latency=230.068µs -12:42.078 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/update/ from=127.0.0.1:51474 error="method POST not allowed" -12:42.078 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/update/ from=127.0.0.1:51474 latency=200.011µs -12:42.079 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/update/ from=127.0.0.1:51476 error="method DELETE not allowed" -12:42.079 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/update/ from=127.0.0.1:51476 latency=142.345µs -12:42.080 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/update/ from=127.0.0.1:51478 error="method HEAD not allowed" -12:42.080 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/update/ from=127.0.0.1:51478 latency=21.027µs -12:42.080 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/update/ from=127.0.0.1:51478 latency=3.103µs -12:42.080 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/role from=127.0.0.1:51478 error="method GET not allowed" -12:42.080 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role from=127.0.0.1:51478 latency=129.619µs -12:42.081 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/role from=127.0.0.1:51480 error="Bad request: Role decoding failed: EOF" -12:42.081 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:51480 latency=238.98µs -12:42.086 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/role from=127.0.0.1:51482 error="method POST not allowed" -12:42.086 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/role from=127.0.0.1:51482 latency=208.463µs -12:42.090 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/role from=127.0.0.1:51484 error="method DELETE not allowed" -12:42.090 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role from=127.0.0.1:51484 latency=268.347µs -12:42.098 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/role from=127.0.0.1:51486 error="method HEAD not allowed" -12:42.098 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/role from=127.0.0.1:51486 latency=45.75µs -12:42.098 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/role from=127.0.0.1:51486 latency=2.273µs -12:42.099 [ERROR] TestAgent.http: Request error: method=GET url=/v1/coordinate/node/ from=127.0.0.1:51486 error="Permission denied" -12:42.099 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/node/ from=127.0.0.1:51486 latency=332.357µs -12:42.099 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/coordinate/node/ from=127.0.0.1:51488 error="method PUT not allowed" -12:42.099 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/coordinate/node/ from=127.0.0.1:51488 latency=109.186µs -12:42.100 [ERROR] TestAgent.http: Request error: method=POST url=/v1/coordinate/node/ from=127.0.0.1:51490 error="method POST not allowed" -12:42.100 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/coordinate/node/ from=127.0.0.1:51490 latency=224.318µs -12:42.106 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/coordinate/node/ from=127.0.0.1:51492 error="method DELETE not allowed" -12:42.107 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/coordinate/node/ from=127.0.0.1:51492 latency=268.8µs -12:42.107 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/coordinate/node/ from=127.0.0.1:51494 error="method HEAD not allowed" -12:42.107 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/coordinate/node/ from=127.0.0.1:51494 latency=17.476µs -12:42.108 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/coordinate/node/ from=127.0.0.1:51494 latency=1.886µs -12:42.108 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:51494 latency=185.178µs -12:42.109 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/services from=127.0.0.1:51496 error="method PUT not allowed" -12:42.109 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/services from=127.0.0.1:51496 latency=255.964µs -12:42.118 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/services from=127.0.0.1:51498 error="method POST not allowed" -12:42.118 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/services from=127.0.0.1:51498 latency=227.631µs -12:42.119 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/services from=127.0.0.1:51500 error="method DELETE not allowed" -12:42.119 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/services from=127.0.0.1:51500 latency=130.915µs -12:42.132 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/services from=127.0.0.1:51502 error="method HEAD not allowed" -12:42.132 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/services from=127.0.0.1:51502 latency=89.097µs -12:42.132 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/services from=127.0.0.1:51502 latency=2.986µs -12:42.133 [ERROR] TestAgent.http: Request error: method=GET url=/v1/connect/ca/configuration from=127.0.0.1:51502 error="Permission denied" -12:42.133 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/ca/configuration from=127.0.0.1:51502 latency=316.935µs -12:42.134 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/ca/configuration from=127.0.0.1:51504 error="Bad request: Request decode failed: EOF" -12:42.134 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/ca/configuration from=127.0.0.1:51504 latency=378.972µs -12:42.135 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/ca/configuration from=127.0.0.1:51506 error="method POST not allowed" -12:42.135 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/ca/configuration from=127.0.0.1:51506 latency=208.362µs -12:42.136 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/ca/configuration from=127.0.0.1:51508 error="method DELETE not allowed" -12:42.139 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/ca/configuration from=127.0.0.1:51508 latency=2.943398ms -12:42.146 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/ca/configuration from=127.0.0.1:51510 error="method HEAD not allowed" -12:42.146 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/ca/configuration from=127.0.0.1:51510 latency=66.07µs -12:42.158 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/ca/configuration from=127.0.0.1:51510 latency=4.677µs -12:42.158 [ERROR] TestAgent.http: Request error: method=GET url=/v1/connect/intentions/ from=127.0.0.1:51510 error="Bad request: failed intention lookup: index error: UUID must be 36 characters" -12:42.158 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/ from=127.0.0.1:51510 latency=298.525µs -12:42.161 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/intentions/ from=127.0.0.1:51520 error="Bad request: Request decode failed: EOF" -12:42.161 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions/ from=127.0.0.1:51520 latency=180.708µs -12:42.166 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions/ from=127.0.0.1:51522 error="method POST not allowed" -12:42.167 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions/ from=127.0.0.1:51522 latency=208.666µs -12:42.167 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/intentions/ from=127.0.0.1:51524 error="Intention lookup failed: failed intention lookup: index error: UUID must be 36 characters" -12:42.167 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions/ from=127.0.0.1:51524 latency=266.454µs -12:42.168 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/intentions/ from=127.0.0.1:51526 error="method HEAD not allowed" -12:42.168 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/intentions/ from=127.0.0.1:51526 latency=29.17µs -12:42.168 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/intentions/ from=127.0.0.1:51526 latency=2.439µs -12:42.169 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/ from=127.0.0.1:51526 latency=123.34µs -12:42.169 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/session/info/ from=127.0.0.1:51528 error="method PUT not allowed" -12:42.170 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/info/ from=127.0.0.1:51528 latency=1.003165ms -12:42.186 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/info/ from=127.0.0.1:51530 error="method POST not allowed" -12:42.187 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/info/ from=127.0.0.1:51530 latency=446.226µs -12:42.188 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/info/ from=127.0.0.1:51532 error="method DELETE not allowed" -12:42.189 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/info/ from=127.0.0.1:51532 latency=1.152022ms -12:42.190 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/info/ from=127.0.0.1:51534 error="method HEAD not allowed" -12:42.190 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/info/ from=127.0.0.1:51534 latency=31.952µs -12:42.190 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/info/ from=127.0.0.1:51534 latency=2.191µs -12:42.191 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/role/name/ from=127.0.0.1:51534 error="Bad request: Missing role Name" -12:42.191 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/name/ from=127.0.0.1:51534 latency=241.327µs -12:42.192 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/role/name/ from=127.0.0.1:51536 error="method PUT not allowed" -12:42.192 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/name/ from=127.0.0.1:51536 latency=262.952µs -12:42.194 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/role/name/ from=127.0.0.1:51538 error="method POST not allowed" -12:42.194 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/role/name/ from=127.0.0.1:51538 latency=198.99µs -12:42.195 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/role/name/ from=127.0.0.1:51540 error="method DELETE not allowed" -12:42.196 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role/name/ from=127.0.0.1:51540 latency=224.504µs -12:42.196 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/role/name/ from=127.0.0.1:51542 error="method HEAD not allowed" -12:42.196 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/role/name/ from=127.0.0.1:51542 latency=29.005µs -12:42.202 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/role/name/ from=127.0.0.1:51542 latency=6.37µs -12:42.202 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:51542 latency=325.057µs -12:42.203 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:51544 error="method PUT not allowed" -12:42.203 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:51544 latency=134.093µs -12:42.204 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:51546 error="method POST not allowed" -12:42.204 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:51546 latency=134.295µs -12:42.210 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:51548 error="method DELETE not allowed" -12:42.211 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:51548 latency=452.338µs -12:42.211 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:51550 error="method HEAD not allowed" -12:42.211 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:51550 latency=24.333µs -12:42.212 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:51550 latency=2.178µs -12:42.212 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/ui/node/ from=127.0.0.1:51550 latency=132.949µs -12:42.212 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/ui/node/ from=127.0.0.1:51552 error="method PUT not allowed" -12:42.213 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/ui/node/ from=127.0.0.1:51552 latency=120.828µs -12:42.213 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/ui/node/ from=127.0.0.1:51554 error="method POST not allowed" -12:42.213 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/ui/node/ from=127.0.0.1:51554 latency=116.474µs -12:42.214 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/ui/node/ from=127.0.0.1:51556 error="method DELETE not allowed" -12:42.215 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/ui/node/ from=127.0.0.1:51556 latency=157.948µs -12:42.221 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/ui/node/ from=127.0.0.1:51558 error="method HEAD not allowed" -12:42.221 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/ui/node/ from=127.0.0.1:51558 latency=42.619µs -12:42.226 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/ui/node/ from=127.0.0.1:51558 latency=4.813µs -12:42.231 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policies from=127.0.0.1:51558 error="Permission denied" -12:42.231 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policies from=127.0.0.1:51558 latency=339.429µs -12:42.232 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/policies from=127.0.0.1:51560 error="method PUT not allowed" -12:42.232 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policies from=127.0.0.1:51560 latency=122.954µs -12:42.233 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/policies from=127.0.0.1:51562 error="method POST not allowed" -12:42.233 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/policies from=127.0.0.1:51562 latency=117.662µs -12:42.234 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/policies from=127.0.0.1:51564 error="method DELETE not allowed" -12:42.234 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policies from=127.0.0.1:51564 latency=282.846µs -12:42.239 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/policies from=127.0.0.1:51566 error="method HEAD not allowed" -12:42.239 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/policies from=127.0.0.1:51566 latency=37.297µs -12:42.246 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/policies from=127.0.0.1:51566 latency=3.953µs -12:42.254 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/join/ from=127.0.0.1:51566 error="method GET not allowed" -12:42.254 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/join/ from=127.0.0.1:51566 latency=204.714µs -12:42.255 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/join/ from=127.0.0.1:51568 error="Permission denied" -12:42.255 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/join/ from=127.0.0.1:51568 latency=238.858µs -12:42.256 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/join/ from=127.0.0.1:51570 error="method POST not allowed" -12:42.256 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/join/ from=127.0.0.1:51570 latency=116.931µs -12:42.257 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/join/ from=127.0.0.1:51572 error="method DELETE not allowed" -12:42.257 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/join/ from=127.0.0.1:51572 latency=212.449µs -12:42.259 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/join/ from=127.0.0.1:51574 error="method HEAD not allowed" -12:42.259 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/join/ from=127.0.0.1:51574 latency=32.72µs -12:42.259 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/join/ from=127.0.0.1:51574 latency=1.793µs -12:42.259 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/update from=127.0.0.1:51574 error="method GET not allowed" -12:42.260 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/update from=127.0.0.1:51574 latency=155.96µs -12:42.260 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/update from=127.0.0.1:51576 latency=169.352µs -12:42.261 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/update from=127.0.0.1:51578 error="method POST not allowed" -12:42.261 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/update from=127.0.0.1:51578 latency=142.675µs -12:42.263 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/update from=127.0.0.1:51580 error="method DELETE not allowed" -12:42.263 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/update from=127.0.0.1:51580 latency=161.579µs -12:42.264 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/update from=127.0.0.1:51582 error="method HEAD not allowed" -12:42.264 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/update from=127.0.0.1:51582 latency=20.374µs -12:42.264 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/update from=127.0.0.1:51582 latency=1.651µs -12:42.264 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:51582 latency=189.684µs -12:42.265 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/checks from=127.0.0.1:51584 error="method PUT not allowed" -12:42.265 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/checks from=127.0.0.1:51584 latency=231.547µs -12:42.280 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/checks from=127.0.0.1:51586 error="method POST not allowed" -12:42.280 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/checks from=127.0.0.1:51586 latency=181.291µs -12:42.281 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/checks from=127.0.0.1:51588 error="method DELETE not allowed" -12:42.281 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/checks from=127.0.0.1:51588 latency=94.492µs -12:42.286 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/checks from=127.0.0.1:51590 error="method HEAD not allowed" -12:42.286 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/checks from=127.0.0.1:51590 latency=43.598µs -12:42.287 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/checks from=127.0.0.1:51590 latency=1.832µs -12:42.287 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-eb2bf899-e5f7-814f-a40d-2f73de031663 -12:42.287 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/ui/nodes from=127.0.0.1:51590 latency=475.237µs -12:42.288 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/ui/nodes from=127.0.0.1:51592 error="method PUT not allowed" -12:42.288 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/ui/nodes from=127.0.0.1:51592 latency=195.721µs -12:42.294 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/ui/nodes from=127.0.0.1:51594 error="method POST not allowed" -12:42.294 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/ui/nodes from=127.0.0.1:51594 latency=268.668µs -12:42.295 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/ui/nodes from=127.0.0.1:51596 error="method DELETE not allowed" -12:42.295 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/ui/nodes from=127.0.0.1:51596 latency=134.359µs -12:42.306 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/ui/nodes from=127.0.0.1:51598 error="method HEAD not allowed" -12:42.306 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/ui/nodes from=127.0.0.1:51598 latency=51.821µs -12:42.306 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/ui/nodes from=127.0.0.1:51598 latency=2.56µs -12:42.306 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/ from=127.0.0.1:51598 latency=179.003µs -12:42.308 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/ from=127.0.0.1:51600 latency=281.669µs -12:42.313 [ERROR] TestAgent.http: Request error: method=POST url=/v1/kv/ from=127.0.0.1:51602 error="method POST not allowed" -12:42.313 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/kv/ from=127.0.0.1:51602 latency=170.738µs -12:42.314 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/ from=127.0.0.1:51604 latency=109.961µs -12:42.322 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/kv/ from=127.0.0.1:51606 error="method HEAD not allowed" -12:42.322 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/kv/ from=127.0.0.1:51606 latency=40.864µs -12:42.322 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/kv/ from=127.0.0.1:51606 latency=2.31µs -12:42.323 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/ca/roots from=127.0.0.1:51606 latency=494.112µs -12:42.324 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/ca/roots from=127.0.0.1:51608 error="method PUT not allowed" -12:42.324 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/ca/roots from=127.0.0.1:51608 latency=140.671µs -12:42.325 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/ca/roots from=127.0.0.1:51610 error="method POST not allowed" -12:42.325 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/ca/roots from=127.0.0.1:51610 latency=105.432µs -12:42.325 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/ca/roots from=127.0.0.1:51612 error="method DELETE not allowed" -12:42.325 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/ca/roots from=127.0.0.1:51612 latency=135.106µs -12:42.330 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/ca/roots from=127.0.0.1:51614 error="method HEAD not allowed" -12:42.330 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/ca/roots from=127.0.0.1:51614 latency=54.949µs -12:42.334 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/ca/roots from=127.0.0.1:51614 latency=4.588µs -12:42.335 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/bootstrap from=127.0.0.1:51614 error="method GET not allowed" -12:42.335 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/bootstrap from=127.0.0.1:51614 latency=281.248µs -12:42.336 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/bootstrap from=127.0.0.1:51616 latency=206.529µs -12:42.336 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/bootstrap from=127.0.0.1:51618 error="method POST not allowed" -12:42.337 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/bootstrap from=127.0.0.1:51618 latency=122.685µs -12:42.337 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/bootstrap from=127.0.0.1:51620 error="method DELETE not allowed" -12:42.337 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/bootstrap from=127.0.0.1:51620 latency=197.068µs -12:42.339 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/bootstrap from=127.0.0.1:51622 error="method HEAD not allowed" -12:42.339 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/bootstrap from=127.0.0.1:51622 latency=32.165µs -12:42.339 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/bootstrap from=127.0.0.1:51622 latency=2.253µs -12:42.339 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policy from=127.0.0.1:51622 error="method GET not allowed" -12:42.339 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy from=127.0.0.1:51622 latency=173.6µs -12:42.340 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/policy from=127.0.0.1:51624 error="Bad request: Policy decoding failed: EOF" -12:42.340 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:51624 latency=166.942µs -12:42.341 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/policy from=127.0.0.1:51626 error="method POST not allowed" -12:42.341 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/policy from=127.0.0.1:51626 latency=163.367µs -12:42.342 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/policy from=127.0.0.1:51628 error="method DELETE not allowed" -12:42.342 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policy from=127.0.0.1:51628 latency=272.717µs -12:42.348 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/policy from=127.0.0.1:51630 error="method HEAD not allowed" -12:42.348 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/policy from=127.0.0.1:51630 latency=42.77µs -12:42.353 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/policy from=127.0.0.1:51630 latency=3.48µs -12:42.355 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/list from=127.0.0.1:51630 latency=1.153979ms -12:42.355 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/session/list from=127.0.0.1:51632 error="method PUT not allowed" -12:42.356 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/list from=127.0.0.1:51632 latency=167.961µs -12:42.366 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/list from=127.0.0.1:51634 error="method POST not allowed" -12:42.366 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/list from=127.0.0.1:51634 latency=185.684µs -12:42.367 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/list from=127.0.0.1:51636 error="method DELETE not allowed" -12:42.367 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/list from=127.0.0.1:51636 latency=211.886µs -12:42.368 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/list from=127.0.0.1:51638 error="method HEAD not allowed" -12:42.368 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/list from=127.0.0.1:51638 latency=44.535µs -12:42.369 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/list from=127.0.0.1:51638 latency=2.478µs -12:42.369 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/list from=127.0.0.1:51638 error="Permission denied" -12:42.369 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/list from=127.0.0.1:51638 latency=259.516µs -12:42.374 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/list from=127.0.0.1:51640 error="method PUT not allowed" -12:42.374 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/list from=127.0.0.1:51640 latency=192.353µs -12:42.375 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/list from=127.0.0.1:51642 error="method POST not allowed" -12:42.375 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/list from=127.0.0.1:51642 latency=122.008µs -12:42.375 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/list from=127.0.0.1:51644 error="method DELETE not allowed" -12:42.376 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/list from=127.0.0.1:51644 latency=128.2µs -12:42.376 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/list from=127.0.0.1:51646 error="method HEAD not allowed" -12:42.376 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/list from=127.0.0.1:51646 latency=45.603µs -12:42.377 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/list from=127.0.0.1:51646 latency=2.403µs -12:42.377 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/rules/translate/ from=127.0.0.1:51646 error="Bad request: Missing token ID" -12:42.378 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/rules/translate/ from=127.0.0.1:51646 latency=425.862µs -12:42.382 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/rules/translate/ from=127.0.0.1:51648 error="method PUT not allowed" -12:42.382 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/rules/translate/ from=127.0.0.1:51648 latency=190.569µs -12:42.383 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/rules/translate/ from=127.0.0.1:51650 error="method POST not allowed" -12:42.383 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/rules/translate/ from=127.0.0.1:51650 latency=102.957µs -12:42.383 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/rules/translate/ from=127.0.0.1:51652 error="method DELETE not allowed" -12:42.383 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/rules/translate/ from=127.0.0.1:51652 latency=98.075µs -12:42.384 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/rules/translate/ from=127.0.0.1:51654 error="method HEAD not allowed" -12:42.384 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/rules/translate/ from=127.0.0.1:51654 latency=18.589µs -12:42.384 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/rules/translate/ from=127.0.0.1:51654 latency=1.62µs -12:42.390 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:51654 latency=385.984µs -12:42.391 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/coordinate/nodes from=127.0.0.1:51656 error="method PUT not allowed" -12:42.391 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/coordinate/nodes from=127.0.0.1:51656 latency=144.21µs -12:42.391 [ERROR] TestAgent.http: Request error: method=POST url=/v1/coordinate/nodes from=127.0.0.1:51658 error="method POST not allowed" -12:42.392 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/coordinate/nodes from=127.0.0.1:51658 latency=114.31µs -12:42.392 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/coordinate/nodes from=127.0.0.1:51660 error="method DELETE not allowed" -12:42.392 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/coordinate/nodes from=127.0.0.1:51660 latency=138.695µs -12:42.398 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/coordinate/nodes from=127.0.0.1:51662 error="method HEAD not allowed" -12:42.398 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/coordinate/nodes from=127.0.0.1:51662 latency=49.022µs -12:42.406 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/coordinate/nodes from=127.0.0.1:51662 latency=5.455µs -12:42.407 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:51662 error="URI must be either service or agent" -12:42.407 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:51662 latency=421.397µs -12:42.408 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:51664 error="method PUT not allowed" -12:42.408 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:51664 latency=191.293µs -12:42.414 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:51666 error="method POST not allowed" -12:42.414 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:51666 latency=271.881µs -12:42.415 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:51668 error="method DELETE not allowed" -12:42.415 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:51668 latency=103.386µs -12:42.415 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:51670 error="method HEAD not allowed" -12:42.415 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:51670 latency=17.636µs -12:42.417 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:51670 latency=3.383µs -12:42.421 [ERROR] TestAgent.http: Request error: method=GET url=/v1/event/fire/ from=127.0.0.1:51670 error="method GET not allowed" -12:42.422 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/event/fire/ from=127.0.0.1:51670 latency=314.916µs -12:42.423 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/ from=127.0.0.1:51672 latency=147.756µs -12:42.432 [ERROR] TestAgent.http: Request error: method=POST url=/v1/event/fire/ from=127.0.0.1:51674 error="method POST not allowed" -12:42.432 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/event/fire/ from=127.0.0.1:51674 latency=174.484µs -12:42.432 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/event/fire/ from=127.0.0.1:51676 error="method DELETE not allowed" -12:42.433 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/event/fire/ from=127.0.0.1:51676 latency=111.428µs -12:42.435 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/event/fire/ from=127.0.0.1:51678 error="method HEAD not allowed" -12:42.435 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/event/fire/ from=127.0.0.1:51678 latency=30.664µs -12:42.435 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/event/fire/ from=127.0.0.1:51678 latency=1.934µs -12:42.436 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/ from=127.0.0.1:51678 latency=185.703µs -12:42.438 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/service/ from=127.0.0.1:51680 error="method PUT not allowed" -12:42.438 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/service/ from=127.0.0.1:51680 latency=247.831µs -12:42.439 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/service/ from=127.0.0.1:51682 error="method POST not allowed" -12:42.439 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/service/ from=127.0.0.1:51682 latency=146.652µs -12:42.439 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/service/ from=127.0.0.1:51684 error="method DELETE not allowed" -12:42.440 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/service/ from=127.0.0.1:51684 latency=100.958µs -12:42.440 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/service/ from=127.0.0.1:51686 error="method HEAD not allowed" -12:42.440 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/service/ from=127.0.0.1:51686 latency=44.004µs -12:42.443 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/service/ from=127.0.0.1:51686 latency=3.313µs -12:42.443 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/auth-method from=127.0.0.1:51686 error="method GET not allowed" -12:42.443 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method from=127.0.0.1:51686 latency=162.093µs -12:42.444 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/auth-method from=127.0.0.1:51688 error="Bad request: AuthMethod decoding failed: EOF" -12:42.444 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:51688 latency=147.519µs -12:42.445 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/auth-method from=127.0.0.1:51690 error="method POST not allowed" -12:42.445 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/auth-method from=127.0.0.1:51690 latency=420.773µs -12:42.447 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/auth-method from=127.0.0.1:51692 error="method DELETE not allowed" -12:42.448 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-method from=127.0.0.1:51692 latency=171.09µs -12:42.452 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/auth-method from=127.0.0.1:51694 error="method HEAD not allowed" -12:42.452 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/auth-method from=127.0.0.1:51694 latency=46.113µs -12:42.452 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/auth-method from=127.0.0.1:51694 latency=1.786µs -12:42.452 [ERROR] TestAgent.http: Request error: method=GET url=/v1/config from=127.0.0.1:51694 error="method GET not allowed" -12:42.453 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config from=127.0.0.1:51694 latency=155.502µs -12:42.453 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/config from=127.0.0.1:51696 error="Bad request: Request decoding failed: EOF" -12:42.454 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:51696 latency=360.824µs -12:42.462 [ERROR] TestAgent.http: Request error: method=POST url=/v1/config from=127.0.0.1:51698 error="method POST not allowed" -12:42.462 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/config from=127.0.0.1:51698 latency=271.453µs -12:42.463 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/config from=127.0.0.1:51700 error="method DELETE not allowed" -12:42.463 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/config from=127.0.0.1:51700 latency=120.665µs -12:42.464 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/config from=127.0.0.1:51702 error="method HEAD not allowed" -12:42.464 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/config from=127.0.0.1:51702 latency=21.72µs -12:42.464 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/config from=127.0.0.1:51702 latency=1.898µs -12:42.464 [ERROR] TestAgent.http: Request error: method=GET url=/v1/connect/intentions/match from=127.0.0.1:51702 error="required query parameter 'by' not set" -12:42.464 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/match from=127.0.0.1:51702 latency=139.64µs -12:42.465 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/intentions/match from=127.0.0.1:51704 error="method PUT not allowed" -12:42.465 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions/match from=127.0.0.1:51704 latency=125.166µs -12:42.474 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions/match from=127.0.0.1:51706 error="method POST not allowed" -12:42.474 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions/match from=127.0.0.1:51706 latency=223.039µs -12:42.477 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/intentions/match from=127.0.0.1:51708 error="method DELETE not allowed" -12:42.478 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions/match from=127.0.0.1:51708 latency=434.541µs -12:42.478 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/intentions/match from=127.0.0.1:51710 error="method HEAD not allowed" -12:42.478 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/intentions/match from=127.0.0.1:51710 latency=17.234µs -12:42.478 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/intentions/match from=127.0.0.1:51710 latency=1.776µs -12:42.481 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/node/ from=127.0.0.1:51710 latency=247.454µs -12:42.486 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/session/node/ from=127.0.0.1:51712 error="method PUT not allowed" -12:42.486 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/node/ from=127.0.0.1:51712 latency=249.429µs -12:42.490 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/node/ from=127.0.0.1:51714 error="method POST not allowed" -12:42.490 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/node/ from=127.0.0.1:51714 latency=316.705µs -12:42.491 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/node/ from=127.0.0.1:51716 error="method DELETE not allowed" -12:42.491 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/node/ from=127.0.0.1:51716 latency=149.314µs -12:42.492 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/node/ from=127.0.0.1:51718 error="method HEAD not allowed" -12:42.492 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/node/ from=127.0.0.1:51718 latency=23.387µs -12:42.492 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/node/ from=127.0.0.1:51718 latency=2.323µs -12:42.493 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/rules/translate from=127.0.0.1:51718 error="method GET not allowed" -12:42.493 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/rules/translate from=127.0.0.1:51718 latency=145.421µs -12:42.493 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/rules/translate from=127.0.0.1:51720 error="method PUT not allowed" -12:42.494 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/rules/translate from=127.0.0.1:51720 latency=203.736µs -12:42.502 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/rules/translate from=127.0.0.1:51722 error="Permission denied" -12:42.502 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/rules/translate from=127.0.0.1:51722 latency=287.632µs -12:42.503 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/rules/translate from=127.0.0.1:51724 error="method DELETE not allowed" -12:42.503 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/rules/translate from=127.0.0.1:51724 latency=126.454µs -12:42.503 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/rules/translate from=127.0.0.1:51726 error="method HEAD not allowed" -12:42.503 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/rules/translate from=127.0.0.1:51726 latency=24.608µs -12:42.504 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/rules/translate from=127.0.0.1:51726 latency=2.157µs -12:42.504 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/token/ from=127.0.0.1:51726 error="Bad request: Missing token ID" -12:42.504 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/ from=127.0.0.1:51726 latency=134.105µs -12:42.505 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/token/ from=127.0.0.1:51728 error="Bad request: Token decoding failed: EOF" -12:42.505 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/ from=127.0.0.1:51728 latency=406.707µs -12:42.510 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/token/ from=127.0.0.1:51730 error="method POST not allowed" -12:42.511 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/token/ from=127.0.0.1:51730 latency=212.886µs -12:42.511 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/token/ from=127.0.0.1:51732 error="Bad request: Missing token ID" -12:42.511 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/token/ from=127.0.0.1:51732 latency=130.089µs -12:42.512 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/token/ from=127.0.0.1:51734 error="method HEAD not allowed" -12:42.512 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/token/ from=127.0.0.1:51734 latency=25.286µs -12:42.512 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/token/ from=127.0.0.1:51734 latency=2.039µs -12:42.513 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/host from=127.0.0.1:51734 error="Permission denied" -12:42.513 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/host from=127.0.0.1:51734 latency=242.62µs -12:42.517 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/host from=127.0.0.1:51736 error="method PUT not allowed" -12:42.517 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/host from=127.0.0.1:51736 latency=180.082µs -12:42.522 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/host from=127.0.0.1:51738 error="method POST not allowed" -12:42.522 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/host from=127.0.0.1:51738 latency=245.382µs -12:42.523 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/host from=127.0.0.1:51740 error="method DELETE not allowed" -12:42.523 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/host from=127.0.0.1:51740 latency=120.465µs -12:42.524 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/host from=127.0.0.1:51742 error="method HEAD not allowed" -12:42.524 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/host from=127.0.0.1:51742 latency=19.994µs -12:42.524 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/host from=127.0.0.1:51742 latency=1.7µs -12:42.524 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/health/service/name/ from=127.0.0.1:51742 error="Bad request: Missing service Name" -12:42.524 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/health/service/name/ from=127.0.0.1:51742 latency=109.402µs -12:42.525 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/health/service/name/ from=127.0.0.1:51744 error="method PUT not allowed" -12:42.525 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/health/service/name/ from=127.0.0.1:51744 latency=238.332µs -12:42.530 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/health/service/name/ from=127.0.0.1:51746 error="method POST not allowed" -12:42.531 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/health/service/name/ from=127.0.0.1:51746 latency=223.978µs -12:42.531 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/health/service/name/ from=127.0.0.1:51748 error="method DELETE not allowed" -12:42.531 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/health/service/name/ from=127.0.0.1:51748 latency=222.925µs -12:42.532 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/health/service/name/ from=127.0.0.1:51750 error="method HEAD not allowed" -12:42.532 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/health/service/name/ from=127.0.0.1:51750 latency=22.455µs -12:42.532 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/health/service/name/ from=127.0.0.1:51750 latency=1.84µs -12:42.533 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/service/register from=127.0.0.1:51750 error="method GET not allowed" -12:42.533 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/register from=127.0.0.1:51750 latency=122.233µs -12:42.534 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:51752 latency=375.453µs -12:42.539 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/service/register from=127.0.0.1:51754 error="method POST not allowed" -12:42.539 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/service/register from=127.0.0.1:51754 latency=173.015µs -12:42.544 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/service/register from=127.0.0.1:51756 error="method DELETE not allowed" -12:42.544 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/service/register from=127.0.0.1:51756 latency=238.065µs -12:42.545 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/service/register from=127.0.0.1:51758 error="method HEAD not allowed" -12:42.545 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/service/register from=127.0.0.1:51758 latency=22.024µs -12:42.545 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/service/register from=127.0.0.1:51758 latency=2.067µs -12:42.545 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policy/name/ from=127.0.0.1:51758 error="Bad request: Missing policy Name" -12:42.545 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/name/ from=127.0.0.1:51758 latency=231.185µs -12:42.554 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/policy/name/ from=127.0.0.1:51760 error="method PUT not allowed" -12:42.554 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy/name/ from=127.0.0.1:51760 latency=187.349µs -12:42.562 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/policy/name/ from=127.0.0.1:51762 error="method POST not allowed" -12:42.562 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/policy/name/ from=127.0.0.1:51762 latency=253.99µs -12:42.570 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/policy/name/ from=127.0.0.1:51764 error="method DELETE not allowed" -12:42.570 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policy/name/ from=127.0.0.1:51764 latency=192.146µs -12:42.571 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/policy/name/ from=127.0.0.1:51766 error="method HEAD not allowed" -12:42.571 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/policy/name/ from=127.0.0.1:51766 latency=21.326µs -12:42.571 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/policy/name/ from=127.0.0.1:51766 latency=2.08µs -12:42.571 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/ from=127.0.0.1:51766 latency=156.666µs -12:42.572 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/service/ from=127.0.0.1:51768 error="method PUT not allowed" -12:42.572 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/ from=127.0.0.1:51768 latency=107.01µs -12:42.572 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/service/ from=127.0.0.1:51770 error="method POST not allowed" -12:42.573 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/service/ from=127.0.0.1:51770 latency=273.641µs -12:42.578 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/service/ from=127.0.0.1:51772 error="method DELETE not allowed" -12:42.578 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/service/ from=127.0.0.1:51772 latency=266.288µs -12:42.579 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/service/ from=127.0.0.1:51774 error="method HEAD not allowed" -12:42.579 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/service/ from=127.0.0.1:51774 latency=19.38µs -12:42.579 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/service/ from=127.0.0.1:51774 latency=1.651µs -12:42.579 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/raft/peer from=127.0.0.1:51774 error="method GET not allowed" -12:42.580 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/raft/peer from=127.0.0.1:51774 latency=120.893µs -12:42.580 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/raft/peer from=127.0.0.1:51776 error="method PUT not allowed" -12:42.580 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/raft/peer from=127.0.0.1:51776 latency=135.051µs -12:42.587 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/raft/peer from=127.0.0.1:51778 error="method POST not allowed" -12:42.587 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/raft/peer from=127.0.0.1:51778 latency=234.668µs -12:42.588 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/raft/peer from=127.0.0.1:51780 error="Bad request: Must specify either ?id with the server's ID or ?address with IP:port of peer to remove" -12:42.588 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/raft/peer from=127.0.0.1:51780 latency=144.562µs -12:42.588 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/raft/peer from=127.0.0.1:51782 error="method HEAD not allowed" -12:42.588 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/raft/peer from=127.0.0.1:51782 latency=22.543µs -12:42.589 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/raft/peer from=127.0.0.1:51782 latency=1.953µs -12:42.589 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/replication from=127.0.0.1:51782 latency=262.622µs -12:42.602 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/replication from=127.0.0.1:51784 error="method PUT not allowed" -12:42.602 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/replication from=127.0.0.1:51784 latency=290.498µs -12:42.603 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/replication from=127.0.0.1:51786 error="method POST not allowed" -12:42.603 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/replication from=127.0.0.1:51786 latency=125.734µs -12:42.604 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/replication from=127.0.0.1:51788 error="method DELETE not allowed" -12:42.604 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/replication from=127.0.0.1:51788 latency=116.281µs -12:42.604 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/replication from=127.0.0.1:51790 error="method HEAD not allowed" -12:42.604 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/replication from=127.0.0.1:51790 latency=37.626µs -12:42.610 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/replication from=127.0.0.1:51790 latency=2.952µs -12:42.610 [ERROR] TestAgent.http: Request error: method=GET url=/v1/internal/acl/authorize from=127.0.0.1:51790 error="method GET not allowed" -12:42.610 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/acl/authorize from=127.0.0.1:51790 latency=280.384µs -12:42.611 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/acl/authorize from=127.0.0.1:51792 error="method PUT not allowed" -12:42.612 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/acl/authorize from=127.0.0.1:51792 latency=219.86µs -12:42.615 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/acl/authorize from=127.0.0.1:51794 error="Bad request: Failed to decode request body: EOF" -12:42.615 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/acl/authorize from=127.0.0.1:51794 latency=206.711µs -12:42.616 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/acl/authorize from=127.0.0.1:51796 error="method DELETE not allowed" -12:42.616 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/acl/authorize from=127.0.0.1:51796 latency=114.751µs -12:42.616 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/acl/authorize from=127.0.0.1:51798 error="method HEAD not allowed" -12:42.616 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/acl/authorize from=127.0.0.1:51798 latency=27.874µs -12:42.621 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/acl/authorize from=127.0.0.1:51798 latency=3.692µs -12:42.621 [ERROR] TestAgent.http: Request error: method=GET url=/v1/session/destroy/ from=127.0.0.1:51798 error="method GET not allowed" -12:42.622 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/destroy/ from=127.0.0.1:51798 latency=357.274µs -12:42.622 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/ from=127.0.0.1:51800 latency=98.828µs -12:42.623 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/destroy/ from=127.0.0.1:51802 error="method POST not allowed" -12:42.623 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/destroy/ from=127.0.0.1:51802 latency=111.663µs -12:42.623 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/destroy/ from=127.0.0.1:51804 error="method DELETE not allowed" -12:42.624 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/destroy/ from=127.0.0.1:51804 latency=109.589µs -12:42.624 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/destroy/ from=127.0.0.1:51806 error="method HEAD not allowed" -12:42.624 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/destroy/ from=127.0.0.1:51806 latency=43.135µs -12:42.631 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/destroy/ from=127.0.0.1:51806 latency=5.481µs -12:42.632 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/deregister/ from=127.0.0.1:51806 error="method GET not allowed" -12:42.632 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/deregister/ from=127.0.0.1:51806 latency=193.21µs -12:42.633 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/check/deregister/ from=127.0.0.1:51808 error="Unknown check """ -12:42.633 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/deregister/ from=127.0.0.1:51808 latency=167.259µs -12:42.634 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/deregister/ from=127.0.0.1:51810 error="method POST not allowed" -12:42.634 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/deregister/ from=127.0.0.1:51810 latency=340.341µs -12:42.639 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/deregister/ from=127.0.0.1:51814 error="method DELETE not allowed" -12:42.639 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/deregister/ from=127.0.0.1:51814 latency=226.646µs -12:42.640 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/deregister/ from=127.0.0.1:51816 error="method HEAD not allowed" -12:42.640 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/deregister/ from=127.0.0.1:51816 latency=19.126µs -12:42.640 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/deregister/ from=127.0.0.1:51816 latency=1.94µs -12:42.640 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/ingress/ from=127.0.0.1:51816 latency=116.053µs -12:42.641 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/ingress/ from=127.0.0.1:51818 error="method PUT not allowed" -12:42.641 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/ingress/ from=127.0.0.1:51818 latency=244.063µs -12:42.651 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/ingress/ from=127.0.0.1:51820 error="method POST not allowed" -12:42.652 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/ingress/ from=127.0.0.1:51820 latency=225.823µs -12:42.665 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/ingress/ from=127.0.0.1:51824 error="method DELETE not allowed" -12:42.665 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/ingress/ from=127.0.0.1:51824 latency=181.376µs -12:42.670 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/ingress/ from=127.0.0.1:51826 error="method HEAD not allowed" -12:42.670 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/ingress/ from=127.0.0.1:51826 latency=77.708µs -12:42.670 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/ingress/ from=127.0.0.1:51826 latency=2.193µs -12:42.670 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/autopilot/health from=127.0.0.1:51826 error="Permission denied" -12:42.671 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/autopilot/health from=127.0.0.1:51826 latency=390.657µs -12:42.671 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/autopilot/health from=127.0.0.1:51830 error="method PUT not allowed" -12:42.671 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/autopilot/health from=127.0.0.1:51830 latency=131.432µs -12:42.672 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/autopilot/health from=127.0.0.1:51832 error="method POST not allowed" -12:42.672 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/autopilot/health from=127.0.0.1:51832 latency=132.991µs -12:42.682 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/autopilot/health from=127.0.0.1:51834 error="method DELETE not allowed" -12:42.682 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/autopilot/health from=127.0.0.1:51834 latency=210.539µs -12:42.683 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/autopilot/health from=127.0.0.1:51838 error="method HEAD not allowed" -12:42.683 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/autopilot/health from=127.0.0.1:51838 latency=23.898µs -12:42.683 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/autopilot/health from=127.0.0.1:51838 latency=2.041µs -12:42.683 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/status/leader from=127.0.0.1:51838 latency=185.147µs -12:42.684 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/status/leader from=127.0.0.1:51840 error="method PUT not allowed" -12:42.684 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/status/leader from=127.0.0.1:51840 latency=199.634µs -12:42.690 [ERROR] TestAgent.http: Request error: method=POST url=/v1/status/leader from=127.0.0.1:51844 error="method POST not allowed" -12:42.690 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/status/leader from=127.0.0.1:51844 latency=387.704µs -12:42.691 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/status/leader from=127.0.0.1:51846 error="method DELETE not allowed" -12:42.691 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/status/leader from=127.0.0.1:51846 latency=114.38µs -12:42.692 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/status/leader from=127.0.0.1:51848 error="method HEAD not allowed" -12:42.692 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/status/leader from=127.0.0.1:51848 latency=20.673µs -12:42.692 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/status/leader from=127.0.0.1:51848 latency=1.882µs -12:42.692 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/node/ from=127.0.0.1:51848 latency=116.575µs -12:42.693 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/node/ from=127.0.0.1:51850 error="method PUT not allowed" -12:42.693 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/node/ from=127.0.0.1:51850 latency=170.65µs -12:42.695 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/node/ from=127.0.0.1:51852 error="method POST not allowed" -12:42.695 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/node/ from=127.0.0.1:51852 latency=298.06µs -12:42.696 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/node/ from=127.0.0.1:51854 error="method DELETE not allowed" -12:42.696 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/node/ from=127.0.0.1:51854 latency=109.585µs -12:42.701 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/node/ from=127.0.0.1:51856 error="method HEAD not allowed" -12:42.701 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/node/ from=127.0.0.1:51856 latency=40.745µs -12:42.701 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/node/ from=127.0.0.1:51856 latency=1.924µs -12:42.701 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/tokens from=127.0.0.1:51856 error="Permission denied" -12:42.702 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/tokens from=127.0.0.1:51856 latency=275.788µs -12:42.706 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/tokens from=127.0.0.1:51858 error="method PUT not allowed" -12:42.706 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/tokens from=127.0.0.1:51858 latency=199.701µs -12:42.715 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/tokens from=127.0.0.1:51860 error="method POST not allowed" -12:42.715 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/tokens from=127.0.0.1:51860 latency=203.732µs -12:42.718 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/tokens from=127.0.0.1:51864 error="method DELETE not allowed" -12:42.718 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/tokens from=127.0.0.1:51864 latency=277.781µs -12:42.719 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/tokens from=127.0.0.1:51866 error="method HEAD not allowed" -12:42.719 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/tokens from=127.0.0.1:51866 latency=19.853µs -12:42.719 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/tokens from=127.0.0.1:51866 latency=1.827µs -12:42.719 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/connect/authorize from=127.0.0.1:51866 error="method GET not allowed" -12:42.720 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/connect/authorize from=127.0.0.1:51866 latency=114.77µs -12:42.720 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/connect/authorize from=127.0.0.1:51868 error="method PUT not allowed" -12:42.720 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/connect/authorize from=127.0.0.1:51868 latency=244.334µs -12:42.727 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:51872 error="Bad request: Request decode failed: EOF" -12:42.727 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:51872 latency=197.51µs -12:42.728 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/connect/authorize from=127.0.0.1:51874 error="method DELETE not allowed" -12:42.728 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/connect/authorize from=127.0.0.1:51874 latency=266.149µs -12:42.731 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/connect/authorize from=127.0.0.1:51876 error="method HEAD not allowed" -12:42.731 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/connect/authorize from=127.0.0.1:51876 latency=56.304µs -12:42.732 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/connect/authorize from=127.0.0.1:51876 latency=2.032µs -12:42.732 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/connect/ca/roots from=127.0.0.1:51876 latency=287.174µs -12:42.733 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/connect/ca/roots from=127.0.0.1:51878 error="method PUT not allowed" -12:42.734 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/connect/ca/roots from=127.0.0.1:51878 latency=407.179µs -12:42.739 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/connect/ca/roots from=127.0.0.1:51880 error="method POST not allowed" -12:42.739 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/connect/ca/roots from=127.0.0.1:51880 latency=234.76µs -12:42.739 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/connect/ca/roots from=127.0.0.1:51882 error="method DELETE not allowed" -12:42.740 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/connect/ca/roots from=127.0.0.1:51882 latency=108.198µs -12:42.740 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/connect/ca/roots from=127.0.0.1:51884 error="method HEAD not allowed" -12:42.740 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/connect/ca/roots from=127.0.0.1:51884 latency=23.53µs -12:42.750 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/connect/ca/roots from=127.0.0.1:51884 latency=6.772µs -12:42.758 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/node-services/ from=127.0.0.1:51884 latency=300.575µs -12:42.762 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/node-services/ from=127.0.0.1:51886 error="method PUT not allowed" -12:42.762 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/node-services/ from=127.0.0.1:51886 latency=164.528µs -12:42.763 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/node-services/ from=127.0.0.1:51888 error="method POST not allowed" -12:42.763 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/node-services/ from=127.0.0.1:51888 latency=116.673µs -12:42.764 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/node-services/ from=127.0.0.1:51890 error="method DELETE not allowed" -12:42.764 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/node-services/ from=127.0.0.1:51890 latency=112.563µs -12:42.764 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/node-services/ from=127.0.0.1:51892 error="method HEAD not allowed" -12:42.764 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/node-services/ from=127.0.0.1:51892 latency=19.074µs -12:42.771 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/node-services/ from=127.0.0.1:51892 latency=4.628µs -12:42.771 [ERROR] TestAgent.http: Request error: method=GET url=/v1/discovery-chain/ from=127.0.0.1:51892 error="Bad request: Missing chain name" -12:42.772 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/discovery-chain/ from=127.0.0.1:51892 latency=178.335µs -12:42.772 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/discovery-chain/ from=127.0.0.1:51894 error="method PUT not allowed" -12:42.773 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/discovery-chain/ from=127.0.0.1:51894 latency=227.265µs -12:42.774 [ERROR] TestAgent.http: Request error: method=POST url=/v1/discovery-chain/ from=127.0.0.1:51896 error="Bad request: Missing chain name" -12:42.775 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/discovery-chain/ from=127.0.0.1:51896 latency=146.35µs -12:42.775 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/discovery-chain/ from=127.0.0.1:51898 error="method DELETE not allowed" -12:42.775 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/discovery-chain/ from=127.0.0.1:51898 latency=96.448µs -12:42.776 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/discovery-chain/ from=127.0.0.1:51900 error="method HEAD not allowed" -12:42.776 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/discovery-chain/ from=127.0.0.1:51900 latency=18.165µs -12:42.776 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/discovery-chain/ from=127.0.0.1:51900 latency=1.967µs -12:42.776 [INFO] TestAgent: Requesting shutdown -12:42.776 [INFO] TestAgent.server: shutting down server -12:42.776 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:42.776 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:42.776 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:42.776 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:42.776 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:42.776 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:42.776 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:42.776 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:42.776 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:42.776 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:42.776 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:42.776 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:42.776 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:42.776 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:42.776 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:42.777Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:42.777Z [INFO] agent.router.manager: shutting down -12:42.777 [INFO] TestAgent: consul server down -12:42.777 [INFO] TestAgent: shutdown complete -12:42.777 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17009 network=tcp -12:42.777 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17009 network=udp -12:42.777 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17010 network=tcp -12:43.277 [INFO] TestAgent: Waiting for endpoints to shut down -12:43.277 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPAPI_MethodNotAllowed_OSS (2.28s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query (0.02s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query/ (0.02s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query/xxx/execute (0.01s) +2022-12-04T05:48:55.110Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:55.131Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:55.133 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0d456efc-6231-9ceb-2643-32c992012de5 Address:127.0.0.1:24161}]" +48:55.133 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24161 [Follower]" leader= +48:55.133 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0d456efc-6231-9ceb-2643-32c992012de5.dc1 127.0.0.1 +48:55.134 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0d456efc-6231-9ceb-2643-32c992012de5 127.0.0.1 +2022-12-04T05:48:55.134Z [INFO] agent.router: Initializing LAN area manager +48:55.134 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0d456efc-6231-9ceb-2643-32c992012de5.dc1 area=wan +48:55.134 [INFO] TestAgent.server: Adding LAN server: server="Node-0d456efc-6231-9ceb-2643-32c992012de5 (Addr: tcp/127.0.0.1:24161) (DC: dc1)" +48:55.134 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24156 network=udp +48:55.134 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24156 network=tcp +48:55.135 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24157 network=tcp +48:55.135 [INFO] TestAgent: started state syncer +48:55.135 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24162 network=tcp +48:55.184 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:55.188 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:55.188 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24161 [Candidate]" term=2 +48:55.188 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:55.188 [DEBUG] TestAgent.server.raft: vote granted: from=0d456efc-6231-9ceb-2643-32c992012de5 term=2 tally=1 +48:55.188 [INFO] TestAgent.server.raft: election won: tally=1 +48:55.188 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24161 [Leader]" +48:55.188 [INFO] TestAgent.server: cluster leadership acquired +48:55.188 [INFO] TestAgent.server: initializing acls +48:55.189 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:55.189 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:55.189 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:55.189 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:55.189 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:55.189 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:55.189 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0d456efc-6231-9ceb-2643-32c992012de5 +48:55.189 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0d456efc-6231-9ceb-2643-32c992012de5.dc1 +48:55.190 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:55.191 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:55.191 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:55.191 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:55.191 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:55.191 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:55.191 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d456efc-6231-9ceb-2643-32c992012de5 +48:55.191 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0d456efc-6231-9ceb-2643-32c992012de5 +48:55.191 [INFO] TestAgent.server: New leader elected: payload=Node-0d456efc-6231-9ceb-2643-32c992012de5 +48:55.191 [INFO] TestAgent.server: Updating LAN server: server="Node-0d456efc-6231-9ceb-2643-32c992012de5 (Addr: tcp/127.0.0.1:24161) (DC: dc1)" +48:55.191 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d456efc-6231-9ceb-2643-32c992012de5 +48:55.191 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0d456efc-6231-9ceb-2643-32c992012de5.dc1 area=wan +48:55.191 [INFO] TestAgent.server: federation state anti-entropy synced +48:55.218 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:55.219 [INFO] TestAgent: Synced node info +48:55.219 [DEBUG] TestAgent: Node info in sync +48:55.228 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0d456efc-6231-9ceb-2643-32c992012de5 +48:55.230 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/query/xxx/execute from=127.0.0.1:42904 latency=454.354µs +48:55.230 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/query/xxx/execute from=127.0.0.1:42906 error="method PUT not allowed" +48:55.231 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/query/xxx/execute from=127.0.0.1:42906 latency=336.478µs +48:55.231 [ERROR] TestAgent.http: Request error: method=POST url=/v1/query/xxx/execute from=127.0.0.1:42908 error="method POST not allowed" +48:55.232 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/query/xxx/execute from=127.0.0.1:42908 latency=223.185µs +48:55.232 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/query/xxx/execute from=127.0.0.1:42910 error="method DELETE not allowed" +48:55.232 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/query/xxx/execute from=127.0.0.1:42910 latency=195.821µs +48:55.233 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/query/xxx/execute from=127.0.0.1:42912 error="method HEAD not allowed" +48:55.233 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/query/xxx/execute from=127.0.0.1:42912 latency=16.546µs +48:55.234 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/query/xxx/execute from=127.0.0.1:42912 latency=303.443µs +48:55.234 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/query/xxx/explain from=127.0.0.1:42914 latency=167.445µs +48:55.235 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/query/xxx/explain from=127.0.0.1:42916 error="method PUT not allowed" +48:55.235 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/query/xxx/explain from=127.0.0.1:42916 latency=177.222µs +48:55.235 [ERROR] TestAgent.http: Request error: method=POST url=/v1/query/xxx/explain from=127.0.0.1:42918 error="method POST not allowed" +48:55.236 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/query/xxx/explain from=127.0.0.1:42918 latency=202.765µs +48:55.236 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/query/xxx/explain from=127.0.0.1:42920 error="method DELETE not allowed" +48:55.236 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/query/xxx/explain from=127.0.0.1:42920 latency=176.598µs +48:55.237 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/query/xxx/explain from=127.0.0.1:42922 error="method HEAD not allowed" +48:55.237 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/query/xxx/explain from=127.0.0.1:42922 latency=14.113µs +48:55.237 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/query/xxx/explain from=127.0.0.1:42922 latency=178.038µs +48:55.238 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/query from=127.0.0.1:42924 latency=457.75µs +48:55.239 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/query from=127.0.0.1:42926 error="method PUT not allowed" +48:55.239 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/query from=127.0.0.1:42926 latency=167.436µs +48:55.241 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/query from=127.0.0.1:42928 latency=1.162725ms +48:55.241 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/query from=127.0.0.1:42930 error="method DELETE not allowed" +48:55.241 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/query from=127.0.0.1:42930 latency=175.326µs +48:55.242 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/query from=127.0.0.1:42932 error="method HEAD not allowed" +48:55.242 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/query from=127.0.0.1:42932 latency=22.315µs +48:55.242 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/query from=127.0.0.1:42932 latency=1.74µs +48:55.243 [ERROR] TestAgent.http: Request error: method=GET url=/v1/query/ from=127.0.0.1:42932 error="failed prepared query lookup: index error: UUID must be 36 characters" +48:55.243 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/query/ from=127.0.0.1:42932 latency=181.7µs +48:55.243 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/query/ from=127.0.0.1:42936 error="Prepared Query lookup failed: failed prepared query lookup: index error: UUID must be 36 characters" +48:55.243 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/query/ from=127.0.0.1:42936 latency=254.713µs +48:55.244 [ERROR] TestAgent.http: Request error: method=POST url=/v1/query/ from=127.0.0.1:42938 error="method POST not allowed" +48:55.244 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/query/ from=127.0.0.1:42938 latency=165.026µs +48:55.245 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/query/ from=127.0.0.1:42940 error="Prepared Query lookup failed: failed prepared query lookup: index error: UUID must be 36 characters" +48:55.245 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/query/ from=127.0.0.1:42940 latency=164.259µs +48:55.245 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/query/ from=127.0.0.1:42942 error="method HEAD not allowed" +48:55.245 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/query/ from=127.0.0.1:42942 latency=16.148µs +48:55.246 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/query/ from=127.0.0.1:42942 latency=152.507µs +48:55.246 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/tokens from=127.0.0.1:42946 error="Permission denied" +48:55.246 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/tokens from=127.0.0.1:42946 latency=161.108µs +48:55.247 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/tokens from=127.0.0.1:42948 error="method PUT not allowed" +48:55.247 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/tokens from=127.0.0.1:42948 latency=202.241µs +48:55.248 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/tokens from=127.0.0.1:42950 error="method POST not allowed" +48:55.248 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/tokens from=127.0.0.1:42950 latency=127.823µs +48:55.248 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/tokens from=127.0.0.1:42954 error="method DELETE not allowed" +48:55.248 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/tokens from=127.0.0.1:42954 latency=165.475µs +48:55.249 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/tokens from=127.0.0.1:42956 error="method HEAD not allowed" +48:55.249 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/tokens from=127.0.0.1:42956 latency=16.694µs +48:55.249 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/tokens from=127.0.0.1:42956 latency=1.433µs +48:55.250 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:42956 latency=220.991µs +48:55.250 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/services from=127.0.0.1:42958 error="method PUT not allowed" +48:55.251 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/services from=127.0.0.1:42958 latency=140.939µs +48:55.251 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/services from=127.0.0.1:42960 error="method POST not allowed" +48:55.251 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/services from=127.0.0.1:42960 latency=193.054µs +48:55.252 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/services from=127.0.0.1:42962 error="method DELETE not allowed" +48:55.252 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/services from=127.0.0.1:42962 latency=188.71µs +48:55.253 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/services from=127.0.0.1:42964 error="method HEAD not allowed" +48:55.253 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/services from=127.0.0.1:42964 latency=20.013µs +48:55.253 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/services from=127.0.0.1:42964 latency=3.303µs +48:55.254 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:42964 latency=394.791µs +48:55.254 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/intentions from=127.0.0.1:42966 error="method PUT not allowed" +48:55.254 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions from=127.0.0.1:42966 latency=122.195µs +48:55.255 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions from=127.0.0.1:42968 error="Failed to decode request body: EOF" +48:55.255 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:42968 latency=193.443µs +48:55.255 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/intentions from=127.0.0.1:42970 error="method DELETE not allowed" +48:55.256 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions from=127.0.0.1:42970 latency=190.873µs +48:55.256 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/intentions from=127.0.0.1:42972 error="method HEAD not allowed" +48:55.256 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/intentions from=127.0.0.1:42972 latency=12.898µs +48:55.256 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/intentions from=127.0.0.1:42972 latency=2.175µs +48:55.257 [ERROR] TestAgent.http: Request error: method=GET url=/v1/session/renew/ from=127.0.0.1:42972 error="method GET not allowed" +48:55.257 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/renew/ from=127.0.0.1:42972 latency=154.439µs +48:55.257 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/renew/ from=127.0.0.1:42974 latency=106.226µs +48:55.258 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/renew/ from=127.0.0.1:42976 error="method POST not allowed" +48:55.258 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/renew/ from=127.0.0.1:42976 latency=124.124µs +48:55.258 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/renew/ from=127.0.0.1:42978 error="method DELETE not allowed" +48:55.259 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/renew/ from=127.0.0.1:42978 latency=187.535µs +48:55.259 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/renew/ from=127.0.0.1:42980 error="method HEAD not allowed" +48:55.259 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/renew/ from=127.0.0.1:42980 latency=15.049µs +48:55.259 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/renew/ from=127.0.0.1:42980 latency=2.69µs +48:55.260 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policy from=127.0.0.1:42980 error="method GET not allowed" +48:55.260 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy from=127.0.0.1:42980 latency=177.128µs +48:55.260 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/policy from=127.0.0.1:42982 error="Bad request: Policy decoding failed: EOF" +48:55.260 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:42982 latency=182.45µs +48:55.261 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/policy from=127.0.0.1:42984 error="method POST not allowed" +48:55.261 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/policy from=127.0.0.1:42984 latency=180.745µs +48:55.262 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/policy from=127.0.0.1:42986 error="method DELETE not allowed" +48:55.262 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policy from=127.0.0.1:42986 latency=198.489µs +48:55.262 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/policy from=127.0.0.1:42988 error="method HEAD not allowed" +48:55.263 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/policy from=127.0.0.1:42988 latency=22.18µs +48:55.263 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/policy from=127.0.0.1:42988 latency=1.73µs +48:55.263 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/role from=127.0.0.1:42988 error="method GET not allowed" +48:55.263 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role from=127.0.0.1:42988 latency=209.033µs +48:55.264 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/role from=127.0.0.1:42990 error="Bad request: Role decoding failed: EOF" +48:55.264 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:42990 latency=166.301µs +48:55.264 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/role from=127.0.0.1:42992 error="method POST not allowed" +48:55.265 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/role from=127.0.0.1:42992 latency=222.964µs +48:55.265 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/role from=127.0.0.1:42994 error="method DELETE not allowed" +48:55.265 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role from=127.0.0.1:42994 latency=184.282µs +48:55.266 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/role from=127.0.0.1:42996 error="method HEAD not allowed" +48:55.266 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/role from=127.0.0.1:42996 latency=17.285µs +48:55.266 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/role from=127.0.0.1:42996 latency=2.013µs +48:55.266 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/rules/translate from=127.0.0.1:42996 error="method GET not allowed" +48:55.267 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/rules/translate from=127.0.0.1:42996 latency=136.543µs +48:55.267 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/rules/translate from=127.0.0.1:42998 error="method PUT not allowed" +48:55.267 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/rules/translate from=127.0.0.1:42998 latency=185.184µs +48:55.268 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/rules/translate from=127.0.0.1:43002 error="Permission denied" +48:55.268 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/rules/translate from=127.0.0.1:43002 latency=172.396µs +48:55.269 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/rules/translate from=127.0.0.1:43004 error="method DELETE not allowed" +48:55.269 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/rules/translate from=127.0.0.1:43004 latency=223.274µs +48:55.269 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/rules/translate from=127.0.0.1:43006 error="method HEAD not allowed" +48:55.269 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/rules/translate from=127.0.0.1:43006 latency=22.967µs +48:55.270 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/rules/translate from=127.0.0.1:43006 latency=3.928µs +48:55.270 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/binding-rule from=127.0.0.1:43006 error="method GET not allowed" +48:55.270 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule from=127.0.0.1:43006 latency=135.847µs +48:55.271 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:43010 error="Bad request: BindingRule decoding failed: EOF" +48:55.271 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:43010 latency=116.901µs +48:55.271 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/binding-rule from=127.0.0.1:43012 error="method POST not allowed" +48:55.271 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/binding-rule from=127.0.0.1:43012 latency=177.089µs +48:55.272 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/binding-rule from=127.0.0.1:43016 error="method DELETE not allowed" +48:55.272 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule from=127.0.0.1:43016 latency=99.01µs +48:55.273 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/binding-rule from=127.0.0.1:43018 error="method HEAD not allowed" +48:55.273 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/binding-rule from=127.0.0.1:43018 latency=19.518µs +48:55.273 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/binding-rule from=127.0.0.1:43018 latency=2.034µs +48:55.274 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/connect/ca/roots from=127.0.0.1:43018 latency=561.342µs +48:55.274 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/connect/ca/roots from=127.0.0.1:43020 error="method PUT not allowed" +48:55.275 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/connect/ca/roots from=127.0.0.1:43020 latency=213.02µs +48:55.281 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/connect/ca/roots from=127.0.0.1:43022 error="method POST not allowed" +48:55.282 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/connect/ca/roots from=127.0.0.1:43022 latency=362.078µs +48:55.283 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/connect/ca/roots from=127.0.0.1:43024 error="method DELETE not allowed" +48:55.283 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/connect/ca/roots from=127.0.0.1:43024 latency=106.498µs +48:55.284 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/connect/ca/roots from=127.0.0.1:43026 error="method HEAD not allowed" +48:55.284 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/connect/ca/roots from=127.0.0.1:43026 latency=65.171µs +48:55.284 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/connect/ca/roots from=127.0.0.1:43026 latency=3.375µs +48:55.285 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/force-leave/ from=127.0.0.1:43026 error="method GET not allowed" +48:55.285 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/force-leave/ from=127.0.0.1:43026 latency=224.852µs +48:55.286 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/force-leave/ from=127.0.0.1:43028 error="Permission denied" +48:55.286 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/force-leave/ from=127.0.0.1:43028 latency=198.154µs +48:55.287 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/force-leave/ from=127.0.0.1:43030 error="method POST not allowed" +48:55.287 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/force-leave/ from=127.0.0.1:43030 latency=275.408µs +48:55.288 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/force-leave/ from=127.0.0.1:43032 error="method DELETE not allowed" +48:55.288 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/force-leave/ from=127.0.0.1:43032 latency=275.162µs +48:55.289 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/force-leave/ from=127.0.0.1:43034 error="method HEAD not allowed" +48:55.289 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/force-leave/ from=127.0.0.1:43034 latency=20.688µs +48:55.289 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/force-leave/ from=127.0.0.1:43034 latency=2.238µs +48:55.289 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/leave from=127.0.0.1:43034 error="method GET not allowed" +48:55.289 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/leave from=127.0.0.1:43034 latency=219.319µs +48:55.290 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/leave from=127.0.0.1:43036 error="Permission denied" +48:55.291 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/leave from=127.0.0.1:43036 latency=1.238581ms +48:55.292 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/leave from=127.0.0.1:43038 error="method POST not allowed" +48:55.292 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/leave from=127.0.0.1:43038 latency=180.604µs +48:55.293 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/leave from=127.0.0.1:43040 error="method DELETE not allowed" +48:55.293 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/leave from=127.0.0.1:43040 latency=140.395µs +48:55.293 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/leave from=127.0.0.1:43044 error="method HEAD not allowed" +48:55.293 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/leave from=127.0.0.1:43044 latency=15.566µs +48:55.294 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/leave from=127.0.0.1:43044 latency=1.344µs +48:55.294 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/fail/ from=127.0.0.1:43044 error="method GET not allowed" +48:55.294 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/fail/ from=127.0.0.1:43044 latency=160.991µs +48:55.294 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/check/fail/ from=127.0.0.1:43046 error="Unknown check """ +48:55.295 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/fail/ from=127.0.0.1:43046 latency=168.301µs +48:55.295 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/fail/ from=127.0.0.1:43048 error="method POST not allowed" +48:55.296 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/fail/ from=127.0.0.1:43048 latency=226.149µs +48:55.296 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/fail/ from=127.0.0.1:43050 error="method DELETE not allowed" +48:55.297 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/fail/ from=127.0.0.1:43050 latency=179.007µs +48:55.297 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/fail/ from=127.0.0.1:43054 error="method HEAD not allowed" +48:55.297 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/fail/ from=127.0.0.1:43054 latency=18.982µs +48:55.298 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/fail/ from=127.0.0.1:43054 latency=1.905µs +48:55.298 [ERROR] TestAgent.http: Request error: method=GET url=/v1/session/destroy/ from=127.0.0.1:43054 error="method GET not allowed" +48:55.298 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/destroy/ from=127.0.0.1:43054 latency=170.857µs +48:55.299 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/ from=127.0.0.1:43056 latency=185.902µs +48:55.300 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/destroy/ from=127.0.0.1:43058 error="method POST not allowed" +48:55.300 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/destroy/ from=127.0.0.1:43058 latency=158.608µs +48:55.300 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/destroy/ from=127.0.0.1:43060 error="method DELETE not allowed" +48:55.300 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/destroy/ from=127.0.0.1:43060 latency=151.684µs +48:55.301 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/destroy/ from=127.0.0.1:43062 error="method HEAD not allowed" +48:55.301 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/destroy/ from=127.0.0.1:43062 latency=15.161µs +48:55.301 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/destroy/ from=127.0.0.1:43062 latency=1.47µs +48:55.302 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/update/ from=127.0.0.1:43062 error="method GET not allowed" +48:55.302 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/update/ from=127.0.0.1:43062 latency=139.892µs +48:55.302 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/update/ from=127.0.0.1:43064 latency=97.347µs +48:55.303 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/update/ from=127.0.0.1:43066 error="method POST not allowed" +48:55.303 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/update/ from=127.0.0.1:43066 latency=192.517µs +48:55.303 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/update/ from=127.0.0.1:43068 error="method DELETE not allowed" +48:55.304 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/update/ from=127.0.0.1:43068 latency=170.936µs +48:55.304 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/update/ from=127.0.0.1:43070 error="method HEAD not allowed" +48:55.304 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/update/ from=127.0.0.1:43070 latency=24.982µs +48:55.304 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/update/ from=127.0.0.1:43070 latency=1.643µs +48:55.304 [ERROR] TestAgent.http: Request error: method=GET url=/v1/internal/acl/authorize from=127.0.0.1:43070 error="method GET not allowed" +48:55.305 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/acl/authorize from=127.0.0.1:43070 latency=155.248µs +48:55.305 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/acl/authorize from=127.0.0.1:43072 error="method PUT not allowed" +48:55.305 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/acl/authorize from=127.0.0.1:43072 latency=81.563µs +48:55.305 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/acl/authorize from=127.0.0.1:43074 error="Bad request: Failed to decode request body: EOF" +48:55.305 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/acl/authorize from=127.0.0.1:43074 latency=119.86µs +48:55.306 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/acl/authorize from=127.0.0.1:43076 error="method DELETE not allowed" +48:55.306 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/acl/authorize from=127.0.0.1:43076 latency=132.171µs +48:55.306 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/acl/authorize from=127.0.0.1:43078 error="method HEAD not allowed" +48:55.306 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/acl/authorize from=127.0.0.1:43078 latency=17.251µs +48:55.307 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/acl/authorize from=127.0.0.1:43078 latency=1.646µs +48:55.307 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/raft/peer from=127.0.0.1:43078 error="method GET not allowed" +48:55.307 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/raft/peer from=127.0.0.1:43078 latency=186.803µs +48:55.307 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/raft/peer from=127.0.0.1:43080 error="method PUT not allowed" +48:55.307 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/raft/peer from=127.0.0.1:43080 latency=98.889µs +48:55.308 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/raft/peer from=127.0.0.1:43082 error="method POST not allowed" +48:55.308 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/raft/peer from=127.0.0.1:43082 latency=168.169µs +48:55.320 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/raft/peer from=127.0.0.1:43084 error="Bad request: Must specify either ?id with the server's ID or ?address with IP:port of peer to remove" +48:55.320 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/raft/peer from=127.0.0.1:43084 latency=182.929µs +48:55.324 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/raft/peer from=127.0.0.1:43086 error="method HEAD not allowed" +48:55.324 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/raft/peer from=127.0.0.1:43086 latency=35.023µs +48:55.324 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/raft/peer from=127.0.0.1:43086 latency=2.147µs +48:55.325 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/keyring from=127.0.0.1:43086 error="Reading keyring denied by ACLs" +48:55.325 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:43086 latency=249.492µs +48:55.325 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/keyring from=127.0.0.1:43088 error="Bad request: Request decode failed: EOF" +48:55.325 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/keyring from=127.0.0.1:43088 latency=95.656µs +48:55.326 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/keyring from=127.0.0.1:43090 error="Bad request: Request decode failed: EOF" +48:55.326 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/keyring from=127.0.0.1:43090 latency=129.965µs +48:55.326 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/keyring from=127.0.0.1:43092 error="Bad request: Request decode failed: EOF" +48:55.326 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/keyring from=127.0.0.1:43092 latency=108.63µs +48:55.327 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/keyring from=127.0.0.1:43094 error="method HEAD not allowed" +48:55.327 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/keyring from=127.0.0.1:43094 latency=15.236µs +48:55.327 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/keyring from=127.0.0.1:43094 latency=2.032µs +48:55.327 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/login from=127.0.0.1:43094 error="method GET not allowed" +48:55.327 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/login from=127.0.0.1:43094 latency=160.984µs +48:55.328 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/login from=127.0.0.1:43096 error="method PUT not allowed" +48:55.328 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/login from=127.0.0.1:43096 latency=142.463µs +48:55.329 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/login from=127.0.0.1:43098 error="Bad request: Failed to decode request body: EOF" +48:55.329 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:43098 latency=111.905µs +48:55.329 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/login from=127.0.0.1:43100 error="method DELETE not allowed" +48:55.329 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/login from=127.0.0.1:43100 latency=114.984µs +48:55.330 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/login from=127.0.0.1:43102 error="method HEAD not allowed" +48:55.330 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/login from=127.0.0.1:43102 latency=33.752µs +48:55.330 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/login from=127.0.0.1:43102 latency=1.991µs +48:55.330 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:43102 latency=212.9µs +48:55.331 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/checks from=127.0.0.1:43104 error="method PUT not allowed" +48:55.331 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/checks from=127.0.0.1:43104 latency=145.502µs +48:55.331 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/checks from=127.0.0.1:43106 error="method POST not allowed" +48:55.332 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/checks from=127.0.0.1:43106 latency=162.485µs +48:55.332 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/checks from=127.0.0.1:43108 error="method DELETE not allowed" +48:55.332 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/checks from=127.0.0.1:43108 latency=131.997µs +48:55.333 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/checks from=127.0.0.1:43110 error="method HEAD not allowed" +48:55.333 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/checks from=127.0.0.1:43110 latency=17.359µs +48:55.333 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/checks from=127.0.0.1:43110 latency=1.458µs +48:55.333 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/gateway-services/ from=127.0.0.1:43110 latency=171.215µs +48:55.334 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/gateway-services/ from=127.0.0.1:43114 error="method PUT not allowed" +48:55.334 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/gateway-services/ from=127.0.0.1:43114 latency=123.19µs +48:55.334 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/gateway-services/ from=127.0.0.1:43116 error="method POST not allowed" +48:55.334 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/gateway-services/ from=127.0.0.1:43116 latency=188.03µs +48:55.335 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/gateway-services/ from=127.0.0.1:43118 error="method DELETE not allowed" +48:55.335 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/gateway-services/ from=127.0.0.1:43118 latency=104.192µs +48:55.335 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/gateway-services/ from=127.0.0.1:43120 error="method HEAD not allowed" +48:55.335 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/gateway-services/ from=127.0.0.1:43120 latency=18.362µs +48:55.336 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/gateway-services/ from=127.0.0.1:43120 latency=1.62µs +48:55.340 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/clone/ from=127.0.0.1:43120 error="method GET not allowed" +48:55.340 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/clone/ from=127.0.0.1:43120 latency=146.391µs +48:55.341 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/clone/ from=127.0.0.1:43122 latency=153.165µs +48:55.342 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/clone/ from=127.0.0.1:43124 error="method POST not allowed" +48:55.342 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/clone/ from=127.0.0.1:43124 latency=128.437µs +48:55.342 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/clone/ from=127.0.0.1:43128 error="method DELETE not allowed" +48:55.342 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/clone/ from=127.0.0.1:43128 latency=171.869µs +48:55.343 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/clone/ from=127.0.0.1:43130 error="method HEAD not allowed" +48:55.343 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/clone/ from=127.0.0.1:43130 latency=20.094µs +48:55.343 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/clone/ from=127.0.0.1:43130 latency=3.298µs +48:55.344 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/ from=127.0.0.1:43130 latency=252.267µs +48:55.344 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/service/ from=127.0.0.1:43132 error="method PUT not allowed" +48:55.345 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/ from=127.0.0.1:43132 latency=152.753µs +48:55.345 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/service/ from=127.0.0.1:43134 error="method POST not allowed" +48:55.345 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/service/ from=127.0.0.1:43134 latency=208.443µs +48:55.346 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/service/ from=127.0.0.1:43136 error="method DELETE not allowed" +48:55.346 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/service/ from=127.0.0.1:43136 latency=144.233µs +48:55.347 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/service/ from=127.0.0.1:43138 error="method HEAD not allowed" +48:55.347 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/service/ from=127.0.0.1:43138 latency=25.966µs +48:55.347 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/service/ from=127.0.0.1:43138 latency=3.155µs +48:55.348 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/connect/authorize from=127.0.0.1:43138 error="method GET not allowed" +48:55.348 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/connect/authorize from=127.0.0.1:43138 latency=201.955µs +48:55.348 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/connect/authorize from=127.0.0.1:43140 error="method PUT not allowed" +48:55.349 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/connect/authorize from=127.0.0.1:43140 latency=190.889µs +48:55.349 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:43142 error="Bad request: Request decode failed: EOF" +48:55.349 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:43142 latency=186.85µs +48:55.350 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/connect/authorize from=127.0.0.1:43144 error="method DELETE not allowed" +48:55.350 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/connect/authorize from=127.0.0.1:43144 latency=100.043µs +48:55.351 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/connect/authorize from=127.0.0.1:43148 error="method HEAD not allowed" +48:55.351 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/connect/authorize from=127.0.0.1:43148 latency=25.73µs +48:55.351 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/connect/authorize from=127.0.0.1:43148 latency=2.904µs +48:55.351 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/update from=127.0.0.1:43148 error="method GET not allowed" +48:55.351 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/update from=127.0.0.1:43148 latency=226.285µs +48:55.352 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/update from=127.0.0.1:43150 latency=160.304µs +48:55.353 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/update from=127.0.0.1:43154 error="method POST not allowed" +48:55.353 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/update from=127.0.0.1:43154 latency=168.614µs +48:55.354 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/update from=127.0.0.1:43156 error="method DELETE not allowed" +48:55.354 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/update from=127.0.0.1:43156 latency=113.433µs +48:55.354 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/update from=127.0.0.1:43158 error="method HEAD not allowed" +48:55.354 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/update from=127.0.0.1:43158 latency=16.96µs +48:55.355 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/update from=127.0.0.1:43158 latency=2.056µs +48:55.355 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policies from=127.0.0.1:43158 error="Permission denied" +48:55.356 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policies from=127.0.0.1:43158 latency=368.664µs +48:55.356 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/policies from=127.0.0.1:43162 error="method PUT not allowed" +48:55.357 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policies from=127.0.0.1:43162 latency=195.406µs +48:55.357 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/policies from=127.0.0.1:43164 error="method POST not allowed" +48:55.358 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/policies from=127.0.0.1:43164 latency=261.574µs +48:55.358 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/policies from=127.0.0.1:43166 error="method DELETE not allowed" +48:55.359 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policies from=127.0.0.1:43166 latency=187.543µs +48:55.360 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/policies from=127.0.0.1:43168 error="method HEAD not allowed" +48:55.360 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/policies from=127.0.0.1:43168 latency=25.639µs +48:55.360 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/policies from=127.0.0.1:43168 latency=2.488µs +48:55.361 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/self from=127.0.0.1:43168 error="Permission denied" +48:55.361 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:43168 latency=251.316µs +48:55.361 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/self from=127.0.0.1:43170 error="method PUT not allowed" +48:55.361 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/self from=127.0.0.1:43170 latency=174.229µs +48:55.362 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/self from=127.0.0.1:43172 error="method POST not allowed" +48:55.362 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/self from=127.0.0.1:43172 latency=166.601µs +48:55.363 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/self from=127.0.0.1:43174 error="method DELETE not allowed" +48:55.363 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/self from=127.0.0.1:43174 latency=253.487µs +48:55.364 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/self from=127.0.0.1:43176 error="method HEAD not allowed" +48:55.364 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/self from=127.0.0.1:43176 latency=14.685µs +48:55.364 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/self from=127.0.0.1:43176 latency=3.308µs +48:55.364 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/state/ from=127.0.0.1:43176 latency=129.108µs +48:55.365 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/state/ from=127.0.0.1:43178 error="method PUT not allowed" +48:55.365 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/state/ from=127.0.0.1:43178 latency=108.055µs +48:55.366 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/state/ from=127.0.0.1:43180 error="method POST not allowed" +48:55.366 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/state/ from=127.0.0.1:43180 latency=167.689µs +48:55.366 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/state/ from=127.0.0.1:43182 error="method DELETE not allowed" +48:55.367 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/state/ from=127.0.0.1:43182 latency=115.446µs +48:55.367 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/state/ from=127.0.0.1:43186 error="method HEAD not allowed" +48:55.367 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/state/ from=127.0.0.1:43186 latency=20.055µs +48:55.367 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/state/ from=127.0.0.1:43186 latency=3.318µs +48:55.368 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/ from=127.0.0.1:43186 latency=138.098µs +48:55.368 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/session/info/ from=127.0.0.1:43188 error="method PUT not allowed" +48:55.368 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/info/ from=127.0.0.1:43188 latency=183.378µs +48:55.369 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/info/ from=127.0.0.1:43190 error="method POST not allowed" +48:55.369 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/info/ from=127.0.0.1:43190 latency=214.451µs +48:55.370 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/info/ from=127.0.0.1:43192 error="method DELETE not allowed" +48:55.370 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/info/ from=127.0.0.1:43192 latency=103.298µs +48:55.370 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/info/ from=127.0.0.1:43194 error="method HEAD not allowed" +48:55.370 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/info/ from=127.0.0.1:43194 latency=16.351µs +48:55.371 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/info/ from=127.0.0.1:43194 latency=1.975µs +48:55.371 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/role/ from=127.0.0.1:43194 error="Bad request: Missing role ID" +48:55.371 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/ from=127.0.0.1:43194 latency=128.415µs +48:55.371 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/role/ from=127.0.0.1:43196 error="Bad request: Role decoding failed: EOF" +48:55.372 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/ from=127.0.0.1:43196 latency=189.269µs +48:55.372 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/role/ from=127.0.0.1:43198 error="method POST not allowed" +48:55.373 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/role/ from=127.0.0.1:43198 latency=206.064µs +48:55.373 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/role/ from=127.0.0.1:43200 error="Bad request: Missing role ID" +48:55.373 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role/ from=127.0.0.1:43200 latency=97.034µs +48:55.374 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/role/ from=127.0.0.1:43202 error="method HEAD not allowed" +48:55.374 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/role/ from=127.0.0.1:43202 latency=16.665µs +48:55.374 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/role/ from=127.0.0.1:43202 latency=1.935µs +48:55.374 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/datacenters from=127.0.0.1:43202 latency=397.535µs +48:55.375 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/coordinate/datacenters from=127.0.0.1:43204 error="method PUT not allowed" +48:55.375 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/coordinate/datacenters from=127.0.0.1:43204 latency=145.744µs +48:55.376 [ERROR] TestAgent.http: Request error: method=POST url=/v1/coordinate/datacenters from=127.0.0.1:43208 error="method POST not allowed" +48:55.376 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/coordinate/datacenters from=127.0.0.1:43208 latency=204.459µs +48:55.376 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/coordinate/datacenters from=127.0.0.1:43210 error="method DELETE not allowed" +48:55.377 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/coordinate/datacenters from=127.0.0.1:43210 latency=193.256µs +48:55.377 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/coordinate/datacenters from=127.0.0.1:43212 error="method HEAD not allowed" +48:55.377 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/coordinate/datacenters from=127.0.0.1:43212 latency=21.983µs +48:55.377 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/coordinate/datacenters from=127.0.0.1:43212 latency=2.782µs +48:55.378 [ERROR] TestAgent.http: Request error: method=GET url=/v1/internal/federation-state/ from=127.0.0.1:43212 error="Bad request: Missing datacenter name" +48:55.378 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/federation-state/ from=127.0.0.1:43212 latency=190.17µs +48:55.378 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/federation-state/ from=127.0.0.1:43214 error="method PUT not allowed" +48:55.379 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/federation-state/ from=127.0.0.1:43214 latency=112.002µs +48:55.379 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/federation-state/ from=127.0.0.1:43216 error="method POST not allowed" +48:55.379 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/federation-state/ from=127.0.0.1:43216 latency=104.292µs +48:55.380 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/federation-state/ from=127.0.0.1:43218 error="method DELETE not allowed" +48:55.380 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/federation-state/ from=127.0.0.1:43218 latency=237.08µs +48:55.380 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/federation-state/ from=127.0.0.1:43220 error="method HEAD not allowed" +48:55.380 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/federation-state/ from=127.0.0.1:43220 latency=17.339µs +48:55.381 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/federation-state/ from=127.0.0.1:43220 latency=2.006µs +48:55.381 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/node/ from=127.0.0.1:43220 latency=116.376µs +48:55.382 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/node/ from=127.0.0.1:43222 error="method PUT not allowed" +48:55.382 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/node/ from=127.0.0.1:43222 latency=146.687µs +48:55.382 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/node/ from=127.0.0.1:43224 error="method POST not allowed" +48:55.382 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/node/ from=127.0.0.1:43224 latency=221.399µs +48:55.383 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/node/ from=127.0.0.1:43226 error="method DELETE not allowed" +48:55.383 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/node/ from=127.0.0.1:43226 latency=133.947µs +48:55.384 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/node/ from=127.0.0.1:43228 error="method HEAD not allowed" +48:55.384 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/node/ from=127.0.0.1:43228 latency=17.684µs +48:55.384 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/node/ from=127.0.0.1:43228 latency=1.923µs +48:55.384 [ERROR] TestAgent.http: Request error: method=GET url=/v1/session/create from=127.0.0.1:43228 error="method GET not allowed" +48:55.384 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/create from=127.0.0.1:43228 latency=102.613µs +48:55.385 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/session/create from=127.0.0.1:43230 error="Permission denied" +48:55.385 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:43230 latency=242.462µs +48:55.385 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/create from=127.0.0.1:43232 error="method POST not allowed" +48:55.385 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/create from=127.0.0.1:43232 latency=83.726µs +48:55.386 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/create from=127.0.0.1:43234 error="method DELETE not allowed" +48:55.386 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/create from=127.0.0.1:43234 latency=142.647µs +48:55.387 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/create from=127.0.0.1:43236 error="method HEAD not allowed" +48:55.387 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/create from=127.0.0.1:43236 latency=16.891µs +48:55.387 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/create from=127.0.0.1:43236 latency=1.615µs +48:55.387 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/create from=127.0.0.1:43236 error="method GET not allowed" +48:55.387 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/create from=127.0.0.1:43236 latency=180.19µs +48:55.388 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/create from=127.0.0.1:43238 error="Permission denied" +48:55.388 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/create from=127.0.0.1:43238 latency=160.414µs +48:55.388 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/create from=127.0.0.1:43240 error="method POST not allowed" +48:55.388 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/create from=127.0.0.1:43240 latency=124.467µs +48:55.389 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/create from=127.0.0.1:43242 error="method DELETE not allowed" +48:55.389 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/create from=127.0.0.1:43242 latency=187.465µs +48:55.390 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/create from=127.0.0.1:43244 error="method HEAD not allowed" +48:55.390 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/create from=127.0.0.1:43244 latency=18.523µs +48:55.390 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/create from=127.0.0.1:43244 latency=1.904µs +48:55.390 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0d456efc-6231-9ceb-2643-32c992012de5 +48:55.391 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:43244 latency=352.194µs +48:55.391 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/nodes from=127.0.0.1:43246 error="method PUT not allowed" +48:55.391 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/nodes from=127.0.0.1:43246 latency=119.035µs +48:55.392 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/nodes from=127.0.0.1:43248 error="method POST not allowed" +48:55.392 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/nodes from=127.0.0.1:43248 latency=229.537µs +48:55.392 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/nodes from=127.0.0.1:43250 error="method DELETE not allowed" +48:55.393 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/nodes from=127.0.0.1:43250 latency=154.56µs +48:55.393 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/nodes from=127.0.0.1:43254 error="method HEAD not allowed" +48:55.393 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/nodes from=127.0.0.1:43254 latency=18.922µs +48:55.394 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/nodes from=127.0.0.1:43254 latency=2.068µs +48:55.394 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/ from=127.0.0.1:43254 latency=150.168µs +48:55.394 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/service/ from=127.0.0.1:43256 error="method PUT not allowed" +48:55.395 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/service/ from=127.0.0.1:43256 latency=123.001µs +48:55.395 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/service/ from=127.0.0.1:43258 error="method POST not allowed" +48:55.395 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/service/ from=127.0.0.1:43258 latency=122.359µs +48:55.396 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/service/ from=127.0.0.1:43260 error="method DELETE not allowed" +48:55.396 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/service/ from=127.0.0.1:43260 latency=200.731µs +48:55.397 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/service/ from=127.0.0.1:43262 error="method HEAD not allowed" +48:55.397 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/service/ from=127.0.0.1:43262 latency=31.76µs +48:55.397 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/service/ from=127.0.0.1:43262 latency=1.998µs +48:55.397 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/destroy/ from=127.0.0.1:43262 error="method GET not allowed" +48:55.397 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/destroy/ from=127.0.0.1:43262 latency=137.028µs +48:55.398 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/destroy/ from=127.0.0.1:43264 latency=90.699µs +48:55.398 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/destroy/ from=127.0.0.1:43266 error="method POST not allowed" +48:55.398 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/destroy/ from=127.0.0.1:43266 latency=92.845µs +48:55.399 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/destroy/ from=127.0.0.1:43268 error="method DELETE not allowed" +48:55.399 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/destroy/ from=127.0.0.1:43268 latency=153.573µs +48:55.400 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/destroy/ from=127.0.0.1:43270 error="method HEAD not allowed" +48:55.400 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/destroy/ from=127.0.0.1:43270 latency=21.335µs +48:55.400 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/destroy/ from=127.0.0.1:43270 latency=1.753µs +48:55.400 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/role/name/ from=127.0.0.1:43270 error="Bad request: Missing role Name" +48:55.400 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/name/ from=127.0.0.1:43270 latency=178.4µs +48:55.401 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/role/name/ from=127.0.0.1:43274 error="method PUT not allowed" +48:55.401 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/name/ from=127.0.0.1:43274 latency=148.364µs +48:55.402 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/role/name/ from=127.0.0.1:43276 error="method POST not allowed" +48:55.402 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/role/name/ from=127.0.0.1:43276 latency=168.978µs +48:55.402 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/role/name/ from=127.0.0.1:43278 error="method DELETE not allowed" +48:55.402 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role/name/ from=127.0.0.1:43278 latency=99.585µs +48:55.403 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/role/name/ from=127.0.0.1:43280 error="method HEAD not allowed" +48:55.403 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/role/name/ from=127.0.0.1:43280 latency=23.369µs +48:55.403 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/role/name/ from=127.0.0.1:43280 latency=2.679µs +48:55.404 [ERROR] TestAgent.http: Request error: method=GET url=/v1/coordinate/node/ from=127.0.0.1:43280 error="Permission denied" +48:55.404 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/node/ from=127.0.0.1:43280 latency=348.807µs +48:55.404 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/coordinate/node/ from=127.0.0.1:43284 error="method PUT not allowed" +48:55.404 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/coordinate/node/ from=127.0.0.1:43284 latency=129.834µs +48:55.405 [ERROR] TestAgent.http: Request error: method=POST url=/v1/coordinate/node/ from=127.0.0.1:43286 error="method POST not allowed" +48:55.405 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/coordinate/node/ from=127.0.0.1:43286 latency=205.273µs +48:55.406 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/coordinate/node/ from=127.0.0.1:43288 error="method DELETE not allowed" +48:55.406 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/coordinate/node/ from=127.0.0.1:43288 latency=146.954µs +48:55.406 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/coordinate/node/ from=127.0.0.1:43290 error="method HEAD not allowed" +48:55.406 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/coordinate/node/ from=127.0.0.1:43290 latency=22.801µs +48:55.407 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/coordinate/node/ from=127.0.0.1:43290 latency=2.871µs +48:55.407 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/autopilot/health from=127.0.0.1:43290 error="Permission denied" +48:55.407 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/autopilot/health from=127.0.0.1:43290 latency=289.834µs +48:55.408 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/autopilot/health from=127.0.0.1:43292 error="method PUT not allowed" +48:55.408 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/autopilot/health from=127.0.0.1:43292 latency=95.752µs +48:55.408 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/autopilot/health from=127.0.0.1:43294 error="method POST not allowed" +48:55.409 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/autopilot/health from=127.0.0.1:43294 latency=93.755µs +48:55.409 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/autopilot/health from=127.0.0.1:43296 error="method DELETE not allowed" +48:55.409 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/autopilot/health from=127.0.0.1:43296 latency=148.908µs +48:55.410 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/autopilot/health from=127.0.0.1:43298 error="method HEAD not allowed" +48:55.410 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/autopilot/health from=127.0.0.1:43298 latency=22.596µs +48:55.410 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/autopilot/health from=127.0.0.1:43298 latency=2.08µs +48:55.411 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/list from=127.0.0.1:43298 latency=422.456µs +48:55.411 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/session/list from=127.0.0.1:43300 error="method PUT not allowed" +48:55.411 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/list from=127.0.0.1:43300 latency=223.814µs +48:55.412 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/list from=127.0.0.1:43302 error="method POST not allowed" +48:55.412 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/list from=127.0.0.1:43302 latency=187.022µs +48:55.413 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/list from=127.0.0.1:43304 error="method DELETE not allowed" +48:55.413 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/list from=127.0.0.1:43304 latency=247.363µs +48:55.414 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/list from=127.0.0.1:43306 error="method HEAD not allowed" +48:55.414 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/list from=127.0.0.1:43306 latency=26.144µs +48:55.414 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/list from=127.0.0.1:43306 latency=2.392µs +48:55.414 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/token/ from=127.0.0.1:43306 error="method GET not allowed" +48:55.414 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/token/ from=127.0.0.1:43306 latency=233.399µs +48:55.415 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/token/ from=127.0.0.1:43308 error="Permission denied" +48:55.415 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/ from=127.0.0.1:43308 latency=176.906µs +48:55.416 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/token/ from=127.0.0.1:43310 error="method POST not allowed" +48:55.416 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/token/ from=127.0.0.1:43310 latency=218.815µs +48:55.416 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/token/ from=127.0.0.1:43312 error="method DELETE not allowed" +48:55.416 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/token/ from=127.0.0.1:43312 latency=278.184µs +48:55.417 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/token/ from=127.0.0.1:43314 error="method HEAD not allowed" +48:55.417 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/token/ from=127.0.0.1:43314 latency=26.644µs +48:55.424 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/token/ from=127.0.0.1:43314 latency=3.444µs +48:55.425 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0d456efc-6231-9ceb-2643-32c992012de5 +48:55.425 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/ui/services from=127.0.0.1:43314 latency=549.057µs +48:55.426 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/ui/services from=127.0.0.1:43316 error="method PUT not allowed" +48:55.426 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/ui/services from=127.0.0.1:43316 latency=235.255µs +48:55.427 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/ui/services from=127.0.0.1:43318 error="method POST not allowed" +48:55.427 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/ui/services from=127.0.0.1:43318 latency=209.719µs +48:55.428 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/ui/services from=127.0.0.1:43320 error="method DELETE not allowed" +48:55.428 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/ui/services from=127.0.0.1:43320 latency=158.504µs +48:55.429 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/ui/services from=127.0.0.1:43322 error="method HEAD not allowed" +48:55.429 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/ui/services from=127.0.0.1:43322 latency=32.597µs +48:55.429 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/ui/services from=127.0.0.1:43322 latency=1.913µs +48:55.429 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:43322 latency=140.591µs +48:55.430 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:43324 error="method PUT not allowed" +48:55.430 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:43324 latency=154.311µs +48:55.430 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:43326 error="method POST not allowed" +48:55.431 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:43326 latency=210.606µs +48:55.431 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:43328 error="method DELETE not allowed" +48:55.431 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:43328 latency=194.873µs +48:55.432 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:43330 error="method HEAD not allowed" +48:55.432 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:43330 latency=23.401µs +48:55.432 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/ui/gateway-services-nodes/ from=127.0.0.1:43330 latency=1.975µs +48:55.432 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/service/deregister/ from=127.0.0.1:43330 error="method GET not allowed" +48:55.433 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/deregister/ from=127.0.0.1:43330 latency=162.328µs +48:55.433 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/service/deregister/ from=127.0.0.1:43332 error="Unknown service {"" {}}" +48:55.433 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/ from=127.0.0.1:43332 latency=274.488µs +48:55.434 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/service/deregister/ from=127.0.0.1:43336 error="method POST not allowed" +48:55.434 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/service/deregister/ from=127.0.0.1:43336 latency=168.509µs +48:55.435 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/service/deregister/ from=127.0.0.1:43338 error="method DELETE not allowed" +48:55.435 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/service/deregister/ from=127.0.0.1:43338 latency=110.894µs +48:55.435 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/service/deregister/ from=127.0.0.1:43340 error="method HEAD not allowed" +48:55.435 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/service/deregister/ from=127.0.0.1:43340 latency=20.711µs +48:55.435 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/service/deregister/ from=127.0.0.1:43340 latency=2.135µs +48:55.436 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/node/ from=127.0.0.1:43340 latency=109.407µs +48:55.436 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/session/node/ from=127.0.0.1:43342 error="method PUT not allowed" +48:55.436 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/node/ from=127.0.0.1:43342 latency=136.159µs +48:55.437 [ERROR] TestAgent.http: Request error: method=POST url=/v1/session/node/ from=127.0.0.1:43346 error="method POST not allowed" +48:55.437 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/session/node/ from=127.0.0.1:43346 latency=138.47µs +48:55.438 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/session/node/ from=127.0.0.1:43348 error="method DELETE not allowed" +48:55.438 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/session/node/ from=127.0.0.1:43348 latency=139.731µs +48:55.438 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/session/node/ from=127.0.0.1:43350 error="method HEAD not allowed" +48:55.438 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/session/node/ from=127.0.0.1:43350 latency=18.43µs +48:55.439 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/session/node/ from=127.0.0.1:43350 latency=1.798µs +48:55.439 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policy/name/ from=127.0.0.1:43350 error="Bad request: Missing policy Name" +48:55.439 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/name/ from=127.0.0.1:43350 latency=123.006µs +48:55.440 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/policy/name/ from=127.0.0.1:43352 error="method PUT not allowed" +48:55.440 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy/name/ from=127.0.0.1:43352 latency=112.342µs +48:55.440 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/policy/name/ from=127.0.0.1:43354 error="method POST not allowed" +48:55.441 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/policy/name/ from=127.0.0.1:43354 latency=208.253µs +48:55.441 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/policy/name/ from=127.0.0.1:43356 error="method DELETE not allowed" +48:55.441 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policy/name/ from=127.0.0.1:43356 latency=137.699µs +48:55.442 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/policy/name/ from=127.0.0.1:43358 error="method HEAD not allowed" +48:55.442 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/policy/name/ from=127.0.0.1:43358 latency=19.206µs +48:55.442 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/policy/name/ from=127.0.0.1:43358 latency=1.782µs +48:55.442 [ERROR] TestAgent.http: Request error: method=GET url=/v1/config from=127.0.0.1:43358 error="method GET not allowed" +48:55.442 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config from=127.0.0.1:43358 latency=162.741µs +48:55.443 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/config from=127.0.0.1:43360 error="Bad request: Request decoding failed: EOF" +48:55.443 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:43360 latency=146.725µs +48:55.444 [ERROR] TestAgent.http: Request error: method=POST url=/v1/config from=127.0.0.1:43362 error="method POST not allowed" +48:55.444 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/config from=127.0.0.1:43362 latency=124.925µs +48:55.444 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/config from=127.0.0.1:43364 error="method DELETE not allowed" +48:55.445 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/config from=127.0.0.1:43364 latency=182.828µs +48:55.445 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/config from=127.0.0.1:43366 error="method HEAD not allowed" +48:55.445 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/config from=127.0.0.1:43366 latency=16.889µs +48:55.446 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/config from=127.0.0.1:43366 latency=2.519µs +48:55.446 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/ from=127.0.0.1:43366 latency=109.057µs +48:55.447 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/ from=127.0.0.1:43368 latency=149.561µs +48:55.447 [ERROR] TestAgent.http: Request error: method=POST url=/v1/kv/ from=127.0.0.1:43370 error="method POST not allowed" +48:55.447 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/kv/ from=127.0.0.1:43370 latency=201.906µs +48:55.448 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/ from=127.0.0.1:43372 latency=165.026µs +48:55.449 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/kv/ from=127.0.0.1:43374 error="method HEAD not allowed" +48:55.449 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/kv/ from=127.0.0.1:43374 latency=22.844µs +48:55.449 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/kv/ from=127.0.0.1:43374 latency=1.541µs +48:55.449 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/binding-rule/ from=127.0.0.1:43374 error="Bad request: Missing binding rule ID" +48:55.449 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/ from=127.0.0.1:43374 latency=190.988µs +48:55.450 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule/ from=127.0.0.1:43376 error="Bad request: BindingRule decoding failed: EOF" +48:55.450 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/ from=127.0.0.1:43376 latency=184.994µs +48:55.450 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/binding-rule/ from=127.0.0.1:43378 error="method POST not allowed" +48:55.451 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/binding-rule/ from=127.0.0.1:43378 latency=139.803µs +48:55.451 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/binding-rule/ from=127.0.0.1:43380 error="Bad request: Missing binding rule ID" +48:55.451 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/ from=127.0.0.1:43380 latency=131.973µs +48:55.452 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/binding-rule/ from=127.0.0.1:43382 error="method HEAD not allowed" +48:55.452 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/binding-rule/ from=127.0.0.1:43382 latency=18.629µs +48:55.456 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/binding-rule/ from=127.0.0.1:43382 latency=3.513µs +48:55.456 [ERROR] TestAgent.http: Request error: method=GET url=/v1/discovery-chain/ from=127.0.0.1:43382 error="Bad request: Missing chain name" +48:55.456 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/discovery-chain/ from=127.0.0.1:43382 latency=207.835µs +48:55.457 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/discovery-chain/ from=127.0.0.1:43384 error="method PUT not allowed" +48:55.457 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/discovery-chain/ from=127.0.0.1:43384 latency=185.438µs +48:55.458 [ERROR] TestAgent.http: Request error: method=POST url=/v1/discovery-chain/ from=127.0.0.1:43386 error="Bad request: Missing chain name" +48:55.458 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/discovery-chain/ from=127.0.0.1:43386 latency=127.222µs +48:55.459 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/discovery-chain/ from=127.0.0.1:43388 error="method DELETE not allowed" +48:55.459 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/discovery-chain/ from=127.0.0.1:43388 latency=103.459µs +48:55.459 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/discovery-chain/ from=127.0.0.1:43390 error="method HEAD not allowed" +48:55.459 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/discovery-chain/ from=127.0.0.1:43390 latency=19.791µs +48:55.459 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/discovery-chain/ from=127.0.0.1:43390 latency=2.674µs +48:55.460 [ERROR] TestAgent.http: Request error: method=GET url=/v1/event/fire/ from=127.0.0.1:43390 error="method GET not allowed" +48:55.460 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/event/fire/ from=127.0.0.1:43390 latency=233.264µs +48:55.461 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/ from=127.0.0.1:43392 latency=186.616µs +48:55.461 [ERROR] TestAgent.http: Request error: method=POST url=/v1/event/fire/ from=127.0.0.1:43394 error="method POST not allowed" +48:55.461 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/event/fire/ from=127.0.0.1:43394 latency=263.92µs +48:55.462 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/event/fire/ from=127.0.0.1:43398 error="method DELETE not allowed" +48:55.462 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/event/fire/ from=127.0.0.1:43398 latency=174.402µs +48:55.463 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/event/fire/ from=127.0.0.1:43400 error="method HEAD not allowed" +48:55.463 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/event/fire/ from=127.0.0.1:43400 latency=26.972µs +48:55.463 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/event/fire/ from=127.0.0.1:43400 latency=1.792µs +48:55.463 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:43400 error="URI must be either service or agent" +48:55.463 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:43400 latency=332.459µs +48:55.464 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:43402 error="method PUT not allowed" +48:55.464 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:43402 latency=200.277µs +48:55.465 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:43404 error="method POST not allowed" +48:55.465 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:43404 latency=126.174µs +48:55.466 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:43406 error="method DELETE not allowed" +48:55.466 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:43406 latency=240.234µs +48:55.467 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:43408 error="method HEAD not allowed" +48:55.467 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:43408 latency=44.674µs +48:55.467 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/connect/ca/leaf/ from=127.0.0.1:43408 latency=2.158µs +48:55.467 [ERROR] TestAgent.http: Request error: method=GET url=/v1/connect/intentions/check from=127.0.0.1:43408 error="required query parameter 'source' not set" +48:55.467 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/check from=127.0.0.1:43408 latency=145.315µs +48:55.468 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/intentions/check from=127.0.0.1:43410 error="method PUT not allowed" +48:55.468 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions/check from=127.0.0.1:43410 latency=251.154µs +48:55.469 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions/check from=127.0.0.1:43412 error="method POST not allowed" +48:55.469 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions/check from=127.0.0.1:43412 latency=149.301µs +48:55.469 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/intentions/check from=127.0.0.1:43414 error="method DELETE not allowed" +48:55.469 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions/check from=127.0.0.1:43414 latency=139.173µs +48:55.470 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/intentions/check from=127.0.0.1:43416 error="method HEAD not allowed" +48:55.470 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/intentions/check from=127.0.0.1:43416 latency=27.659µs +48:55.470 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/intentions/check from=127.0.0.1:43416 latency=4.357µs +48:55.470 [ERROR] TestAgent.http: Request error: method=GET url=/v1/catalog/register from=127.0.0.1:43416 error="method GET not allowed" +48:55.471 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/register from=127.0.0.1:43416 latency=124.027µs +48:55.471 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/register from=127.0.0.1:43418 latency=105.509µs +48:55.472 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/register from=127.0.0.1:43420 error="method POST not allowed" +48:55.472 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/register from=127.0.0.1:43420 latency=110.777µs +48:55.472 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/register from=127.0.0.1:43422 error="method DELETE not allowed" +48:55.472 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/register from=127.0.0.1:43422 latency=148.963µs +48:55.473 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/register from=127.0.0.1:43424 error="method HEAD not allowed" +48:55.473 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/register from=127.0.0.1:43424 latency=21.992µs +48:55.473 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/register from=127.0.0.1:43424 latency=1.967µs +48:55.474 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/connect/ from=127.0.0.1:43424 latency=104.02µs +48:55.474 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/connect/ from=127.0.0.1:43426 error="method PUT not allowed" +48:55.474 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/connect/ from=127.0.0.1:43426 latency=137.816µs +48:55.475 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/connect/ from=127.0.0.1:43428 error="method POST not allowed" +48:55.475 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/connect/ from=127.0.0.1:43428 latency=128.75µs +48:55.476 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/connect/ from=127.0.0.1:43430 error="method DELETE not allowed" +48:55.476 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/connect/ from=127.0.0.1:43430 latency=123.208µs +48:55.476 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/connect/ from=127.0.0.1:43432 error="method HEAD not allowed" +48:55.476 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/connect/ from=127.0.0.1:43432 latency=18.209µs +48:55.477 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/connect/ from=127.0.0.1:43432 latency=1.77µs +48:55.477 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/raft/configuration from=127.0.0.1:43432 error="Permission denied" +48:55.477 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/raft/configuration from=127.0.0.1:43432 latency=229.583µs +48:55.478 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/raft/configuration from=127.0.0.1:43434 error="method PUT not allowed" +48:55.478 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/raft/configuration from=127.0.0.1:43434 latency=128.206µs +48:55.478 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/raft/configuration from=127.0.0.1:43436 error="method POST not allowed" +48:55.478 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/raft/configuration from=127.0.0.1:43436 latency=103.028µs +48:55.479 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/raft/configuration from=127.0.0.1:43438 error="method DELETE not allowed" +48:55.479 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/raft/configuration from=127.0.0.1:43438 latency=139.012µs +48:55.480 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/raft/configuration from=127.0.0.1:43440 error="method HEAD not allowed" +48:55.480 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/raft/configuration from=127.0.0.1:43440 latency=23.583µs +48:55.480 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/raft/configuration from=127.0.0.1:43440 latency=2.275µs +48:55.480 [ERROR] TestAgent.http: Request error: method=GET url=/v1/operator/autopilot/configuration from=127.0.0.1:43440 error="Permission denied" +48:55.481 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/autopilot/configuration from=127.0.0.1:43440 latency=202.667µs +48:55.481 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/operator/autopilot/configuration from=127.0.0.1:43442 error="Bad request: Error parsing autopilot config: EOF" +48:55.481 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/autopilot/configuration from=127.0.0.1:43442 latency=186.215µs +48:55.482 [ERROR] TestAgent.http: Request error: method=POST url=/v1/operator/autopilot/configuration from=127.0.0.1:43444 error="method POST not allowed" +48:55.482 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/autopilot/configuration from=127.0.0.1:43444 latency=170.026µs +48:55.483 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/autopilot/configuration from=127.0.0.1:43446 error="method DELETE not allowed" +48:55.483 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/autopilot/configuration from=127.0.0.1:43446 latency=130.06µs +48:55.483 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/operator/autopilot/configuration from=127.0.0.1:43448 error="method HEAD not allowed" +48:55.483 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/operator/autopilot/configuration from=127.0.0.1:43448 latency=21.918µs +48:55.484 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/operator/autopilot/configuration from=127.0.0.1:43448 latency=1.898µs +48:55.484 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/metrics from=127.0.0.1:43448 error="Permission denied" +48:55.484 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/metrics from=127.0.0.1:43448 latency=163.586µs +48:55.485 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/metrics from=127.0.0.1:43450 error="method PUT not allowed" +48:55.485 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/metrics from=127.0.0.1:43450 latency=151.698µs +48:55.485 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/metrics from=127.0.0.1:43452 error="method POST not allowed" +48:55.486 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/metrics from=127.0.0.1:43452 latency=224.483µs +48:55.486 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/metrics from=127.0.0.1:43454 error="method DELETE not allowed" +48:55.487 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/metrics from=127.0.0.1:43454 latency=205.936µs +48:55.487 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/metrics from=127.0.0.1:43456 error="method HEAD not allowed" +48:55.487 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/metrics from=127.0.0.1:43456 latency=38.416µs +48:55.487 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/metrics from=127.0.0.1:43456 latency=2.227µs +48:55.488 [DEBUG] TestAgent: dropping node from result due to ACLs: node=Node-0d456efc-6231-9ceb-2643-32c992012de5 accessorID=00000000-0000-0000-0000-000000000002 +48:55.488 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members from=127.0.0.1:43456 latency=297.69µs +48:55.488 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/members from=127.0.0.1:43458 error="method PUT not allowed" +48:55.489 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/members from=127.0.0.1:43458 latency=142.917µs +48:55.489 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/members from=127.0.0.1:43460 error="method POST not allowed" +48:55.489 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/members from=127.0.0.1:43460 latency=194.202µs +48:55.490 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/members from=127.0.0.1:43462 error="method DELETE not allowed" +48:55.490 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/members from=127.0.0.1:43462 latency=122.498µs +48:55.491 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/members from=127.0.0.1:43466 error="method HEAD not allowed" +48:55.491 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/members from=127.0.0.1:43466 latency=23.004µs +48:55.491 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/members from=127.0.0.1:43466 latency=2.154µs +48:55.491 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/deregister/ from=127.0.0.1:43466 error="method GET not allowed" +48:55.491 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/deregister/ from=127.0.0.1:43466 latency=173.074µs +48:55.492 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/check/deregister/ from=127.0.0.1:43468 error="Unknown check """ +48:55.492 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/deregister/ from=127.0.0.1:43468 latency=179.701µs +48:55.492 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/deregister/ from=127.0.0.1:43470 error="method POST not allowed" +48:55.493 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/deregister/ from=127.0.0.1:43470 latency=158.448µs +48:55.493 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/deregister/ from=127.0.0.1:43472 error="method DELETE not allowed" +48:55.493 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/deregister/ from=127.0.0.1:43472 latency=181.738µs +48:55.494 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/deregister/ from=127.0.0.1:43474 error="method HEAD not allowed" +48:55.494 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/deregister/ from=127.0.0.1:43474 latency=17.219µs +48:55.494 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/deregister/ from=127.0.0.1:43474 latency=1.896µs +48:55.495 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:43474 latency=209.458µs +48:55.495 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:43476 error="method PUT not allowed" +48:55.496 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:43476 latency=178.452µs +48:55.496 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:43478 error="method POST not allowed" +48:55.496 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:43478 latency=124.835µs +48:55.497 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:43480 error="method DELETE not allowed" +48:55.497 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:43480 latency=97.48µs +48:55.497 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:43482 error="method HEAD not allowed" +48:55.497 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:43482 latency=22.885µs +48:55.498 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/federation-states/mesh-gateways from=127.0.0.1:43482 latency=2.037µs +48:55.498 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/logout from=127.0.0.1:43482 error="method GET not allowed" +48:55.498 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/logout from=127.0.0.1:43482 latency=168.926µs +48:55.499 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/logout from=127.0.0.1:43484 error="method PUT not allowed" +48:55.499 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/logout from=127.0.0.1:43484 latency=98.466µs +48:55.499 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:43486 error="ACL not found" +48:55.499 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:43486 latency=207.897µs +48:55.500 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/logout from=127.0.0.1:43488 error="method DELETE not allowed" +48:55.500 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/logout from=127.0.0.1:43488 latency=168.977µs +48:55.501 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/logout from=127.0.0.1:43490 error="method HEAD not allowed" +48:55.501 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/logout from=127.0.0.1:43490 latency=22.903µs +48:55.501 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/logout from=127.0.0.1:43490 latency=1.718µs +48:55.501 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/health/service/id/ from=127.0.0.1:43490 error="Bad request: Missing serviceID" +48:55.502 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/health/service/id/ from=127.0.0.1:43490 latency=142.615µs +48:55.502 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/health/service/id/ from=127.0.0.1:43492 error="method PUT not allowed" +48:55.502 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/health/service/id/ from=127.0.0.1:43492 latency=169.164µs +48:55.503 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/health/service/id/ from=127.0.0.1:43494 error="method POST not allowed" +48:55.503 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/health/service/id/ from=127.0.0.1:43494 latency=162.598µs +48:55.503 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/health/service/id/ from=127.0.0.1:43496 error="method DELETE not allowed" +48:55.504 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/health/service/id/ from=127.0.0.1:43496 latency=104.436µs +48:55.504 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/health/service/id/ from=127.0.0.1:43498 error="method HEAD not allowed" +48:55.504 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/health/service/id/ from=127.0.0.1:43498 latency=17.806µs +48:55.504 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/health/service/id/ from=127.0.0.1:43498 latency=1.508µs +48:55.504 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/pass/ from=127.0.0.1:43498 error="method GET not allowed" +48:55.505 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/pass/ from=127.0.0.1:43498 latency=166.06µs +48:55.505 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/check/pass/ from=127.0.0.1:43500 error="Unknown check """ +48:55.505 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/pass/ from=127.0.0.1:43500 latency=136.464µs +48:55.506 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/pass/ from=127.0.0.1:43502 error="method POST not allowed" +48:55.506 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/pass/ from=127.0.0.1:43502 latency=126.968µs +48:55.506 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/pass/ from=127.0.0.1:43504 error="method DELETE not allowed" +48:55.507 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/pass/ from=127.0.0.1:43504 latency=136.203µs +48:55.507 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/pass/ from=127.0.0.1:43506 error="method HEAD not allowed" +48:55.507 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/pass/ from=127.0.0.1:43506 latency=16.022µs +48:55.507 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/pass/ from=127.0.0.1:43506 latency=1.75µs +48:55.508 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/replication from=127.0.0.1:43506 latency=249.088µs +48:55.508 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/replication from=127.0.0.1:43508 error="method PUT not allowed" +48:55.509 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/replication from=127.0.0.1:43508 latency=174.331µs +48:55.509 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/replication from=127.0.0.1:43510 error="method POST not allowed" +48:55.509 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/replication from=127.0.0.1:43510 latency=164.47µs +48:55.510 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/replication from=127.0.0.1:43512 error="method DELETE not allowed" +48:55.510 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/replication from=127.0.0.1:43512 latency=178.823µs +48:55.510 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/replication from=127.0.0.1:43514 error="method HEAD not allowed" +48:55.510 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/replication from=127.0.0.1:43514 latency=18.721µs +48:55.511 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/replication from=127.0.0.1:43514 latency=2.645µs +48:55.511 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/roles from=127.0.0.1:43514 error="Permission denied" +48:55.511 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:43514 latency=296.165µs +48:55.512 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/roles from=127.0.0.1:43516 error="method PUT not allowed" +48:55.512 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/roles from=127.0.0.1:43516 latency=126.583µs +48:55.513 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/roles from=127.0.0.1:43518 error="method POST not allowed" +48:55.513 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/roles from=127.0.0.1:43518 latency=95.185µs +48:55.513 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/roles from=127.0.0.1:43520 error="method DELETE not allowed" +48:55.514 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/roles from=127.0.0.1:43520 latency=168.649µs +48:55.514 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/roles from=127.0.0.1:43522 error="method HEAD not allowed" +48:55.514 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/roles from=127.0.0.1:43522 latency=22.427µs +48:55.514 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/roles from=127.0.0.1:43522 latency=1.933µs +48:55.515 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/node/ from=127.0.0.1:43522 latency=173.876µs +48:55.515 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/node/ from=127.0.0.1:43524 error="method PUT not allowed" +48:55.515 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/node/ from=127.0.0.1:43524 latency=184.538µs +48:55.516 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/node/ from=127.0.0.1:43526 error="method POST not allowed" +48:55.516 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/node/ from=127.0.0.1:43526 latency=144.183µs +48:55.517 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/node/ from=127.0.0.1:43528 error="method DELETE not allowed" +48:55.517 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/node/ from=127.0.0.1:43528 latency=187.871µs +48:55.518 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/node/ from=127.0.0.1:43530 error="method HEAD not allowed" +48:55.518 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/node/ from=127.0.0.1:43530 latency=14.884µs +48:55.518 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/node/ from=127.0.0.1:43530 latency=2.142µs +48:55.518 [ERROR] TestAgent.http: Request error: method=GET url=/v1/internal/federation-states from=127.0.0.1:43530 error="Permission denied" +48:55.518 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/federation-states from=127.0.0.1:43530 latency=226.853µs +48:55.519 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/federation-states from=127.0.0.1:43532 error="method PUT not allowed" +48:55.519 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/federation-states from=127.0.0.1:43532 latency=102.992µs +48:55.519 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/federation-states from=127.0.0.1:43534 error="method POST not allowed" +48:55.519 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/federation-states from=127.0.0.1:43534 latency=152.092µs +48:55.520 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/federation-states from=127.0.0.1:43536 error="method DELETE not allowed" +48:55.520 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/federation-states from=127.0.0.1:43536 latency=119.25µs +48:55.521 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/federation-states from=127.0.0.1:43538 error="method HEAD not allowed" +48:55.521 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/federation-states from=127.0.0.1:43538 latency=33.645µs +48:55.521 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/federation-states from=127.0.0.1:43538 latency=2.657µs +48:55.521 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/checks/ from=127.0.0.1:43538 latency=210.709µs +48:55.522 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/checks/ from=127.0.0.1:43540 error="method PUT not allowed" +48:55.522 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/checks/ from=127.0.0.1:43540 latency=137.579µs +48:55.523 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/checks/ from=127.0.0.1:43544 error="method POST not allowed" +48:55.523 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/checks/ from=127.0.0.1:43544 latency=265.948µs +48:55.523 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/checks/ from=127.0.0.1:43546 error="method DELETE not allowed" +48:55.523 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/checks/ from=127.0.0.1:43546 latency=150.54µs +48:55.524 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/checks/ from=127.0.0.1:43550 error="method HEAD not allowed" +48:55.524 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/checks/ from=127.0.0.1:43550 latency=27.268µs +48:55.524 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/checks/ from=127.0.0.1:43550 latency=2.338µs +48:55.525 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/ from=127.0.0.1:43550 latency=143.615µs +48:55.525 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/service/ from=127.0.0.1:43552 error="method PUT not allowed" +48:55.526 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/service/ from=127.0.0.1:43552 latency=144.757µs +48:55.526 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/service/ from=127.0.0.1:43554 error="method POST not allowed" +48:55.526 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/service/ from=127.0.0.1:43554 latency=169.863µs +48:55.527 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/service/ from=127.0.0.1:43556 error="method DELETE not allowed" +48:55.527 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/service/ from=127.0.0.1:43556 latency=112.831µs +48:55.528 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/service/ from=127.0.0.1:43558 error="method HEAD not allowed" +48:55.528 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/service/ from=127.0.0.1:43558 latency=26.025µs +48:55.528 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/service/ from=127.0.0.1:43558 latency=1.791µs +48:55.528 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/health/service/name/ from=127.0.0.1:43558 error="Bad request: Missing service Name" +48:55.528 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/health/service/name/ from=127.0.0.1:43558 latency=102.04µs +48:55.529 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/health/service/name/ from=127.0.0.1:43562 error="method PUT not allowed" +48:55.529 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/health/service/name/ from=127.0.0.1:43562 latency=185.96µs +48:55.529 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/health/service/name/ from=127.0.0.1:43564 error="method POST not allowed" +48:55.530 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/health/service/name/ from=127.0.0.1:43564 latency=120.174µs +48:55.530 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/health/service/name/ from=127.0.0.1:43566 error="method DELETE not allowed" +48:55.530 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/health/service/name/ from=127.0.0.1:43566 latency=151.399µs +48:55.531 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/health/service/name/ from=127.0.0.1:43568 error="method HEAD not allowed" +48:55.531 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/health/service/name/ from=127.0.0.1:43568 latency=17.306µs +48:55.531 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/health/service/name/ from=127.0.0.1:43568 latency=1.75µs +48:55.531 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:43568 latency=223.952µs +48:55.532 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/coordinate/nodes from=127.0.0.1:43570 error="method PUT not allowed" +48:55.532 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/coordinate/nodes from=127.0.0.1:43570 latency=135.146µs +48:55.532 [ERROR] TestAgent.http: Request error: method=POST url=/v1/coordinate/nodes from=127.0.0.1:43572 error="method POST not allowed" +48:55.532 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/coordinate/nodes from=127.0.0.1:43572 latency=191.435µs +48:55.533 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/coordinate/nodes from=127.0.0.1:43574 error="method DELETE not allowed" +48:55.533 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/coordinate/nodes from=127.0.0.1:43574 latency=197.846µs +48:55.534 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/coordinate/nodes from=127.0.0.1:43578 error="method HEAD not allowed" +48:55.534 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/coordinate/nodes from=127.0.0.1:43578 latency=17.132µs +48:55.534 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/coordinate/nodes from=127.0.0.1:43578 latency=2.321µs +48:55.534 [ERROR] TestAgent.http: Request error: method=GET url=/v1/connect/ca/configuration from=127.0.0.1:43578 error="Permission denied" +48:55.535 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/ca/configuration from=127.0.0.1:43578 latency=213.229µs +48:55.535 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/ca/configuration from=127.0.0.1:43580 error="Bad request: Request decode failed: EOF" +48:55.535 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/ca/configuration from=127.0.0.1:43580 latency=249.29µs +48:55.536 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/ca/configuration from=127.0.0.1:43582 error="method POST not allowed" +48:55.536 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/ca/configuration from=127.0.0.1:43582 latency=195.557µs +48:55.537 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/ca/configuration from=127.0.0.1:43586 error="method DELETE not allowed" +48:55.537 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/ca/configuration from=127.0.0.1:43586 latency=112.192µs +48:55.537 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/ca/configuration from=127.0.0.1:43588 error="method HEAD not allowed" +48:55.537 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/ca/configuration from=127.0.0.1:43588 latency=22.126µs +48:55.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/ca/configuration from=127.0.0.1:43588 latency=1.425µs +48:55.538 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/ui/node/ from=127.0.0.1:43588 latency=96.872µs +48:55.538 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/ui/node/ from=127.0.0.1:43590 error="method PUT not allowed" +48:55.538 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/ui/node/ from=127.0.0.1:43590 latency=135.87µs +48:55.539 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/ui/node/ from=127.0.0.1:43592 error="method POST not allowed" +48:55.539 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/ui/node/ from=127.0.0.1:43592 latency=113.179µs +48:55.539 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/ui/node/ from=127.0.0.1:43594 error="method DELETE not allowed" +48:55.540 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/ui/node/ from=127.0.0.1:43594 latency=189.909µs +48:55.540 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/ui/node/ from=127.0.0.1:43596 error="method HEAD not allowed" +48:55.540 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/ui/node/ from=127.0.0.1:43596 latency=32.734µs +48:55.541 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/ui/node/ from=127.0.0.1:43596 latency=2.825µs +48:55.541 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/status/leader from=127.0.0.1:43596 latency=320.084µs +48:55.542 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/status/leader from=127.0.0.1:43598 error="method PUT not allowed" +48:55.542 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/status/leader from=127.0.0.1:43598 latency=220.139µs +48:55.553 [ERROR] TestAgent.http: Request error: method=POST url=/v1/status/leader from=127.0.0.1:43600 error="method POST not allowed" +48:55.553 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/status/leader from=127.0.0.1:43600 latency=377.646µs +48:55.555 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/status/leader from=127.0.0.1:43602 error="method DELETE not allowed" +48:55.556 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/status/leader from=127.0.0.1:43602 latency=1.247139ms +48:55.557 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/status/leader from=127.0.0.1:43604 error="method HEAD not allowed" +48:55.557 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/status/leader from=127.0.0.1:43604 latency=23.398µs +48:55.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/status/leader from=127.0.0.1:43604 latency=2.689µs +48:55.562 [ERROR] TestAgent.http: Request error: method=GET url=/v1/snapshot from=127.0.0.1:43604 error="Permission denied" +48:55.562 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:43604 latency=211.628µs +48:55.563 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:43608 error="Permission denied" +48:55.563 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:43608 latency=277.766µs +48:55.564 [ERROR] TestAgent.http: Request error: method=POST url=/v1/snapshot from=127.0.0.1:43610 error="method POST not allowed" +48:55.565 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/snapshot from=127.0.0.1:43610 latency=1.100066ms +48:55.565 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/snapshot from=127.0.0.1:43612 error="method DELETE not allowed" +48:55.566 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/snapshot from=127.0.0.1:43612 latency=163.979µs +48:55.566 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/snapshot from=127.0.0.1:43614 error="method HEAD not allowed" +48:55.566 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/snapshot from=127.0.0.1:43614 latency=28.901µs +48:55.566 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/snapshot from=127.0.0.1:43614 latency=2.109µs +48:55.567 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/auth-methods from=127.0.0.1:43614 error="Permission denied" +48:55.567 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-methods from=127.0.0.1:43614 latency=184.359µs +48:55.567 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/auth-methods from=127.0.0.1:43616 error="method PUT not allowed" +48:55.567 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-methods from=127.0.0.1:43616 latency=194.277µs +48:55.568 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/auth-methods from=127.0.0.1:43618 error="method POST not allowed" +48:55.568 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/auth-methods from=127.0.0.1:43618 latency=212.45µs +48:55.569 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/auth-methods from=127.0.0.1:43620 error="method DELETE not allowed" +48:55.569 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-methods from=127.0.0.1:43620 latency=100.758µs +48:55.569 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/auth-methods from=127.0.0.1:43622 error="method HEAD not allowed" +48:55.569 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/auth-methods from=127.0.0.1:43622 latency=18.456µs +48:55.570 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/auth-methods from=127.0.0.1:43622 latency=1.968µs +48:55.570 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/warn/ from=127.0.0.1:43622 error="method GET not allowed" +48:55.570 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/warn/ from=127.0.0.1:43622 latency=200.13µs +48:55.571 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/check/warn/ from=127.0.0.1:43624 error="Unknown check """ +48:55.571 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/warn/ from=127.0.0.1:43624 latency=351.583µs +48:55.572 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/warn/ from=127.0.0.1:43626 error="method POST not allowed" +48:55.572 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/warn/ from=127.0.0.1:43626 latency=196.289µs +48:55.572 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/warn/ from=127.0.0.1:43628 error="method DELETE not allowed" +48:55.573 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/warn/ from=127.0.0.1:43628 latency=142.419µs +48:55.573 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/warn/ from=127.0.0.1:43630 error="method HEAD not allowed" +48:55.573 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/warn/ from=127.0.0.1:43630 latency=22.645µs +48:55.573 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/warn/ from=127.0.0.1:43630 latency=1.94µs +48:55.574 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/token/self from=127.0.0.1:43630 error="ACL not found" +48:55.574 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:43630 latency=196.186µs +48:55.575 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/token/self from=127.0.0.1:43632 error="method PUT not allowed" +48:55.575 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/self from=127.0.0.1:43632 latency=200.113µs +48:55.575 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/token/self from=127.0.0.1:43634 error="method POST not allowed" +48:55.576 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/token/self from=127.0.0.1:43634 latency=249.085µs +48:55.576 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/token/self from=127.0.0.1:43636 error="method DELETE not allowed" +48:55.576 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/token/self from=127.0.0.1:43636 latency=170.892µs +48:55.577 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/token/self from=127.0.0.1:43638 error="method HEAD not allowed" +48:55.577 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/token/self from=127.0.0.1:43638 latency=22.185µs +48:55.578 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/token/self from=127.0.0.1:43638 latency=2.491µs +48:55.579 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/maintenance from=127.0.0.1:43638 error="method GET not allowed" +48:55.579 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/maintenance from=127.0.0.1:43638 latency=201.047µs +48:55.580 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/maintenance from=127.0.0.1:43640 latency=130.138µs +48:55.580 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/maintenance from=127.0.0.1:43642 error="method POST not allowed" +48:55.580 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/maintenance from=127.0.0.1:43642 latency=203.781µs +48:55.581 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/maintenance from=127.0.0.1:43644 error="method DELETE not allowed" +48:55.581 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/maintenance from=127.0.0.1:43644 latency=129.658µs +48:55.587 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/maintenance from=127.0.0.1:43648 error="method HEAD not allowed" +48:55.588 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/maintenance from=127.0.0.1:43648 latency=38.3µs +48:55.588 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/maintenance from=127.0.0.1:43648 latency=3.196µs +48:55.588 [ERROR] TestAgent.http: Request error: method=GET url=/v1/connect/intentions/match from=127.0.0.1:43648 error="required query parameter 'by' not set" +48:55.589 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/match from=127.0.0.1:43648 latency=156.733µs +48:55.589 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/intentions/match from=127.0.0.1:43650 error="method PUT not allowed" +48:55.589 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions/match from=127.0.0.1:43650 latency=170.334µs +48:55.590 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions/match from=127.0.0.1:43652 error="method POST not allowed" +48:55.590 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions/match from=127.0.0.1:43652 latency=191.704µs +48:55.591 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/intentions/match from=127.0.0.1:43654 error="method DELETE not allowed" +48:55.591 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions/match from=127.0.0.1:43654 latency=188.111µs +48:55.592 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/intentions/match from=127.0.0.1:43656 error="method HEAD not allowed" +48:55.592 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/intentions/match from=127.0.0.1:43656 latency=25.983µs +48:55.592 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/intentions/match from=127.0.0.1:43656 latency=1.502µs +48:55.592 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/ingress/ from=127.0.0.1:43656 latency=119.157µs +48:55.593 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/health/ingress/ from=127.0.0.1:43658 error="method PUT not allowed" +48:55.593 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/health/ingress/ from=127.0.0.1:43658 latency=87.063µs +48:55.594 [ERROR] TestAgent.http: Request error: method=POST url=/v1/health/ingress/ from=127.0.0.1:43660 error="method POST not allowed" +48:55.594 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/health/ingress/ from=127.0.0.1:43660 latency=84.919µs +48:55.594 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/health/ingress/ from=127.0.0.1:43662 error="method DELETE not allowed" +48:55.594 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/health/ingress/ from=127.0.0.1:43662 latency=162.636µs +48:55.595 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/health/ingress/ from=127.0.0.1:43664 error="method HEAD not allowed" +48:55.595 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/health/ingress/ from=127.0.0.1:43664 latency=36.907µs +48:55.595 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/health/ingress/ from=127.0.0.1:43664 latency=1.701µs +48:55.596 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/list from=127.0.0.1:43664 error="Permission denied" +48:55.596 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/list from=127.0.0.1:43664 latency=301.562µs +48:55.597 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/list from=127.0.0.1:43666 error="method PUT not allowed" +48:55.597 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/list from=127.0.0.1:43666 latency=215.964µs +48:55.598 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/list from=127.0.0.1:43670 error="method POST not allowed" +48:55.598 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/list from=127.0.0.1:43670 latency=126.127µs +48:55.598 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/list from=127.0.0.1:43672 error="method DELETE not allowed" +48:55.599 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/list from=127.0.0.1:43672 latency=229.713µs +48:55.599 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/list from=127.0.0.1:43674 error="method HEAD not allowed" +48:55.599 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/list from=127.0.0.1:43674 latency=18.961µs +48:55.599 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/list from=127.0.0.1:43674 latency=6.95µs +48:55.601 [DEBUG] TestAgent.acl: dropping service from result due to ACLs: service=consul +48:55.601 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/services from=127.0.0.1:43674 latency=566.327µs +48:55.602 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/services from=127.0.0.1:43676 error="method PUT not allowed" +48:55.602 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/services from=127.0.0.1:43676 latency=106.488µs +48:55.602 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/services from=127.0.0.1:43678 error="method POST not allowed" +48:55.603 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/services from=127.0.0.1:43678 latency=316.97µs +48:55.603 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/services from=127.0.0.1:43680 error="method DELETE not allowed" +48:55.604 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/services from=127.0.0.1:43680 latency=138.963µs +48:55.604 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/services from=127.0.0.1:43682 error="method HEAD not allowed" +48:55.604 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/services from=127.0.0.1:43682 latency=18.933µs +48:55.605 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/services from=127.0.0.1:43682 latency=4.1µs +48:55.605 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/node-services/ from=127.0.0.1:43682 latency=185.178µs +48:55.605 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/node-services/ from=127.0.0.1:43684 error="method PUT not allowed" +48:55.606 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/node-services/ from=127.0.0.1:43684 latency=120.574µs +48:55.606 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/node-services/ from=127.0.0.1:43686 error="method POST not allowed" +48:55.606 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/node-services/ from=127.0.0.1:43686 latency=176.103µs +48:55.607 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/node-services/ from=127.0.0.1:43688 error="method DELETE not allowed" +48:55.607 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/node-services/ from=127.0.0.1:43688 latency=216.129µs +48:55.608 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/node-services/ from=127.0.0.1:43690 error="method HEAD not allowed" +48:55.608 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/node-services/ from=127.0.0.1:43690 latency=65.815µs +48:55.610 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/node-services/ from=127.0.0.1:43690 latency=4.403µs +48:55.610 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/ca/roots from=127.0.0.1:43690 latency=245.878µs +48:55.611 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/ca/roots from=127.0.0.1:43692 error="method PUT not allowed" +48:55.611 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/ca/roots from=127.0.0.1:43692 latency=212.084µs +48:55.612 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/ca/roots from=127.0.0.1:43694 error="method POST not allowed" +48:55.612 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/ca/roots from=127.0.0.1:43694 latency=216.367µs +48:55.614 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/ca/roots from=127.0.0.1:43696 error="method DELETE not allowed" +48:55.615 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/ca/roots from=127.0.0.1:43696 latency=191.101µs +48:55.615 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/ca/roots from=127.0.0.1:43698 error="method HEAD not allowed" +48:55.615 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/ca/roots from=127.0.0.1:43698 latency=24.442µs +48:55.616 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/ca/roots from=127.0.0.1:43698 latency=1.707µs +48:55.616 [ERROR] TestAgent.http: Request error: method=GET url=/v1/coordinate/update from=127.0.0.1:43698 error="method GET not allowed" +48:55.616 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/update from=127.0.0.1:43698 latency=180.163µs +48:55.617 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/coordinate/update from=127.0.0.1:43700 latency=210.618µs +48:55.617 [ERROR] TestAgent.http: Request error: method=POST url=/v1/coordinate/update from=127.0.0.1:43702 error="method POST not allowed" +48:55.618 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/coordinate/update from=127.0.0.1:43702 latency=174.521µs +48:55.618 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/coordinate/update from=127.0.0.1:43704 error="method DELETE not allowed" +48:55.618 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/coordinate/update from=127.0.0.1:43704 latency=186.901µs +48:55.619 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/coordinate/update from=127.0.0.1:43706 error="method HEAD not allowed" +48:55.619 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/coordinate/update from=127.0.0.1:43706 latency=15.704µs +48:55.619 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/coordinate/update from=127.0.0.1:43706 latency=1.369µs +48:55.621 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/bootstrap from=127.0.0.1:43706 error="method GET not allowed" +48:55.621 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/bootstrap from=127.0.0.1:43706 latency=229.774µs +48:55.622 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/bootstrap from=127.0.0.1:43708 latency=215.496µs +48:55.623 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/bootstrap from=127.0.0.1:43710 error="method POST not allowed" +48:55.623 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/bootstrap from=127.0.0.1:43710 latency=210.316µs +48:55.624 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/bootstrap from=127.0.0.1:43712 error="method DELETE not allowed" +48:55.624 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/bootstrap from=127.0.0.1:43712 latency=230.959µs +48:55.625 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/bootstrap from=127.0.0.1:43714 error="method HEAD not allowed" +48:55.625 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/bootstrap from=127.0.0.1:43714 latency=18.659µs +48:55.626 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/bootstrap from=127.0.0.1:43714 latency=2.87µs +48:55.626 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/token/ from=127.0.0.1:43714 error="Bad request: Missing token ID" +48:55.626 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/ from=127.0.0.1:43714 latency=230.294µs +48:55.627 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/token/ from=127.0.0.1:43716 error="Bad request: Token decoding failed: EOF" +48:55.627 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/ from=127.0.0.1:43716 latency=133.586µs +48:55.627 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/token/ from=127.0.0.1:43720 error="method POST not allowed" +48:55.627 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/token/ from=127.0.0.1:43720 latency=218.174µs +48:55.628 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/token/ from=127.0.0.1:43722 error="Bad request: Missing token ID" +48:55.628 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/token/ from=127.0.0.1:43722 latency=192.215µs +48:55.629 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/token/ from=127.0.0.1:43724 error="method HEAD not allowed" +48:55.629 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/token/ from=127.0.0.1:43724 latency=15.852µs +48:55.629 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/token/ from=127.0.0.1:43724 latency=2.536µs +48:55.630 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/event/list from=127.0.0.1:43724 latency=369.42µs +48:55.630 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/event/list from=127.0.0.1:43726 error="method PUT not allowed" +48:55.630 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/list from=127.0.0.1:43726 latency=110.877µs +48:55.631 [ERROR] TestAgent.http: Request error: method=POST url=/v1/event/list from=127.0.0.1:43728 error="method POST not allowed" +48:55.631 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/event/list from=127.0.0.1:43728 latency=94.823µs +48:55.631 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/event/list from=127.0.0.1:43730 error="method DELETE not allowed" +48:55.631 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/event/list from=127.0.0.1:43730 latency=158.687µs +48:55.632 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/event/list from=127.0.0.1:43732 error="method HEAD not allowed" +48:55.632 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/event/list from=127.0.0.1:43732 latency=55.393µs +48:55.633 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/event/list from=127.0.0.1:43732 latency=1.725µs +48:55.633 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0d456efc-6231-9ceb-2643-32c992012de5 +48:55.633 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/internal/ui/nodes from=127.0.0.1:43732 latency=407.542µs +48:55.634 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/internal/ui/nodes from=127.0.0.1:43734 error="method PUT not allowed" +48:55.634 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/internal/ui/nodes from=127.0.0.1:43734 latency=196.969µs +48:55.634 [ERROR] TestAgent.http: Request error: method=POST url=/v1/internal/ui/nodes from=127.0.0.1:43736 error="method POST not allowed" +48:55.635 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/internal/ui/nodes from=127.0.0.1:43736 latency=173.396µs +48:55.635 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/internal/ui/nodes from=127.0.0.1:43738 error="method DELETE not allowed" +48:55.635 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/internal/ui/nodes from=127.0.0.1:43738 latency=166.569µs +48:55.636 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/internal/ui/nodes from=127.0.0.1:43740 error="method HEAD not allowed" +48:55.636 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/internal/ui/nodes from=127.0.0.1:43740 latency=14.238µs +48:55.636 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/internal/ui/nodes from=127.0.0.1:43740 latency=1.77µs +48:55.636 [ERROR] TestAgent.http: Request error: method=GET url=/v1/txn from=127.0.0.1:43740 error="method GET not allowed" +48:55.637 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/txn from=127.0.0.1:43740 latency=129.09µs +48:55.637 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/txn from=127.0.0.1:43742 latency=261.465µs +48:55.638 [ERROR] TestAgent.http: Request error: method=POST url=/v1/txn from=127.0.0.1:43744 error="method POST not allowed" +48:55.638 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/txn from=127.0.0.1:43744 latency=197.427µs +48:55.639 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/txn from=127.0.0.1:43746 error="method DELETE not allowed" +48:55.639 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/txn from=127.0.0.1:43746 latency=190.693µs +48:55.640 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/txn from=127.0.0.1:43748 error="method HEAD not allowed" +48:55.640 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/txn from=127.0.0.1:43748 latency=18.373µs +48:55.640 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/txn from=127.0.0.1:43748 latency=3.571µs +48:55.640 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policy/ from=127.0.0.1:43748 error="Bad request: Missing policy ID" +48:55.640 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/ from=127.0.0.1:43748 latency=167.004µs +48:55.641 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/policy/ from=127.0.0.1:43752 error="Bad request: Policy decoding failed: EOF" +48:55.641 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy/ from=127.0.0.1:43752 latency=193.275µs +48:55.642 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/policy/ from=127.0.0.1:43754 error="method POST not allowed" +48:55.642 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/policy/ from=127.0.0.1:43754 latency=167.079µs +48:55.642 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/policy/ from=127.0.0.1:43756 error="Bad request: Missing policy ID" +48:55.643 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policy/ from=127.0.0.1:43756 latency=196.784µs +48:55.643 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/policy/ from=127.0.0.1:43758 error="method HEAD not allowed" +48:55.643 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/policy/ from=127.0.0.1:43758 latency=19.618µs +48:55.644 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/policy/ from=127.0.0.1:43758 latency=2.259µs +48:55.644 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/service/register from=127.0.0.1:43758 error="method GET not allowed" +48:55.644 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/register from=127.0.0.1:43758 latency=164.45µs +48:55.645 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:43760 latency=209.688µs +48:55.645 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/service/register from=127.0.0.1:43762 error="method POST not allowed" +48:55.646 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/service/register from=127.0.0.1:43762 latency=228.563µs +48:55.646 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/service/register from=127.0.0.1:43764 error="method DELETE not allowed" +48:55.646 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/service/register from=127.0.0.1:43764 latency=229.365µs +48:55.647 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/service/register from=127.0.0.1:43766 error="method HEAD not allowed" +48:55.647 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/service/register from=127.0.0.1:43766 latency=14.832µs +48:55.647 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/service/register from=127.0.0.1:43766 latency=1.434µs +48:55.648 [ERROR] TestAgent.http: Request error: method=GET url=/v1/connect/intentions/ from=127.0.0.1:43766 error="Bad request: failed intention lookup: index error: UUID must be 36 characters" +48:55.648 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/ from=127.0.0.1:43766 latency=361.485µs +48:55.648 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/connect/intentions/ from=127.0.0.1:43768 error="Bad request: Request decode failed: EOF" +48:55.649 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions/ from=127.0.0.1:43768 latency=99.564µs +48:55.649 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions/ from=127.0.0.1:43770 error="method POST not allowed" +48:55.649 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions/ from=127.0.0.1:43770 latency=83.968µs +48:55.650 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/connect/intentions/ from=127.0.0.1:43772 error="Intention lookup failed: failed intention lookup: index error: UUID must be 36 characters" +48:55.650 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions/ from=127.0.0.1:43772 latency=338.837µs +48:55.651 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/connect/intentions/ from=127.0.0.1:43774 error="method HEAD not allowed" +48:55.651 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/connect/intentions/ from=127.0.0.1:43774 latency=18.326µs +48:55.651 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/connect/intentions/ from=127.0.0.1:43774 latency=1.706µs +48:55.652 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ from=127.0.0.1:43774 latency=369.644µs +48:55.652 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/config/ from=127.0.0.1:43776 error="method PUT not allowed" +48:55.653 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config/ from=127.0.0.1:43776 latency=183.177µs +48:55.653 [ERROR] TestAgent.http: Request error: method=POST url=/v1/config/ from=127.0.0.1:43778 error="method POST not allowed" +48:55.653 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/config/ from=127.0.0.1:43778 latency=112.014µs +48:55.654 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/config/ from=127.0.0.1:43780 latency=212.403µs +48:55.655 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/config/ from=127.0.0.1:43782 error="method HEAD not allowed" +48:55.655 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/config/ from=127.0.0.1:43782 latency=21.106µs +48:55.655 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/config/ from=127.0.0.1:43782 latency=1.649µs +48:55.655 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/rules/translate/ from=127.0.0.1:43782 error="Bad request: Missing token ID" +48:55.656 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/rules/translate/ from=127.0.0.1:43782 latency=130.937µs +48:55.656 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/rules/translate/ from=127.0.0.1:43784 error="method PUT not allowed" +48:55.656 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/rules/translate/ from=127.0.0.1:43784 latency=171.172µs +48:55.657 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/rules/translate/ from=127.0.0.1:43788 error="method POST not allowed" +48:55.657 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/rules/translate/ from=127.0.0.1:43788 latency=109.694µs +48:55.658 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/rules/translate/ from=127.0.0.1:43790 error="method DELETE not allowed" +48:55.658 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/rules/translate/ from=127.0.0.1:43790 latency=198.635µs +48:55.658 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/rules/translate/ from=127.0.0.1:43792 error="method HEAD not allowed" +48:55.658 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/rules/translate/ from=127.0.0.1:43792 latency=19.654µs +48:55.659 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/rules/translate/ from=127.0.0.1:43792 latency=1.821µs +48:55.659 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/join/ from=127.0.0.1:43792 error="method GET not allowed" +48:55.659 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/join/ from=127.0.0.1:43792 latency=176.499µs +48:55.659 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/join/ from=127.0.0.1:43794 error="Permission denied" +48:55.660 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/join/ from=127.0.0.1:43794 latency=269.488µs +48:55.660 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/join/ from=127.0.0.1:43796 error="method POST not allowed" +48:55.661 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/join/ from=127.0.0.1:43796 latency=272.715µs +48:55.661 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/join/ from=127.0.0.1:43798 error="method DELETE not allowed" +48:55.661 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/join/ from=127.0.0.1:43798 latency=211.428µs +48:55.662 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/join/ from=127.0.0.1:43800 error="method HEAD not allowed" +48:55.662 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/join/ from=127.0.0.1:43800 latency=21.948µs +48:55.664 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/join/ from=127.0.0.1:43800 latency=5.008µs +48:55.664 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/datacenters from=127.0.0.1:43800 latency=501.71µs +48:55.665 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/datacenters from=127.0.0.1:43802 error="method PUT not allowed" +48:55.666 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/datacenters from=127.0.0.1:43802 latency=223.705µs +48:55.666 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/datacenters from=127.0.0.1:43804 error="method POST not allowed" +48:55.667 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/datacenters from=127.0.0.1:43804 latency=223.861µs +48:55.667 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/datacenters from=127.0.0.1:43806 error="method DELETE not allowed" +48:55.668 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/datacenters from=127.0.0.1:43806 latency=280.712µs +48:55.669 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/datacenters from=127.0.0.1:43808 error="method HEAD not allowed" +48:55.669 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/datacenters from=127.0.0.1:43808 latency=30.376µs +48:55.669 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/datacenters from=127.0.0.1:43808 latency=2.345µs +48:55.670 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/auth-method from=127.0.0.1:43808 error="method GET not allowed" +48:55.670 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method from=127.0.0.1:43808 latency=311.158µs +48:55.671 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/auth-method from=127.0.0.1:43810 error="Bad request: AuthMethod decoding failed: EOF" +48:55.672 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:43810 latency=232.018µs +48:55.677 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/auth-method from=127.0.0.1:43812 error="method POST not allowed" +48:55.677 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/auth-method from=127.0.0.1:43812 latency=276.004µs +48:55.679 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/auth-method from=127.0.0.1:43814 error="method DELETE not allowed" +48:55.679 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-method from=127.0.0.1:43814 latency=158.902µs +48:55.679 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/auth-method from=127.0.0.1:43816 error="method HEAD not allowed" +48:55.679 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/auth-method from=127.0.0.1:43816 latency=48.59µs +48:55.680 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/auth-method from=127.0.0.1:43816 latency=2.53µs +48:55.680 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/host from=127.0.0.1:43816 error="Permission denied" +48:55.680 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/host from=127.0.0.1:43816 latency=279.142µs +48:55.681 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/host from=127.0.0.1:43818 error="method PUT not allowed" +48:55.681 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/host from=127.0.0.1:43818 latency=162.964µs +48:55.682 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/host from=127.0.0.1:43820 error="method POST not allowed" +48:55.682 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/host from=127.0.0.1:43820 latency=207.034µs +48:55.683 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/host from=127.0.0.1:43822 error="method DELETE not allowed" +48:55.683 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/host from=127.0.0.1:43822 latency=170.377µs +48:55.684 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/host from=127.0.0.1:43824 error="method HEAD not allowed" +48:55.684 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/host from=127.0.0.1:43824 latency=19.194µs +48:55.684 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/host from=127.0.0.1:43824 latency=3.29µs +48:55.685 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/auth-method/ from=127.0.0.1:43824 error="Bad request: Missing auth method name" +48:55.685 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/ from=127.0.0.1:43824 latency=206.019µs +48:55.685 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/auth-method/ from=127.0.0.1:43826 error="Bad request: AuthMethod decoding failed: EOF" +48:55.685 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/ from=127.0.0.1:43826 latency=198.235µs +48:55.686 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/auth-method/ from=127.0.0.1:43828 error="method POST not allowed" +48:55.686 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/auth-method/ from=127.0.0.1:43828 latency=99.897µs +48:55.687 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/auth-method/ from=127.0.0.1:43832 error="Bad request: Missing auth method name" +48:55.687 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-method/ from=127.0.0.1:43832 latency=217.876µs +48:55.688 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/auth-method/ from=127.0.0.1:43834 error="method HEAD not allowed" +48:55.688 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/auth-method/ from=127.0.0.1:43834 latency=18.934µs +48:55.688 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/auth-method/ from=127.0.0.1:43834 latency=2.468µs +48:55.688 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/check/register from=127.0.0.1:43834 error="method GET not allowed" +48:55.689 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/check/register from=127.0.0.1:43834 latency=189.418µs +48:55.689 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/register from=127.0.0.1:43836 latency=125.261µs +48:55.690 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/check/register from=127.0.0.1:43838 error="method POST not allowed" +48:55.690 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/check/register from=127.0.0.1:43838 latency=210.073µs +48:55.691 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/check/register from=127.0.0.1:43842 error="method DELETE not allowed" +48:55.691 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/check/register from=127.0.0.1:43842 latency=159.275µs +48:55.692 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/check/register from=127.0.0.1:43844 error="method HEAD not allowed" +48:55.692 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/check/register from=127.0.0.1:43844 latency=28.912µs +48:55.692 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/check/register from=127.0.0.1:43844 latency=2.241µs +48:55.692 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/connect/ from=127.0.0.1:43844 latency=208.101µs +48:55.693 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/catalog/connect/ from=127.0.0.1:43846 error="method PUT not allowed" +48:55.693 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/connect/ from=127.0.0.1:43846 latency=229.455µs +48:55.694 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/connect/ from=127.0.0.1:43848 error="method POST not allowed" +48:55.694 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/connect/ from=127.0.0.1:43848 latency=105.231µs +48:55.694 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/connect/ from=127.0.0.1:43850 error="method DELETE not allowed" +48:55.694 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/connect/ from=127.0.0.1:43850 latency=80.297µs +48:55.695 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/connect/ from=127.0.0.1:43852 error="method HEAD not allowed" +48:55.695 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/connect/ from=127.0.0.1:43852 latency=39.68µs +48:55.695 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/connect/ from=127.0.0.1:43852 latency=3.247µs +48:55.696 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/service/maintenance/ from=127.0.0.1:43852 error="method GET not allowed" +48:55.696 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/maintenance/ from=127.0.0.1:43852 latency=249.766µs +48:55.697 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/maintenance/ from=127.0.0.1:43854 latency=172.388µs +48:55.697 [ERROR] TestAgent.http: Request error: method=POST url=/v1/agent/service/maintenance/ from=127.0.0.1:43858 error="method POST not allowed" +48:55.698 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/agent/service/maintenance/ from=127.0.0.1:43858 latency=222.302µs +48:55.699 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/agent/service/maintenance/ from=127.0.0.1:43860 error="method DELETE not allowed" +48:55.699 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/agent/service/maintenance/ from=127.0.0.1:43860 latency=271.648µs +48:55.700 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/agent/service/maintenance/ from=127.0.0.1:43862 error="method HEAD not allowed" +48:55.700 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/agent/service/maintenance/ from=127.0.0.1:43862 latency=79.6µs +48:55.701 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/agent/service/maintenance/ from=127.0.0.1:43862 latency=2.885µs +48:55.701 [ERROR] TestAgent.http: Request error: method=GET url=/v1/catalog/deregister from=127.0.0.1:43862 error="method GET not allowed" +48:55.701 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/deregister from=127.0.0.1:43862 latency=267.095µs +48:55.702 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/catalog/deregister from=127.0.0.1:43864 latency=155.39µs +48:55.703 [ERROR] TestAgent.http: Request error: method=POST url=/v1/catalog/deregister from=127.0.0.1:43866 error="method POST not allowed" +48:55.703 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/catalog/deregister from=127.0.0.1:43866 latency=238.275µs +48:55.704 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/catalog/deregister from=127.0.0.1:43868 error="method DELETE not allowed" +48:55.704 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/catalog/deregister from=127.0.0.1:43868 latency=212.6µs +48:55.705 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/catalog/deregister from=127.0.0.1:43870 error="method HEAD not allowed" +48:55.705 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/catalog/deregister from=127.0.0.1:43870 latency=33.289µs +48:55.706 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/catalog/deregister from=127.0.0.1:43870 latency=3.871µs +48:55.708 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/info/ from=127.0.0.1:43870 latency=1.140307ms +48:55.708 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/info/ from=127.0.0.1:43872 error="method PUT not allowed" +48:55.709 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/info/ from=127.0.0.1:43872 latency=253.608µs +48:55.710 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/info/ from=127.0.0.1:43876 error="method POST not allowed" +48:55.710 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/info/ from=127.0.0.1:43876 latency=242.827µs +48:55.711 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/info/ from=127.0.0.1:43878 error="method DELETE not allowed" +48:55.711 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/info/ from=127.0.0.1:43878 latency=261.051µs +48:55.714 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/info/ from=127.0.0.1:43880 error="method HEAD not allowed" +48:55.714 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/info/ from=127.0.0.1:43880 latency=38.053µs +48:55.714 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/info/ from=127.0.0.1:43880 latency=2.096µs +48:55.714 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/binding-rules from=127.0.0.1:43880 error="Permission denied" +48:55.715 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:43880 latency=338.239µs +48:55.717 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rules from=127.0.0.1:43882 error="method PUT not allowed" +48:55.717 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rules from=127.0.0.1:43882 latency=173.373µs +48:55.718 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/binding-rules from=127.0.0.1:43884 error="method POST not allowed" +48:55.718 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/binding-rules from=127.0.0.1:43884 latency=113.375µs +48:55.719 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/binding-rules from=127.0.0.1:43886 error="method DELETE not allowed" +48:55.720 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rules from=127.0.0.1:43886 latency=489.807µs +48:55.722 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/binding-rules from=127.0.0.1:43888 error="method HEAD not allowed" +48:55.722 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/binding-rules from=127.0.0.1:43888 latency=45.044µs +48:55.722 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/binding-rules from=127.0.0.1:43888 latency=2.318µs +48:55.723 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/token from=127.0.0.1:43888 error="method GET not allowed" +48:55.723 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token from=127.0.0.1:43888 latency=192.811µs +48:55.724 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/token from=127.0.0.1:43892 error="Bad request: Token decoding failed: EOF" +48:55.725 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:43892 latency=348.189µs +48:55.726 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/token from=127.0.0.1:43894 error="method POST not allowed" +48:55.727 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/token from=127.0.0.1:43894 latency=289.996µs +48:55.727 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/acl/token from=127.0.0.1:43896 error="method DELETE not allowed" +48:55.727 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/token from=127.0.0.1:43896 latency=117.089µs +48:55.728 [ERROR] TestAgent.http: Request error: method=HEAD url=/v1/acl/token from=127.0.0.1:43898 error="method HEAD not allowed" +48:55.728 [DEBUG] TestAgent.http: Request finished: method=HEAD url=/v1/acl/token from=127.0.0.1:43898 latency=28.507µs +48:55.729 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=/v1/acl/token from=127.0.0.1:43898 latency=13.808µs +48:55.729 [INFO] TestAgent: Requesting shutdown +48:55.730 [INFO] TestAgent.server: shutting down server +48:55.730 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:55.730 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:55.730 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:55.730 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:55.730 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:55.730 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:55.730 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:55.730 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:55.730 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:55.730 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:55.730 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:55.730 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:55.730 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:55.730 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:55.730 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:48:55.730Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:55.730Z [INFO] agent.router.manager: shutting down +48:55.730 [INFO] TestAgent: consul server down +48:55.730 [INFO] TestAgent: shutdown complete +48:55.730 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24156 network=tcp +48:55.730 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24156 network=udp +48:55.730 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24157 network=tcp +48:56.231 [INFO] TestAgent: Waiting for endpoints to shut down +48:56.231 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPAPI_MethodNotAllowed_OSS (1.13s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query/xxx/execute (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query/xxx/execute (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query/xxx/execute (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/xxx/execute (0.00s) @@ -17011,225 +17042,417 @@ --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query/xxx/explain (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query/xxx/explain (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query/xxx/explain (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/xxx/explain (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/xxx/explain (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query/xxx/explain (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query/xxx/explain (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rules (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rules (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rules (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rules (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rules (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rules (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/query/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/query/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/query/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/query/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/query/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/query/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/tokens (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/tokens (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/tokens (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/tokens (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/tokens (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/tokens (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/renew/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/renew/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/renew/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/renew/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/renew/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/renew/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/rules/translate (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/rules/translate (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/rules/translate (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/rules/translate (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/rules/translate (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/rules/translate (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rule (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rule (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rule (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rule (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rule (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rule (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/ca/roots (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/ca/roots (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/force-leave/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/force-leave/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/force-leave/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/force-leave/ (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/force-leave/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/force-leave/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/force-leave/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/autopilot/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/autopilot/configuration (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/autopilot/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/autopilot/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/autopilot/configuration (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/autopilot/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/create (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/create (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/nodes (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/check (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/check (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/check (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/check (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/check (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/check (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/checks/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/checks/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/checks/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/checks/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/checks/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/checks/ (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/leave (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/leave (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/leave (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/leave (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/leave (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/leave (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/update/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/update/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/update/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/update/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/update/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/update/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/acl/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/acl/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/acl/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/acl/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/acl/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/acl/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/raft/peer (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/raft/peer (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/raft/peer (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/raft/peer (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/raft/peer (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/raft/peer (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/login (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/login (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/login (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/login (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/login (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/login (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/checks (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/checks (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/checks (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/checks (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/checks (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/checks (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/gateway-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/gateway-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/gateway-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/gateway-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/gateway-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/gateway-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/clone/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/clone/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/clone/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/clone/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/clone/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/clone/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/authorize (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policies (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policies (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policies (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policies (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policies (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policies (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/self (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/state/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/state/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/state/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/state/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/state/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/update (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-methods (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-methods (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-methods (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-methods (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-methods (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-methods (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/members (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/members (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/members (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/members (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/members (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/members (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/config/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/config/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/config/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/config/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/config/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/config/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token/self (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token/self (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/maintenance/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/maintenance/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/maintenance/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/maintenance/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/maintenance/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/info/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/info/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/info/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/info/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/info/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/info/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role/ (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role/ (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rule/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rule/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rule/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rule/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rule/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rule/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/warn/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/warn/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/warn/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/warn/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/warn/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/warn/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rule (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rule (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rule (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rule (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rule (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rule (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/deregister/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/deregister/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/renew/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/renew/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/renew/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/renew/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/renew/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/renew/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/info/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/info/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/info/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/info/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/info/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/info/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-state/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/create (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/autopilot/health (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/autopilot/health (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/autopilot/health (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/autopilot/health (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/autopilot/health (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/autopilot/health (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/list (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/token/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/token/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/token/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/token/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/token/ (0.01s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/metrics (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/metrics (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/metrics (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/metrics (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/metrics (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/metrics (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/deregister (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/deregister (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/deregister (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/deregister (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/deregister (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/deregister (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-states (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-states (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-states (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-states (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-states (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-states (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/keyring (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/keyring (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/keyring (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/keyring (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/keyring (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/keyring (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/snapshot (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/snapshot (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/snapshot (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/snapshot (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/snapshot (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/snapshot (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/destroy/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/gateway-services-nodes/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/gateway-services-nodes/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/gateway-services-nodes/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/gateway-services-nodes/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/gateway-services-nodes/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/gateway-services-nodes/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/config (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/config (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/config (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/config (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/config (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/config (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rule/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rule/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rule/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rule/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rule/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rule/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/discovery-chain/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/discovery-chain/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/discovery-chain/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/discovery-chain/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/discovery-chain/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/discovery-chain/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/event/fire/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/event/fire/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/event/fire/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/event/fire/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/event/fire/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/event/fire/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/ca/leaf/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/ca/leaf/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/ca/leaf/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/ca/leaf/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/ca/leaf/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/ca/leaf/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/check (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/check (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/check (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/check (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/check (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/check (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/register (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/register (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/register (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/register (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/register (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/services (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/event/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/event/list (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/event/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/event/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/event/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/event/list (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/connect/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/connect/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/connect/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/connect/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/connect/ (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/connect/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/raft/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/raft/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/raft/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/raft/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/raft/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/raft/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/autopilot/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/autopilot/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/autopilot/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/autopilot/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/autopilot/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/autopilot/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/metrics (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/metrics (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/metrics (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/metrics (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/metrics (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/metrics (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/members (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/members (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/members (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/members (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/members (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/members (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/deregister/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-states/mesh-gateways (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-states/mesh-gateways (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-states/mesh-gateways (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-states/mesh-gateways (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-states/mesh-gateways (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-states/mesh-gateways (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/logout (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/logout (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/logout (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/logout (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/logout (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/logout (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/health/service/id/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/health/service/id/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/health/service/id/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/health/service/id/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/health/service/id/ (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/health/service/id/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/health/service/id/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/pass/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/pass/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/pass/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/pass/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/pass/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/pass/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/replication (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/replication (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/replication (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/replication (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/replication (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/replication (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/roles (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/roles (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/roles (0.00s) @@ -17237,1184 +17460,1007 @@ --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/roles (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/roles (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/node/ (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/node/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/node/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/node/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/node/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/txn (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/txn (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/txn (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/txn (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/txn (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/txn (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/datacenters (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/datacenters (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-state/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-state/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/logout (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/logout (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/logout (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/logout (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/logout (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/logout (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/register (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/pass/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/pass/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/pass/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/pass/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/pass/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/pass/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/gateway-services/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/gateway-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/gateway-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/gateway-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/gateway-services/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/gateway-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/leave (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/leave (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/leave (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/leave (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/leave (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/leave (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/login (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/login (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/login (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/login (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/login (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/login (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-method/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-method/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-method/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-method/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-method/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-method/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/self (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/self (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/gateway-services-nodes/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/gateway-services-nodes/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/gateway-services-nodes/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/gateway-services-nodes/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/gateway-services-nodes/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/gateway-services-nodes/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/raft/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/raft/configuration (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/raft/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/raft/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/raft/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/raft/configuration (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/clone/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/clone/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/clone/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/clone/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/clone/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/clone/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/maintenance (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/maintenance (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/maintenance (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/maintenance (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/maintenance (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/maintenance (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/service/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/create (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/create (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/fail/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/fail/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/fail/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/fail/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/fail/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/fail/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/update/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/update/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/update/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/update/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/update/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/update/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/node/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/services (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/services (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/services (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-states (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-states (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-states (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-states (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-states (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-states (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/checks/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/checks/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/checks/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/checks/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/checks/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/checks/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/health/service/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/health/service/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/health/service/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/health/service/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/health/service/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/health/service/name/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/nodes (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/ca/configuration (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/ca/configuration (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/ca/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/ca/configuration (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/ca/configuration (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/ca/configuration (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/ca/configuration (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/ca/configuration (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/info/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/info/ (0.02s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/info/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/info/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/info/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/info/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/role/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/role/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/role/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/role/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/role/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/role/name/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/federation-states/mesh-gateways (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/federation-states/mesh-gateways (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/federation-states/mesh-gateways (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/federation-states/mesh-gateways (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/federation-states/mesh-gateways (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/federation-states/mesh-gateways (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/node/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/node/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/node/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/node/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/node/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policies (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policies (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policies (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policies (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policies (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policies (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/join/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/join/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/join/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/join/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/join/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/join/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/update (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/checks (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/checks (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/checks (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/checks (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/checks (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/checks (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/nodes (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/nodes (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/kv/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/kv/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/kv/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/kv/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/kv/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/node/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/status/leader (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/status/leader (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/status/leader (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/status/leader (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/status/leader (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/status/leader (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/snapshot (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/snapshot (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/snapshot (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/snapshot (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/snapshot (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/snapshot (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-methods (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-methods (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-methods (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-methods (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-methods (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-methods (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/warn/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/warn/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/warn/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/warn/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/warn/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/warn/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token/self (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/maintenance (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/maintenance (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/maintenance (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/maintenance (0.01s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/maintenance (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/maintenance (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/match (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/match (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/match (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/match (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/match (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/match (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/ingress/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/ingress/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/ingress/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/ingress/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/ingress/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/ingress/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/services (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/node-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/node-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/node-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/node-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/node-services/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/node-services/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/ca/roots (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/ca/roots (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/ca/roots (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/ca/roots (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/ca/roots (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/update (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/update (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/bootstrap (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/bootstrap (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/bootstrap (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/bootstrap (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/bootstrap (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/bootstrap (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/list (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/list (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/event/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/event/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/event/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/event/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/event/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/event/list (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/ui/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/ui/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/ui/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/ui/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/ui/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/ui/nodes (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/txn (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/txn (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/txn (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/txn (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/txn (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/txn (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/config/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/config/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/config/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/config/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/config/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/config/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/rules/translate/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/rules/translate/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/rules/translate/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/rules/translate/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/rules/translate/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/rules/translate/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/coordinate/nodes (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/coordinate/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/coordinate/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/coordinate/nodes (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/coordinate/nodes (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/coordinate/nodes (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/ca/leaf/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/ca/leaf/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/ca/leaf/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/ca/leaf/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/ca/leaf/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/ca/leaf/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/event/fire/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/event/fire/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/event/fire/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/event/fire/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/event/fire/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/event/fire/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/service/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/join/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/join/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/join/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/join/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/join/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/join/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/datacenters (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/datacenters (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-method (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-method (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-method (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-method (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-method (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-method (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/config (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/config (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/config (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/config (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/config (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/config (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/connect/intentions/match (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/connect/intentions/match (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/connect/intentions/match (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/connect/intentions/match (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/connect/intentions/match (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/connect/intentions/match (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/rules/translate (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/rules/translate (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/rules/translate (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/rules/translate (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/rules/translate (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/rules/translate (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token/ (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/host (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/host (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/host (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/host (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/host (0.00s) --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/host (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/health/service/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/health/service/name/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/health/service/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/health/service/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/health/service/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/health/service/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/register (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/register (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/policy/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/policy/name/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/policy/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/policy/name/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/policy/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/policy/name/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/raft/peer (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/raft/peer (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/raft/peer (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/raft/peer (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/raft/peer (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/raft/peer (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/replication (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/replication (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/replication (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/replication (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/replication (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/replication (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/internal/acl/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/internal/acl/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/internal/acl/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/internal/acl/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/internal/acl/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/internal/acl/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/session/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/session/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/session/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/session/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/session/destroy/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/session/destroy/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/deregister/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/deregister/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/ingress/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/ingress/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/ingress/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/ingress/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/ingress/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/ingress/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/operator/autopilot/health (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/operator/autopilot/health (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/operator/autopilot/health (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/operator/autopilot/health (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/operator/autopilot/health (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/operator/autopilot/health (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/status/leader (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/status/leader (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/status/leader (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/status/leader (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/status/leader (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/status/leader (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/health/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/health/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/health/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/health/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/health/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/health/node/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/tokens (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/tokens (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/tokens (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/tokens (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/tokens (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/tokens (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/authorize (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/authorize (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/connect/ca/roots (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/connect/ca/roots (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/connect/ca/roots (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/connect/ca/roots (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/connect/ca/roots (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/connect/ca/roots (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/node-services/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/node-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/node-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/node-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/node-services/ (0.01s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/node-services/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/discovery-chain/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/discovery-chain/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/discovery-chain/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/discovery-chain/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/discovery-chain/ (0.00s) - --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/discovery-chain/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/check/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/check/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/check/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/check/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/check/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/check/register (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/connect/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/catalog/deregister (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/catalog/deregister (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/catalog/deregister (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/catalog/deregister (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/catalog/deregister (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/catalog/deregister (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/info/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/info/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/info/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/info/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/info/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/info/ (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/binding-rules (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/binding-rules (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/binding-rules (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/binding-rules (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/binding-rules (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/binding-rules (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/GET_/v1/acl/token (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/PUT_/v1/acl/token (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/POST_/v1/acl/token (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/DELETE_/v1/acl/token (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/HEAD_/v1/acl/token (0.00s) + --- PASS: TestHTTPAPI_MethodNotAllowed_OSS/OPTIONS_/v1/acl/token (0.00s) === RUN TestHTTPAPI_OptionMethod_OSS [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query/xxx/execute -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query/xxx/explain === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/nodes -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/list -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/rules/translate/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/ca/leaf/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/event/fire/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/service/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-method -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/config -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/health/service/name/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/register -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/match -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/node/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/rules/translate -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/host -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy/name/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query/xxx/execute +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query/xxx/explain +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/clone/ === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/raft/peer -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/replication -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/acl/authorize -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/destroy/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/status/leader -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/deregister/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/ingress/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/autopilot/health -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/node-services/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/discovery-chain/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/node/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/tokens === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/authorize -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/ca/roots -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/autopilot/configuration -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/create -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rules -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/force-leave/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/datacenters +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/update +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policies +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/self === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/state/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/nodes -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/check -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/checks/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/config/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/update -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-methods -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/members -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/connect/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token/self -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/maintenance/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/info/ === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rule/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/warn/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rule -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/deregister/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/deregister -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-states -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/renew/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/info/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/token/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/metrics -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/event/list -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/connect/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/keyring -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/snapshot +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/datacenters +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-state/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/node/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/create +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/create +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/nodes +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/service/ === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/destroy/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role/name/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/node/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/autopilot/health +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/list +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/token/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/services +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/gateway-services-nodes/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/node/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy/name/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/config +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/kv/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rule/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/discovery-chain/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/event/fire/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/ca/leaf/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/check === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/register -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/services +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/connect/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/raft/configuration +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/autopilot/configuration +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/metrics +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/members +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/deregister/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-states/mesh-gateways +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/logout === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/health/service/id/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/pass/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/replication === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/roles === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/node/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-states +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/checks/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/service/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/health/service/name/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/nodes +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/ca/configuration +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/node/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/status/leader +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/snapshot +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-methods +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/warn/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token/self +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/maintenance +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/match +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/ingress/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/list +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/services +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/node-services/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/ca/roots +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/update +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/bootstrap +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/event/list +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/nodes === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/txn -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/pass/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/gateway-services/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/datacenters -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-state/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/logout -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/register -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/services === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/register +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/config/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/rules/translate/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/join/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/datacenters +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-method +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/host +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/register +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/connect/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/maintenance/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/deregister +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/info/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rules === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/tokens +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/services +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/renew/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/rules/translate +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rule +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/ca/roots +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/force-leave/ === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/leave -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/gateway-services-nodes/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/raft/configuration -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/login -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/self -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/clone/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/maintenance -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/service/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/create === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/destroy/ === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/update/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/node/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/info/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/services -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/ca/configuration -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role/name/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-states/mesh-gateways -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/node/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policies -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/join/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/kv/ -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/update +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/acl/authorize +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/raft/peer +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/keyring +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/login === RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/checks -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/nodes -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/ca/roots -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/bootstrap -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy -=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/list +=== RUN TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/gateway-services/ [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:43.314Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:12:43.314Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:43.337Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:12:43.341Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:43.343 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b04930e3-a393-5177-78b3-58f42f2812d8 Address:127.0.0.1:17021}]" -12:43.343 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17021 [Follower]" leader= -12:43.344 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b04930e3-a393-5177-78b3-58f42f2812d8.dc1 127.0.0.1 -12:43.344 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b04930e3-a393-5177-78b3-58f42f2812d8 127.0.0.1 -2021-10-31T23:12:43.344Z [INFO] agent.router: Initializing LAN area manager -12:43.344 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17016 network=udp -12:43.345 [INFO] TestAgent.server: Adding LAN server: server="Node-b04930e3-a393-5177-78b3-58f42f2812d8 (Addr: tcp/127.0.0.1:17021) (DC: dc1)" -12:43.345 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b04930e3-a393-5177-78b3-58f42f2812d8.dc1 area=wan -12:43.345 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17016 network=tcp -12:43.349 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17017 network=tcp -12:43.350 [INFO] TestAgent: started state syncer -12:43.350 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17022 network=tcp -12:43.385 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:43.385 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17021 [Candidate]" term=2 -12:43.385 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:43.385 [DEBUG] TestAgent.server.raft: vote granted: from=b04930e3-a393-5177-78b3-58f42f2812d8 term=2 tally=1 -12:43.385 [INFO] TestAgent.server.raft: election won: tally=1 -12:43.385 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17021 [Leader]" -12:43.385 [INFO] TestAgent.server: cluster leadership acquired -12:43.385 [INFO] TestAgent.server: initializing acls -12:43.385 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:43.385 [INFO] TestAgent.server: New leader elected: payload=Node-b04930e3-a393-5177-78b3-58f42f2812d8 -12:43.386 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:43.386 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:43.386 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:43.386 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b04930e3-a393-5177-78b3-58f42f2812d8 -12:43.386 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b04930e3-a393-5177-78b3-58f42f2812d8.dc1 -12:43.386 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:43.388 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:43.388 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:43.388 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:43.388 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:43.388 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:43.388 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b04930e3-a393-5177-78b3-58f42f2812d8 -12:43.388 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b04930e3-a393-5177-78b3-58f42f2812d8 -12:43.389 [INFO] TestAgent.server: Updating LAN server: server="Node-b04930e3-a393-5177-78b3-58f42f2812d8 (Addr: tcp/127.0.0.1:17021) (DC: dc1)" -12:43.389 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b04930e3-a393-5177-78b3-58f42f2812d8 -12:43.389 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b04930e3-a393-5177-78b3-58f42f2812d8.dc1 area=wan -12:43.389 [INFO] TestAgent.server: federation state anti-entropy synced -12:43.558 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/query/xxx/execute from= latency=155.53µs -12:43.558 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/query/xxx/explain from= latency=29.448µs -12:43.558 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/query from= latency=2.405µs -12:43.558 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/query/ from= latency=29.55µs -12:43.559 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/coordinate/nodes from= latency=1.775µs -12:43.559 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/list from= latency=1.575µs -12:43.559 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/rules/translate/ from= latency=1.714µs -12:43.559 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/connect/ca/leaf/ from= latency=1.429µs -12:43.559 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/event/fire/ from= latency=1.542µs -12:43.559 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/health/service/ from= latency=1.691µs -12:43.559 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/auth-method from= latency=1.456µs -12:43.559 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/config from= latency=1.709µs -12:43.559 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/health/service/name/ from= latency=1.481µs -12:43.559 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/service/register from= latency=1.354µs -12:43.559 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/connect/intentions/match from= latency=1.713µs -12:43.559 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/session/node/ from= latency=1.489µs -12:43.559 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/rules/translate from= latency=1.66µs -12:43.559 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/token/ from= latency=3.424µs -12:43.560 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/host from= latency=1.616µs -12:43.560 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/policy/name/ from= latency=1.595µs -12:43.560 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/service/ from= latency=1.649µs -12:43.560 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/operator/raft/peer from= latency=1.675µs -12:43.560 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/replication from= latency=1.553µs -12:43.560 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/internal/acl/authorize from= latency=1.756µs -12:43.560 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/session/destroy/ from= latency=1.558µs -12:43.560 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/status/leader from= latency=1.588µs -12:43.560 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/check/deregister/ from= latency=1.579µs -12:43.560 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/health/ingress/ from= latency=1.436µs -12:43.560 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/operator/autopilot/health from= latency=1.524µs -12:43.560 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/catalog/node-services/ from= latency=1.56µs -12:43.560 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/discovery-chain/ from= latency=1.51µs -12:43.560 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/health/node/ from= latency=1.353µs -12:43.561 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/tokens from= latency=1.463µs -12:43.561 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/connect/authorize from= latency=1.427µs -12:43.561 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/connect/ca/roots from= latency=1.427µs -12:43.561 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/operator/autopilot/configuration from= latency=1.51µs -12:43.561 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/session/create from= latency=1.365µs -12:43.561 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/binding-rules from= latency=1.458µs -12:43.561 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/force-leave/ from= latency=1.319µs -12:43.561 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/catalog/datacenters from= latency=1.457µs -12:43.561 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/health/state/ from= latency=1.439µs -12:43.561 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/catalog/nodes from= latency=1.322µs -12:43.561 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/connect/intentions/check from= latency=1.396µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/health/checks/ from= latency=1.713µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/config/ from= latency=1.512µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/connect/intentions from= latency=1.214µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/coordinate/update from= latency=1.081µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/auth-methods from= latency=1.082µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/members from= latency=1.277µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/catalog/connect/ from= latency=1.202µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/token/self from= latency=1.179µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/service/maintenance/ from= latency=1.235µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/role/ from= latency=1.415µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/binding-rule/ from= latency=1.483µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/check/warn/ from= latency=1.501µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/binding-rule from= latency=1.088µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/service/deregister/ from= latency=1.229µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/catalog/deregister from= latency=1.402µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/internal/federation-states from= latency=1.296µs -12:43.562 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/session/renew/ from= latency=1.258µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/info/ from= latency=1.339µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/token/ from= latency=1.131µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/metrics from= latency=1.276µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/event/list from= latency=1.144µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/health/connect/ from= latency=1.28µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/operator/keyring from= latency=1.677µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/snapshot from= latency=1.368µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/destroy/ from= latency=1.514µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/catalog/register from= latency=1.291µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/catalog/services from= latency=1.303µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/health/service/id/ from= latency=1.289µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/roles from= latency=1.254µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/catalog/node/ from= latency=1.046µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/txn from= latency=1.123µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/check/pass/ from= latency=1.141µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/catalog/gateway-services/ from= latency=1.085µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/coordinate/datacenters from= latency=1.339µs -12:43.563 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/internal/federation-state/ from= latency=1.191µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/logout from= latency=1.17µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/check/register from= latency=1.184µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/internal/ui/services from= latency=1.182µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/policy/ from= latency=1.247µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/token from= latency=1.347µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/leave from= latency=1.176µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/internal/ui/gateway-services-nodes/ from= latency=1.112µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/operator/raft/configuration from= latency=1.136µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/login from= latency=1.303µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/auth-method/ from= latency=1.294µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/self from= latency=1.1µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/clone/ from= latency=1.045µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/maintenance from= latency=1.225µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/catalog/service/ from= latency=1.399µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/create from= latency=1.481µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/check/fail/ from= latency=1.069µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/check/update/ from= latency=1.047µs -12:43.564 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/role from= latency=1.224µs -12:43.565 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/coordinate/node/ from= latency=1.192µs -12:43.565 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/session/info/ from= latency=1.149µs -12:43.565 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/services from= latency=1.596µs -12:43.565 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/connect/ca/configuration from= latency=1.38µs -12:43.565 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/connect/intentions/ from= latency=1.435µs -12:43.565 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/role/name/ from= latency=1.52µs -12:43.565 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/internal/federation-states/mesh-gateways from= latency=1.434µs -12:43.565 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/internal/ui/node/ from= latency=1.288µs -12:43.565 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/policies from= latency=1.349µs -12:43.565 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/join/ from= latency=1.283µs -12:43.565 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/kv/ from= latency=1.858µs -12:43.570 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/update from= latency=4.171µs -12:43.570 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/agent/checks from= latency=1.459µs -12:43.570 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/internal/ui/nodes from= latency=1.932µs -12:43.570 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/connect/ca/roots from= latency=1.203µs -12:43.570 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/bootstrap from= latency=1.386µs -12:43.570 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/acl/policy from= latency=1.171µs -12:43.570 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:17017/v1/session/list from= latency=1.2µs -12:43.570 [INFO] TestAgent: Requesting shutdown -12:43.570 [INFO] TestAgent.server: shutting down server -12:43.570 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:43.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:43.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:43.570 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:43.570 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:43.570 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:43.570 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:43.570 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:43.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:43.570 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:43.570 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:43.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:43.570 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:43.570 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:43.570 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:43.570 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:43.571Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:43.571Z [INFO] agent.router.manager: shutting down -12:43.571 [INFO] TestAgent: consul server down -12:43.571 [INFO] TestAgent: shutdown complete -12:43.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17016 network=tcp -12:43.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17016 network=udp -12:43.571 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17017 network=tcp -12:44.073 [INFO] TestAgent: Waiting for endpoints to shut down -12:44.073 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPAPI_OptionMethod_OSS (0.80s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query/xxx/execute (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query/xxx/explain (0.00s) +2022-12-04T05:48:56.244Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:56.244Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:56.256Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:56.256Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:56.257 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ce58d439-9b5e-0e4b-b02e-815d57ed7864 Address:127.0.0.1:24168}]" +48:56.257 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24168 [Follower]" leader= +48:56.257 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ce58d439-9b5e-0e4b-b02e-815d57ed7864.dc1 127.0.0.1 +48:56.258 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ce58d439-9b5e-0e4b-b02e-815d57ed7864 127.0.0.1 +2022-12-04T05:48:56.258Z [INFO] agent.router: Initializing LAN area manager +48:56.258 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24163 network=udp +48:56.258 [INFO] TestAgent.server: Adding LAN server: server="Node-ce58d439-9b5e-0e4b-b02e-815d57ed7864 (Addr: tcp/127.0.0.1:24168) (DC: dc1)" +48:56.258 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ce58d439-9b5e-0e4b-b02e-815d57ed7864.dc1 area=wan +48:56.258 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24163 network=tcp +48:56.259 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24164 network=tcp +48:56.259 [INFO] TestAgent: started state syncer +48:56.259 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24169 network=tcp +48:56.295 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:56.295 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24168 [Candidate]" term=2 +48:56.295 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:56.295 [DEBUG] TestAgent.server.raft: vote granted: from=ce58d439-9b5e-0e4b-b02e-815d57ed7864 term=2 tally=1 +48:56.295 [INFO] TestAgent.server.raft: election won: tally=1 +48:56.295 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24168 [Leader]" +48:56.295 [INFO] TestAgent.server: cluster leadership acquired +48:56.296 [INFO] TestAgent.server: initializing acls +48:56.296 [INFO] TestAgent.server: New leader elected: payload=Node-ce58d439-9b5e-0e4b-b02e-815d57ed7864 +48:56.296 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:56.296 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:56.296 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:56.296 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:56.296 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ce58d439-9b5e-0e4b-b02e-815d57ed7864 +48:56.297 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ce58d439-9b5e-0e4b-b02e-815d57ed7864.dc1 +48:56.297 [INFO] TestAgent.server: Updating LAN server: server="Node-ce58d439-9b5e-0e4b-b02e-815d57ed7864 (Addr: tcp/127.0.0.1:24168) (DC: dc1)" +48:56.297 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ce58d439-9b5e-0e4b-b02e-815d57ed7864.dc1 area=wan +48:56.297 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:56.299 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:56.299 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:56.299 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:56.299 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:56.299 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:56.299 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ce58d439-9b5e-0e4b-b02e-815d57ed7864 +48:56.299 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ce58d439-9b5e-0e4b-b02e-815d57ed7864 +48:56.299 [INFO] TestAgent.server: federation state anti-entropy synced +48:56.299 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ce58d439-9b5e-0e4b-b02e-815d57ed7864 +48:56.535 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/query from= latency=3.142µs +48:56.536 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/query/ from= latency=125.57µs +48:56.536 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/query/xxx/execute from= latency=29.175µs +48:56.536 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/query/xxx/explain from= latency=24.901µs +48:56.536 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/clone/ from= latency=1.642µs +48:56.536 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/service/ from= latency=1.419µs +48:56.536 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/connect/authorize from= latency=1.5µs +48:56.536 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/update from= latency=1.374µs +48:56.536 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/policies from= latency=1.478µs +48:56.536 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/self from= latency=1.593µs +48:56.536 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/health/state/ from= latency=1.558µs +48:56.536 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/session/info/ from= latency=1.428µs +48:56.536 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/role/ from= latency=1.404µs +48:56.536 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/coordinate/datacenters from= latency=1.476µs +48:56.537 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/internal/federation-state/ from= latency=1.813µs +48:56.537 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/health/node/ from= latency=1.262µs +48:56.537 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/session/create from= latency=1.409µs +48:56.537 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/create from= latency=1.308µs +48:56.537 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/catalog/nodes from= latency=2.005µs +48:56.537 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/catalog/service/ from= latency=1.49µs +48:56.537 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/destroy/ from= latency=1.278µs +48:56.537 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/role/name/ from= latency=1.196µs +48:56.537 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/coordinate/node/ from= latency=1.496µs +48:56.537 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/operator/autopilot/health from= latency=1.291µs +48:56.537 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/session/list from= latency=1.598µs +48:56.537 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/token/ from= latency=1.387µs +48:56.537 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/internal/ui/services from= latency=1.418µs +48:56.537 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/internal/ui/gateway-services-nodes/ from= latency=1.479µs +48:56.537 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/service/deregister/ from= latency=1.428µs +48:56.537 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/session/node/ from= latency=1.806µs +48:56.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/policy/name/ from= latency=1.659µs +48:56.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/config from= latency=1.48µs +48:56.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/kv/ from= latency=1.577µs +48:56.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/binding-rule/ from= latency=1.631µs +48:56.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/discovery-chain/ from= latency=1.338µs +48:56.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/event/fire/ from= latency=1.373µs +48:56.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/connect/ca/leaf/ from= latency=1.717µs +48:56.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/connect/intentions/check from= latency=1.246µs +48:56.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/catalog/register from= latency=1.372µs +48:56.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/health/connect/ from= latency=1.437µs +48:56.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/operator/raft/configuration from= latency=1.771µs +48:56.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/operator/autopilot/configuration from= latency=1.615µs +48:56.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/metrics from= latency=1.616µs +48:56.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/members from= latency=1.348µs +48:56.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/check/deregister/ from= latency=1.516µs +48:56.538 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/internal/federation-states/mesh-gateways from= latency=1.581µs +48:56.539 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/logout from= latency=1.549µs +48:56.539 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/health/service/id/ from= latency=1.296µs +48:56.539 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/check/pass/ from= latency=1.628µs +48:56.539 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/replication from= latency=1.235µs +48:56.539 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/roles from= latency=1.453µs +48:56.539 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/catalog/node/ from= latency=1.553µs +48:56.539 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/internal/federation-states from= latency=1.591µs +48:56.539 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/health/checks/ from= latency=1.615µs +48:56.539 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/health/service/ from= latency=3.141µs +48:56.539 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/health/service/name/ from= latency=1.491µs +48:56.539 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/coordinate/nodes from= latency=1.843µs +48:56.539 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/connect/ca/configuration from= latency=1.662µs +48:56.539 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/internal/ui/node/ from= latency=1.5µs +48:56.539 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/status/leader from= latency=1.814µs +48:56.540 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/snapshot from= latency=1.874µs +48:56.540 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/auth-methods from= latency=1.575µs +48:56.540 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/check/warn/ from= latency=1.707µs +48:56.540 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/token/self from= latency=1.727µs +48:56.540 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/maintenance from= latency=1.624µs +48:56.540 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/connect/intentions/match from= latency=1.866µs +48:56.540 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/health/ingress/ from= latency=1.875µs +48:56.540 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/list from= latency=1.9µs +48:56.540 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/catalog/services from= latency=1.927µs +48:56.540 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/catalog/node-services/ from= latency=1.675µs +48:56.540 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/connect/ca/roots from= latency=1.593µs +48:56.540 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/coordinate/update from= latency=1.425µs +48:56.540 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/bootstrap from= latency=1.782µs +48:56.541 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/token/ from= latency=2.188µs +48:56.541 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/event/list from= latency=1.807µs +48:56.541 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/internal/ui/nodes from= latency=2.192µs +48:56.541 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/txn from= latency=1.694µs +48:56.541 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/policy/ from= latency=2.135µs +48:56.541 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/service/register from= latency=1.66µs +48:56.541 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/connect/intentions/ from= latency=1.858µs +48:56.541 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/config/ from= latency=1.564µs +48:56.541 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/rules/translate/ from= latency=1.504µs +48:56.541 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/join/ from= latency=1.796µs +48:56.541 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/catalog/datacenters from= latency=1.771µs +48:56.541 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/auth-method from= latency=2.102µs +48:56.541 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/host from= latency=1.663µs +48:56.541 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/auth-method/ from= latency=1.802µs +48:56.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/check/register from= latency=1.309µs +48:56.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/catalog/connect/ from= latency=1.563µs +48:56.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/service/maintenance/ from= latency=1.502µs +48:56.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/catalog/deregister from= latency=1.356µs +48:56.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/info/ from= latency=1.527µs +48:56.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/binding-rules from= latency=1.638µs +48:56.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/token from= latency=1.385µs +48:56.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/tokens from= latency=1.586µs +48:56.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/services from= latency=1.417µs +48:56.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/connect/intentions from= latency=2.001µs +48:56.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/session/renew/ from= latency=1.509µs +48:56.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/policy from= latency=1.735µs +48:56.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/role from= latency=1.42µs +48:56.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/rules/translate from= latency=1.278µs +48:56.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/binding-rule from= latency=1.903µs +48:56.542 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/connect/ca/roots from= latency=1.26µs +48:56.543 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/force-leave/ from= latency=1.665µs +48:56.543 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/leave from= latency=1.37µs +48:56.543 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/check/fail/ from= latency=3.67µs +48:56.543 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/session/destroy/ from= latency=1.796µs +48:56.543 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/check/update/ from= latency=1.647µs +48:56.543 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/internal/acl/authorize from= latency=1.35µs +48:56.543 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/operator/raft/peer from= latency=1.345µs +48:56.543 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/operator/keyring from= latency=1.918µs +48:56.543 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/acl/login from= latency=1.217µs +48:56.543 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/agent/checks from= latency=1.511µs +48:56.543 [DEBUG] TestAgent.http: Request finished: method=OPTIONS url=http://127.0.0.1:24164/v1/catalog/gateway-services/ from= latency=1.279µs +48:56.543 [INFO] TestAgent: Requesting shutdown +48:56.543 [INFO] TestAgent.server: shutting down server +48:56.543 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:56.543 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:56.543 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:56.543 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:56.543 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:56.543 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:56.543 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:56.543 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:56.543 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:56.543 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:56.543 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:56.543 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:56.543 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:56.543 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:56.543 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:56.543 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:56.544Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:56.544Z [INFO] agent.router.manager: shutting down +48:56.544 [INFO] TestAgent: consul server down +48:56.544 [INFO] TestAgent: shutdown complete +48:56.544 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24163 network=tcp +48:56.544 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24163 network=udp +48:56.544 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24164 network=tcp +48:57.044 [INFO] TestAgent: Waiting for endpoints to shut down +48:57.044 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPAPI_OptionMethod_OSS (0.81s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/nodes (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/list (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/rules/translate/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/ca/leaf/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/event/fire/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/service/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-method (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/config (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/health/service/name/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/register (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/match (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/node/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/rules/translate (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/host (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy/name/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query/xxx/execute (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/query/xxx/explain (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/clone/ (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/raft/peer (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/replication (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/acl/authorize (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/destroy/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/status/leader (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/deregister/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/ingress/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/autopilot/health (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/node-services/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/discovery-chain/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/node/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/tokens (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/authorize (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/ca/roots (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/autopilot/configuration (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/create (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rules (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/force-leave/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/datacenters (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/update (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policies (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/self (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/state/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/nodes (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/check (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/checks/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/config/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/update (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-methods (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/members (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/connect/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token/self (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/info/ (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rule/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/warn/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rule (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/deregister/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/deregister (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-states (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/renew/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/info/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/token/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/metrics (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/event/list (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/connect/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/keyring (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/snapshot (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/datacenters (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-state/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/node/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/create (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/create (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/nodes (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/service/ (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role/name/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/node/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/autopilot/health (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/list (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/token/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/services (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/gateway-services-nodes/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/node/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy/name/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/config (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rule/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/discovery-chain/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/event/fire/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/ca/leaf/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/check (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/register (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/services (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/connect/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/raft/configuration (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/autopilot/configuration (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/metrics (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/members (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/deregister/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-states/mesh-gateways (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/logout (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/health/service/id/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/pass/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/replication (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/roles (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/node/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-states (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/checks/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/service/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/health/service/name/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/nodes (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/ca/configuration (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/node/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/status/leader (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/snapshot (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-methods (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/warn/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token/self (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/maintenance (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/match (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/health/ingress/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/list (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/services (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/node-services/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/update (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/bootstrap (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/event/list (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/nodes (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/txn (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/pass/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/gateway-services/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/datacenters (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-state/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/logout (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/register (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/services (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/register (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/config/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/rules/translate/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/join/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/datacenters (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-method (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/host (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/register (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/connect/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/deregister (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/info/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rules (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/token (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/tokens (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/services (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/renew/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/rules/translate (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/binding-rule (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/connect/ca/roots (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/force-leave/ (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/leave (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/gateway-services-nodes/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/raft/configuration (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/login (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/auth-method/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/self (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/clone/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/maintenance (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/service/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/create (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/destroy/ (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/check/update/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/coordinate/node/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/info/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/services (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/ca/configuration (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/intentions/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/role/name/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/federation-states/mesh-gateways (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/node/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policies (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/join/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/kv/ (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/update (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/acl/authorize (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/raft/peer (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/operator/keyring (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/login (0.00s) --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/agent/checks (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/internal/ui/nodes (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/connect/ca/roots (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/bootstrap (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/acl/policy (0.00s) - --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/session/list (0.00s) + --- PASS: TestHTTPAPI_OptionMethod_OSS/OPTIONS_/v1/catalog/gateway-services/ (0.00s) === RUN TestHTTPAPI_AllowedNets_OSS [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/internal/acl/authorize -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/destroy/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/deregister/ -=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/discovery-chain/ -=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/agent/connect/authorize -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/create -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/force-leave/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/operator/autopilot/configuration +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/rules/translate === RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/connect/intentions -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/coordinate/update -=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/config/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/maintenance/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/role/ -=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/role/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/binding-rule/ -=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/binding-rule/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/warn/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/binding-rule -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/deregister/ === RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/renew/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/token/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/catalog/deregister +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/policy +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/role +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/binding-rule +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/force-leave/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/leave +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/fail/ +=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/operator/raft/peer +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/destroy/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/update/ +=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/internal/acl/authorize === RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/operator/keyring === RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/operator/keyring === RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/operator/keyring -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/snapshot -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/destroy/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/catalog/register -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/txn -=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/logout -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/register -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/pass/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/policy/ -=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/policy/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/token -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/leave === RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/login -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/auth-method/ -=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/agent/connect/authorize === RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/clone/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/maintenance -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/create -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/fail/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/update/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/role -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/connect/ca/configuration -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/connect/intentions/ -=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/connect/intentions/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/join/ === RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/update +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/role/ +=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/role/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/create +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/create +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/destroy/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/token/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/deregister/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/config +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/event/fire/ === RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/kv/ === RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/kv/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/binding-rule/ +=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/binding-rule/ +=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/discovery-chain/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/deregister/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/catalog/register +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/operator/autopilot/configuration +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/pass/ +=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/logout +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/warn/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/connect/ca/configuration +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/snapshot +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/maintenance +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/coordinate/update === RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/bootstrap -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/policy -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/event/fire/ -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/auth-method -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/config -=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/register -=== RUN TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/rules/translate === RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/token/ === RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/token/ -=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/operator/raft/peer +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/connect/intentions/ +=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/connect/intentions/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/txn +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/policy/ +=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/policy/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/register +=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/config/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/join/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/auth-method +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/auth-method/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/register +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/token +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/maintenance/ +=== RUN TestHTTPAPI_AllowedNets_OSS/PUT_/v1/catalog/deregister [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:44.096Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:12:44.096Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:44.120Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:12:44.120Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:44.121 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fbeb4a09-493f-01e4-9e69-9550ea4eb505 Address:127.0.0.1:17028}]" -12:44.121 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17028 [Follower]" leader= -12:44.123 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fbeb4a09-493f-01e4-9e69-9550ea4eb505.dc1 127.0.0.1 -12:44.123 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fbeb4a09-493f-01e4-9e69-9550ea4eb505 127.0.0.1 -2021-10-31T23:12:44.123Z [INFO] agent.router: Initializing LAN area manager -12:44.123 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17023 network=udp -12:44.123 [INFO] TestAgent.server: Adding LAN server: server="Node-fbeb4a09-493f-01e4-9e69-9550ea4eb505 (Addr: tcp/127.0.0.1:17028) (DC: dc1)" -12:44.124 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fbeb4a09-493f-01e4-9e69-9550ea4eb505.dc1 area=wan -12:44.124 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17023 network=tcp -12:44.124 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17024 network=tcp -12:44.124 [INFO] TestAgent: started state syncer -12:44.124 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17029 network=tcp -12:44.177 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -12:44.181 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:44.181 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17028 [Candidate]" term=2 -12:44.182 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:44.182 [DEBUG] TestAgent.server.raft: vote granted: from=fbeb4a09-493f-01e4-9e69-9550ea4eb505 term=2 tally=1 -12:44.182 [INFO] TestAgent.server.raft: election won: tally=1 -12:44.182 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17028 [Leader]" -12:44.182 [INFO] TestAgent.server: cluster leadership acquired -12:44.182 [INFO] TestAgent.server: initializing acls -12:44.182 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:44.183 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:44.183 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:44.183 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:44.183 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-fbeb4a09-493f-01e4-9e69-9550ea4eb505 -12:44.183 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-fbeb4a09-493f-01e4-9e69-9550ea4eb505.dc1 -12:44.183 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:44.185 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:44.185 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:44.185 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:44.185 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:44.185 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:44.185 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fbeb4a09-493f-01e4-9e69-9550ea4eb505 -12:44.185 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fbeb4a09-493f-01e4-9e69-9550ea4eb505 -12:44.185 [INFO] TestAgent.server: New leader elected: payload=Node-fbeb4a09-493f-01e4-9e69-9550ea4eb505 -12:44.185 [INFO] TestAgent.server: Updating LAN server: server="Node-fbeb4a09-493f-01e4-9e69-9550ea4eb505 (Addr: tcp/127.0.0.1:17028) (DC: dc1)" -12:44.186 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-fbeb4a09-493f-01e4-9e69-9550ea4eb505.dc1 area=wan -12:44.194 [INFO] TestAgent.server: federation state anti-entropy synced -12:44.194 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fbeb4a09-493f-01e4-9e69-9550ea4eb505 -12:44.214 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:17024/v1/internal/acl/authorize from=192.168.1.2:5555 error="Access is restricted" -12:44.214 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:17024/v1/internal/acl/authorize from=192.168.1.2:5555 latency=78.376µs -12:44.214 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/session/destroy/ from=192.168.1.2:5555 error="Access is restricted" -12:44.214 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/session/destroy/ from=192.168.1.2:5555 latency=24.779µs -12:44.214 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/agent/check/deregister/ from=192.168.1.2:5555 error="Access is restricted" -12:44.214 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/agent/check/deregister/ from=192.168.1.2:5555 latency=23.349µs -12:44.214 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:17024/v1/discovery-chain/ from=192.168.1.2:5555 error="Access is restricted" -12:44.214 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:17024/v1/discovery-chain/ from=192.168.1.2:5555 latency=22.723µs -12:44.214 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:17024/v1/agent/connect/authorize from=192.168.1.2:5555 error="Access is restricted" -12:44.214 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:17024/v1/agent/connect/authorize from=192.168.1.2:5555 latency=23.402µs -12:44.214 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/session/create from=192.168.1.2:5555 error="Access is restricted" -12:44.214 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/session/create from=192.168.1.2:5555 latency=26.776µs -12:44.214 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/agent/force-leave/ from=192.168.1.2:5555 error="Access is restricted" -12:44.214 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/agent/force-leave/ from=192.168.1.2:5555 latency=22.452µs -12:44.215 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/operator/autopilot/configuration from=192.168.1.2:5555 error="Access is restricted" -12:44.215 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/operator/autopilot/configuration from=192.168.1.2:5555 latency=22.114µs -12:44.215 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:17024/v1/connect/intentions from=192.168.1.2:5555 error="Access is restricted" -12:44.215 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:17024/v1/connect/intentions from=192.168.1.2:5555 latency=22.216µs -12:44.215 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/coordinate/update from=192.168.1.2:5555 error="Access is restricted" -12:44.215 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/coordinate/update from=192.168.1.2:5555 latency=21.668µs -12:44.215 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:17024/v1/config/ from=192.168.1.2:5555 error="Access is restricted" -12:44.215 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:17024/v1/config/ from=192.168.1.2:5555 latency=22.776µs -12:44.215 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/agent/service/maintenance/ from=192.168.1.2:5555 error="Access is restricted" -12:44.215 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/agent/service/maintenance/ from=192.168.1.2:5555 latency=21.703µs -12:44.215 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/acl/role/ from=192.168.1.2:5555 error="Access is restricted" -12:44.215 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/acl/role/ from=192.168.1.2:5555 latency=20.977µs -12:44.215 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:17024/v1/acl/role/ from=192.168.1.2:5555 error="Access is restricted" -12:44.215 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:17024/v1/acl/role/ from=192.168.1.2:5555 latency=21.35µs -12:44.215 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/acl/binding-rule/ from=192.168.1.2:5555 error="Access is restricted" -12:44.215 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/acl/binding-rule/ from=192.168.1.2:5555 latency=22.214µs -12:44.215 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:17024/v1/acl/binding-rule/ from=192.168.1.2:5555 error="Access is restricted" -12:44.215 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:17024/v1/acl/binding-rule/ from=192.168.1.2:5555 latency=22.389µs -12:44.215 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/agent/check/warn/ from=192.168.1.2:5555 error="Access is restricted" -12:44.215 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/agent/check/warn/ from=192.168.1.2:5555 latency=35.999µs -12:44.215 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/acl/binding-rule from=192.168.1.2:5555 error="Access is restricted" -12:44.215 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/acl/binding-rule from=192.168.1.2:5555 latency=21.636µs -12:44.216 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/agent/service/deregister/ from=192.168.1.2:5555 error="Access is restricted" -12:44.216 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/agent/service/deregister/ from=192.168.1.2:5555 latency=22.04µs -12:44.216 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/session/renew/ from=192.168.1.2:5555 error="Access is restricted" -12:44.216 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/session/renew/ from=192.168.1.2:5555 latency=27.341µs -12:44.216 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/agent/token/ from=192.168.1.2:5555 error="Access is restricted" -12:44.216 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/agent/token/ from=192.168.1.2:5555 latency=22.551µs -12:44.216 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/catalog/deregister from=192.168.1.2:5555 error="Access is restricted" -12:44.216 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/catalog/deregister from=192.168.1.2:5555 latency=28.077µs -12:44.216 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:17024/v1/operator/keyring from=192.168.1.2:5555 error="Access is restricted" -12:44.216 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:17024/v1/operator/keyring from=192.168.1.2:5555 latency=22.153µs -12:44.216 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/operator/keyring from=192.168.1.2:5555 error="Access is restricted" -12:44.216 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/operator/keyring from=192.168.1.2:5555 latency=22.202µs -12:44.216 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:17024/v1/operator/keyring from=192.168.1.2:5555 error="Access is restricted" -12:44.216 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:17024/v1/operator/keyring from=192.168.1.2:5555 latency=21.825µs -12:44.216 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/snapshot from=192.168.1.2:5555 error="Access is restricted" -12:44.216 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/snapshot from=192.168.1.2:5555 latency=22.107µs -12:44.216 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/acl/destroy/ from=192.168.1.2:5555 error="Access is restricted" -12:44.216 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/acl/destroy/ from=192.168.1.2:5555 latency=22.47µs -12:44.216 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/catalog/register from=192.168.1.2:5555 error="Access is restricted" -12:44.216 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/catalog/register from=192.168.1.2:5555 latency=20.793µs -12:44.216 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/txn from=192.168.1.2:5555 error="Access is restricted" -12:44.216 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/txn from=192.168.1.2:5555 latency=19.109µs -12:44.216 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:17024/v1/acl/logout from=192.168.1.2:5555 error="Access is restricted" -12:44.216 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:17024/v1/acl/logout from=192.168.1.2:5555 latency=19.154µs -12:44.216 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/agent/check/register from=192.168.1.2:5555 error="Access is restricted" -12:44.217 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/agent/check/register from=192.168.1.2:5555 latency=19.01µs -12:44.217 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/agent/check/pass/ from=192.168.1.2:5555 error="Access is restricted" -12:44.217 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/agent/check/pass/ from=192.168.1.2:5555 latency=19.508µs -12:44.217 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/acl/policy/ from=192.168.1.2:5555 error="Access is restricted" -12:44.217 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/acl/policy/ from=192.168.1.2:5555 latency=18.411µs -12:44.217 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:17024/v1/acl/policy/ from=192.168.1.2:5555 error="Access is restricted" -12:44.217 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:17024/v1/acl/policy/ from=192.168.1.2:5555 latency=39.266µs -12:44.217 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/acl/token from=192.168.1.2:5555 error="Access is restricted" -12:44.217 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/acl/token from=192.168.1.2:5555 latency=19.102µs -12:44.217 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/agent/leave from=192.168.1.2:5555 error="Access is restricted" -12:44.217 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/agent/leave from=192.168.1.2:5555 latency=20.15µs -12:44.217 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:17024/v1/acl/login from=192.168.1.2:5555 error="Access is restricted" -12:44.217 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:17024/v1/acl/login from=192.168.1.2:5555 latency=18.915µs -12:44.217 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/acl/auth-method/ from=192.168.1.2:5555 error="Access is restricted" -12:44.217 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/acl/auth-method/ from=192.168.1.2:5555 latency=25.532µs -12:44.217 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:17024/v1/acl/auth-method/ from=192.168.1.2:5555 error="Access is restricted" -12:44.217 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:17024/v1/acl/auth-method/ from=192.168.1.2:5555 latency=18.652µs -12:44.217 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/acl/clone/ from=192.168.1.2:5555 error="Access is restricted" -12:44.217 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/acl/clone/ from=192.168.1.2:5555 latency=19.055µs -12:44.217 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/agent/maintenance from=192.168.1.2:5555 error="Access is restricted" -12:44.217 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/agent/maintenance from=192.168.1.2:5555 latency=18.31µs -12:44.217 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/acl/create from=192.168.1.2:5555 error="Access is restricted" -12:44.222 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/acl/create from=192.168.1.2:5555 latency=4.489597ms -12:44.222 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/agent/check/fail/ from=192.168.1.2:5555 error="Access is restricted" -12:44.222 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/agent/check/fail/ from=192.168.1.2:5555 latency=46.776µs -12:44.222 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/agent/check/update/ from=192.168.1.2:5555 error="Access is restricted" -12:44.222 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/agent/check/update/ from=192.168.1.2:5555 latency=27.894µs -12:44.222 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/acl/role from=192.168.1.2:5555 error="Access is restricted" -12:44.222 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/acl/role from=192.168.1.2:5555 latency=26.609µs -12:44.222 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/connect/ca/configuration from=192.168.1.2:5555 error="Access is restricted" -12:44.222 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/connect/ca/configuration from=192.168.1.2:5555 latency=23.973µs -12:44.222 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/connect/intentions/ from=192.168.1.2:5555 error="Access is restricted" -12:44.222 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/connect/intentions/ from=192.168.1.2:5555 latency=21.703µs -12:44.223 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:17024/v1/connect/intentions/ from=192.168.1.2:5555 error="Access is restricted" -12:44.223 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:17024/v1/connect/intentions/ from=192.168.1.2:5555 latency=21.452µs -12:44.223 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/agent/join/ from=192.168.1.2:5555 error="Access is restricted" -12:44.223 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/agent/join/ from=192.168.1.2:5555 latency=22.838µs -12:44.223 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/acl/update from=192.168.1.2:5555 error="Access is restricted" -12:44.223 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/acl/update from=192.168.1.2:5555 latency=22.604µs -12:44.223 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/kv/ from=192.168.1.2:5555 error="Access is restricted" -12:44.223 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/kv/ from=192.168.1.2:5555 latency=22.129µs -12:44.223 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:17024/v1/kv/ from=192.168.1.2:5555 error="Access is restricted" -12:44.223 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:17024/v1/kv/ from=192.168.1.2:5555 latency=21.144µs -12:44.223 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/acl/bootstrap from=192.168.1.2:5555 error="Access is restricted" -12:44.223 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/acl/bootstrap from=192.168.1.2:5555 latency=24.305µs -12:44.223 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/acl/policy from=192.168.1.2:5555 error="Access is restricted" -12:44.223 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/acl/policy from=192.168.1.2:5555 latency=26.903µs -12:44.223 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/event/fire/ from=192.168.1.2:5555 error="Access is restricted" -12:44.223 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/event/fire/ from=192.168.1.2:5555 latency=21.741µs -12:44.223 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/acl/auth-method from=192.168.1.2:5555 error="Access is restricted" -12:44.223 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/acl/auth-method from=192.168.1.2:5555 latency=21.645µs -12:44.223 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/config from=192.168.1.2:5555 error="Access is restricted" -12:44.223 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/config from=192.168.1.2:5555 latency=22.02µs -12:44.223 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/agent/service/register from=192.168.1.2:5555 error="Access is restricted" -12:44.223 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/agent/service/register from=192.168.1.2:5555 latency=21.9µs -12:44.223 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:17024/v1/acl/rules/translate from=192.168.1.2:5555 error="Access is restricted" -12:44.223 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:17024/v1/acl/rules/translate from=192.168.1.2:5555 latency=22.593µs -12:44.223 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:17024/v1/acl/token/ from=192.168.1.2:5555 error="Access is restricted" -12:44.224 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:17024/v1/acl/token/ from=192.168.1.2:5555 latency=22.019µs -12:44.224 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:17024/v1/acl/token/ from=192.168.1.2:5555 error="Access is restricted" -12:44.224 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:17024/v1/acl/token/ from=192.168.1.2:5555 latency=22.195µs -12:44.224 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:17024/v1/operator/raft/peer from=192.168.1.2:5555 error="Access is restricted" -12:44.224 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:17024/v1/operator/raft/peer from=192.168.1.2:5555 latency=22.678µs -12:44.224 [INFO] TestAgent: Requesting shutdown -12:44.224 [INFO] TestAgent.server: shutting down server -12:44.224 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:44.224 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:44.224 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:44.224 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:44.224 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:44.224 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:44.224 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:44.224 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:44.224 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:44.224 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:44.224 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:44.224 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:44.224 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:44.224 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:44.224 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:44.224 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:44.224Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:44.224Z [INFO] agent.router.manager: shutting down -12:44.224 [INFO] TestAgent: consul server down -12:44.224 [INFO] TestAgent: shutdown complete -12:44.224 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17023 network=tcp -12:44.224 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17023 network=udp -12:44.224 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17024 network=tcp -12:44.725 [INFO] TestAgent: Waiting for endpoints to shut down -12:44.725 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPAPI_AllowedNets_OSS (0.65s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/internal/acl/authorize (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/destroy/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/deregister/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/discovery-chain/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/agent/connect/authorize (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/create (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/force-leave/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/operator/autopilot/configuration (0.00s) +2022-12-04T05:48:57.078Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:57.078Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:57.091Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:48:57.091Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:57.093 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:70c86f96-93d2-c999-a6b2-58d4e68d4515 Address:127.0.0.1:24175}]" +48:57.094 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-70c86f96-93d2-c999-a6b2-58d4e68d4515.dc1 127.0.0.1 +48:57.094 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-70c86f96-93d2-c999-a6b2-58d4e68d4515 127.0.0.1 +2022-12-04T05:48:57.094Z [INFO] agent.router: Initializing LAN area manager +48:57.094 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24170 network=udp +48:57.094 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24175 [Follower]" leader= +48:57.095 [INFO] TestAgent.server: Adding LAN server: server="Node-70c86f96-93d2-c999-a6b2-58d4e68d4515 (Addr: tcp/127.0.0.1:24175) (DC: dc1)" +48:57.095 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-70c86f96-93d2-c999-a6b2-58d4e68d4515.dc1 area=wan +48:57.095 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24170 network=tcp +48:57.095 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24171 network=tcp +48:57.095 [INFO] TestAgent: started state syncer +48:57.095 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24176 network=tcp +48:57.145 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:57.151 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:57.151 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24175 [Candidate]" term=2 +48:57.151 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:57.151 [DEBUG] TestAgent.server.raft: vote granted: from=70c86f96-93d2-c999-a6b2-58d4e68d4515 term=2 tally=1 +48:57.151 [INFO] TestAgent.server.raft: election won: tally=1 +48:57.151 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24175 [Leader]" +48:57.151 [INFO] TestAgent.server: cluster leadership acquired +48:57.152 [INFO] TestAgent.server: initializing acls +48:57.152 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:57.152 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:57.152 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:57.152 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:57.152 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-70c86f96-93d2-c999-a6b2-58d4e68d4515 +48:57.152 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-70c86f96-93d2-c999-a6b2-58d4e68d4515.dc1 +48:57.153 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:57.154 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:57.154 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:57.154 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:57.154 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:57.154 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:57.154 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-70c86f96-93d2-c999-a6b2-58d4e68d4515 +48:57.154 [INFO] TestAgent.server: member joined, marking health alive: member=Node-70c86f96-93d2-c999-a6b2-58d4e68d4515 +48:57.154 [INFO] TestAgent.server: New leader elected: payload=Node-70c86f96-93d2-c999-a6b2-58d4e68d4515 +48:57.154 [INFO] TestAgent.server: Updating LAN server: server="Node-70c86f96-93d2-c999-a6b2-58d4e68d4515 (Addr: tcp/127.0.0.1:24175) (DC: dc1)" +48:57.154 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-70c86f96-93d2-c999-a6b2-58d4e68d4515 +48:57.155 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-70c86f96-93d2-c999-a6b2-58d4e68d4515.dc1 area=wan +48:57.155 [INFO] TestAgent.server: federation state anti-entropy synced +48:57.258 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:57.258 [INFO] TestAgent: Synced node info +48:57.258 [DEBUG] TestAgent: Node info in sync +48:57.353 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:24171/v1/acl/rules/translate from=192.168.1.2:5555 error="Access is restricted" +48:57.354 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:24171/v1/acl/rules/translate from=192.168.1.2:5555 latency=91.682µs +48:57.354 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:24171/v1/connect/intentions from=192.168.1.2:5555 error="Access is restricted" +48:57.354 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:24171/v1/connect/intentions from=192.168.1.2:5555 latency=25.956µs +48:57.354 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/session/renew/ from=192.168.1.2:5555 error="Access is restricted" +48:57.354 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/session/renew/ from=192.168.1.2:5555 latency=22.244µs +48:57.354 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/acl/policy from=192.168.1.2:5555 error="Access is restricted" +48:57.354 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/acl/policy from=192.168.1.2:5555 latency=22.981µs +48:57.354 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/acl/role from=192.168.1.2:5555 error="Access is restricted" +48:57.354 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/acl/role from=192.168.1.2:5555 latency=19.886µs +48:57.354 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/acl/binding-rule from=192.168.1.2:5555 error="Access is restricted" +48:57.354 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/acl/binding-rule from=192.168.1.2:5555 latency=22.76µs +48:57.354 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/agent/force-leave/ from=192.168.1.2:5555 error="Access is restricted" +48:57.354 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/agent/force-leave/ from=192.168.1.2:5555 latency=25.897µs +48:57.354 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/agent/leave from=192.168.1.2:5555 error="Access is restricted" +48:57.354 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/agent/leave from=192.168.1.2:5555 latency=29.911µs +48:57.354 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/agent/check/fail/ from=192.168.1.2:5555 error="Access is restricted" +48:57.354 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/agent/check/fail/ from=192.168.1.2:5555 latency=32.179µs +48:57.354 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:24171/v1/operator/raft/peer from=192.168.1.2:5555 error="Access is restricted" +48:57.354 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:24171/v1/operator/raft/peer from=192.168.1.2:5555 latency=21.723µs +48:57.354 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/session/destroy/ from=192.168.1.2:5555 error="Access is restricted" +48:57.354 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/session/destroy/ from=192.168.1.2:5555 latency=22.876µs +48:57.355 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/agent/check/update/ from=192.168.1.2:5555 error="Access is restricted" +48:57.355 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/agent/check/update/ from=192.168.1.2:5555 latency=23.062µs +48:57.355 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:24171/v1/internal/acl/authorize from=192.168.1.2:5555 error="Access is restricted" +48:57.355 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:24171/v1/internal/acl/authorize from=192.168.1.2:5555 latency=21.985µs +48:57.355 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:24171/v1/operator/keyring from=192.168.1.2:5555 error="Access is restricted" +48:57.355 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:24171/v1/operator/keyring from=192.168.1.2:5555 latency=21.502µs +48:57.355 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/operator/keyring from=192.168.1.2:5555 error="Access is restricted" +48:57.355 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/operator/keyring from=192.168.1.2:5555 latency=20.017µs +48:57.355 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:24171/v1/operator/keyring from=192.168.1.2:5555 error="Access is restricted" +48:57.355 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:24171/v1/operator/keyring from=192.168.1.2:5555 latency=19.383µs +48:57.355 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:24171/v1/acl/login from=192.168.1.2:5555 error="Access is restricted" +48:57.355 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:24171/v1/acl/login from=192.168.1.2:5555 latency=20.673µs +48:57.355 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:24171/v1/agent/connect/authorize from=192.168.1.2:5555 error="Access is restricted" +48:57.355 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:24171/v1/agent/connect/authorize from=192.168.1.2:5555 latency=21.053µs +48:57.355 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/acl/clone/ from=192.168.1.2:5555 error="Access is restricted" +48:57.355 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/acl/clone/ from=192.168.1.2:5555 latency=22.291µs +48:57.355 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/acl/update from=192.168.1.2:5555 error="Access is restricted" +48:57.355 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/acl/update from=192.168.1.2:5555 latency=23.89µs +48:57.355 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/acl/role/ from=192.168.1.2:5555 error="Access is restricted" +48:57.355 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/acl/role/ from=192.168.1.2:5555 latency=23.943µs +48:57.355 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:24171/v1/acl/role/ from=192.168.1.2:5555 error="Access is restricted" +48:57.355 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:24171/v1/acl/role/ from=192.168.1.2:5555 latency=31.134µs +48:57.355 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/session/create from=192.168.1.2:5555 error="Access is restricted" +48:57.355 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/session/create from=192.168.1.2:5555 latency=26.118µs +48:57.356 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/acl/create from=192.168.1.2:5555 error="Access is restricted" +48:57.356 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/acl/create from=192.168.1.2:5555 latency=30.234µs +48:57.356 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/acl/destroy/ from=192.168.1.2:5555 error="Access is restricted" +48:57.356 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/acl/destroy/ from=192.168.1.2:5555 latency=23.468µs +48:57.356 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/agent/token/ from=192.168.1.2:5555 error="Access is restricted" +48:57.356 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/agent/token/ from=192.168.1.2:5555 latency=23.515µs +48:57.356 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/agent/service/deregister/ from=192.168.1.2:5555 error="Access is restricted" +48:57.356 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/agent/service/deregister/ from=192.168.1.2:5555 latency=25.482µs +48:57.356 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/config from=192.168.1.2:5555 error="Access is restricted" +48:57.356 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/config from=192.168.1.2:5555 latency=22.723µs +48:57.356 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/event/fire/ from=192.168.1.2:5555 error="Access is restricted" +48:57.356 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/event/fire/ from=192.168.1.2:5555 latency=22.369µs +48:57.356 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/kv/ from=192.168.1.2:5555 error="Access is restricted" +48:57.356 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/kv/ from=192.168.1.2:5555 latency=21.9µs +48:57.356 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:24171/v1/kv/ from=192.168.1.2:5555 error="Access is restricted" +48:57.356 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:24171/v1/kv/ from=192.168.1.2:5555 latency=21.031µs +48:57.356 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/acl/binding-rule/ from=192.168.1.2:5555 error="Access is restricted" +48:57.356 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/acl/binding-rule/ from=192.168.1.2:5555 latency=21.79µs +48:57.356 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:24171/v1/acl/binding-rule/ from=192.168.1.2:5555 error="Access is restricted" +48:57.356 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:24171/v1/acl/binding-rule/ from=192.168.1.2:5555 latency=22.582µs +48:57.356 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:24171/v1/discovery-chain/ from=192.168.1.2:5555 error="Access is restricted" +48:57.357 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:24171/v1/discovery-chain/ from=192.168.1.2:5555 latency=22.71µs +48:57.357 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/agent/check/deregister/ from=192.168.1.2:5555 error="Access is restricted" +48:57.357 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/agent/check/deregister/ from=192.168.1.2:5555 latency=21.931µs +48:57.357 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/catalog/register from=192.168.1.2:5555 error="Access is restricted" +48:57.357 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/catalog/register from=192.168.1.2:5555 latency=21.999µs +48:57.357 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/operator/autopilot/configuration from=192.168.1.2:5555 error="Access is restricted" +48:57.357 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/operator/autopilot/configuration from=192.168.1.2:5555 latency=21.89µs +48:57.357 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/agent/check/pass/ from=192.168.1.2:5555 error="Access is restricted" +48:57.357 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/agent/check/pass/ from=192.168.1.2:5555 latency=22.015µs +48:57.357 [ERROR] TestAgent.http: Request error: method=POST url=http://127.0.0.1:24171/v1/acl/logout from=192.168.1.2:5555 error="Access is restricted" +48:57.357 [DEBUG] TestAgent.http: Request finished: method=POST url=http://127.0.0.1:24171/v1/acl/logout from=192.168.1.2:5555 latency=22.612µs +48:57.357 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/agent/check/warn/ from=192.168.1.2:5555 error="Access is restricted" +48:57.357 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/agent/check/warn/ from=192.168.1.2:5555 latency=33.738µs +48:57.357 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/connect/ca/configuration from=192.168.1.2:5555 error="Access is restricted" +48:57.357 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/connect/ca/configuration from=192.168.1.2:5555 latency=24.092µs +48:57.357 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/snapshot from=192.168.1.2:5555 error="Access is restricted" +48:57.357 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/snapshot from=192.168.1.2:5555 latency=21.08µs +48:57.357 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/agent/maintenance from=192.168.1.2:5555 error="Access is restricted" +48:57.357 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/agent/maintenance from=192.168.1.2:5555 latency=23.562µs +48:57.357 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/coordinate/update from=192.168.1.2:5555 error="Access is restricted" +48:57.357 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/coordinate/update from=192.168.1.2:5555 latency=22.172µs +48:57.357 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/acl/bootstrap from=192.168.1.2:5555 error="Access is restricted" +48:57.357 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/acl/bootstrap from=192.168.1.2:5555 latency=28.858µs +48:57.357 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/acl/token/ from=192.168.1.2:5555 error="Access is restricted" +48:57.357 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/acl/token/ from=192.168.1.2:5555 latency=18.608µs +48:57.358 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:24171/v1/acl/token/ from=192.168.1.2:5555 error="Access is restricted" +48:57.358 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:24171/v1/acl/token/ from=192.168.1.2:5555 latency=19.804µs +48:57.358 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/connect/intentions/ from=192.168.1.2:5555 error="Access is restricted" +48:57.358 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/connect/intentions/ from=192.168.1.2:5555 latency=19.547µs +48:57.358 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:24171/v1/connect/intentions/ from=192.168.1.2:5555 error="Access is restricted" +48:57.358 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:24171/v1/connect/intentions/ from=192.168.1.2:5555 latency=22.902µs +48:57.358 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/txn from=192.168.1.2:5555 error="Access is restricted" +48:57.358 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/txn from=192.168.1.2:5555 latency=20.147µs +48:57.358 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/acl/policy/ from=192.168.1.2:5555 error="Access is restricted" +48:57.358 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/acl/policy/ from=192.168.1.2:5555 latency=20.514µs +48:57.358 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:24171/v1/acl/policy/ from=192.168.1.2:5555 error="Access is restricted" +48:57.358 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:24171/v1/acl/policy/ from=192.168.1.2:5555 latency=20.169µs +48:57.358 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/agent/service/register from=192.168.1.2:5555 error="Access is restricted" +48:57.358 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/agent/service/register from=192.168.1.2:5555 latency=18.869µs +48:57.358 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:24171/v1/config/ from=192.168.1.2:5555 error="Access is restricted" +48:57.358 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:24171/v1/config/ from=192.168.1.2:5555 latency=23.323µs +48:57.358 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/agent/join/ from=192.168.1.2:5555 error="Access is restricted" +48:57.358 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/agent/join/ from=192.168.1.2:5555 latency=7.975µs +48:57.358 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/acl/auth-method from=192.168.1.2:5555 error="Access is restricted" +48:57.358 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/acl/auth-method from=192.168.1.2:5555 latency=28.137µs +48:57.358 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/acl/auth-method/ from=192.168.1.2:5555 error="Access is restricted" +48:57.358 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/acl/auth-method/ from=192.168.1.2:5555 latency=21.9µs +48:57.359 [ERROR] TestAgent.http: Request error: method=DELETE url=http://127.0.0.1:24171/v1/acl/auth-method/ from=192.168.1.2:5555 error="Access is restricted" +48:57.359 [DEBUG] TestAgent.http: Request finished: method=DELETE url=http://127.0.0.1:24171/v1/acl/auth-method/ from=192.168.1.2:5555 latency=20.297µs +48:57.359 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/agent/check/register from=192.168.1.2:5555 error="Access is restricted" +48:57.359 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/agent/check/register from=192.168.1.2:5555 latency=21.699µs +48:57.359 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/acl/token from=192.168.1.2:5555 error="Access is restricted" +48:57.359 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/acl/token from=192.168.1.2:5555 latency=19.035µs +48:57.359 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/agent/service/maintenance/ from=192.168.1.2:5555 error="Access is restricted" +48:57.359 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/agent/service/maintenance/ from=192.168.1.2:5555 latency=19.43µs +48:57.359 [ERROR] TestAgent.http: Request error: method=PUT url=http://127.0.0.1:24171/v1/catalog/deregister from=192.168.1.2:5555 error="Access is restricted" +48:57.359 [DEBUG] TestAgent.http: Request finished: method=PUT url=http://127.0.0.1:24171/v1/catalog/deregister from=192.168.1.2:5555 latency=20.242µs +48:57.359 [INFO] TestAgent: Requesting shutdown +48:57.359 [INFO] TestAgent.server: shutting down server +48:57.359 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:57.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:57.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:57.359 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:57.359 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:57.359 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:57.359 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:57.359 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:57.359 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:57.359 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:57.359 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:57.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:57.359 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:57.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:57.359 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:48:57.359Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:57.359Z [INFO] agent.router.manager: shutting down +48:57.360 [INFO] TestAgent: consul server down +48:57.360 [INFO] TestAgent: shutdown complete +48:57.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24170 network=tcp +48:57.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24170 network=udp +48:57.360 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24171 network=tcp +48:57.864 [INFO] TestAgent: Waiting for endpoints to shut down +48:57.864 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPAPI_AllowedNets_OSS (0.82s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/rules/translate (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/connect/intentions (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/coordinate/update (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/config/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/maintenance/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/role/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/role/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/binding-rule/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/binding-rule/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/warn/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/binding-rule (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/deregister/ (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/renew/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/token/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/catalog/deregister (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/policy (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/role (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/binding-rule (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/force-leave/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/leave (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/fail/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/operator/raft/peer (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/destroy/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/update/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/internal/acl/authorize (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/operator/keyring (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/operator/keyring (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/operator/keyring (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/snapshot (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/destroy/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/catalog/register (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/txn (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/logout (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/register (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/pass/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/policy/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/policy/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/token (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/leave (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/login (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/auth-method/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/agent/connect/authorize (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/clone/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/maintenance (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/create (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/fail/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/update/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/role (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/connect/ca/configuration (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/connect/intentions/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/connect/intentions/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/join/ (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/update (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/role/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/role/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/session/create (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/create (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/destroy/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/token/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/deregister/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/config (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/event/fire/ (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/kv/ (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/kv/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/binding-rule/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/binding-rule/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/discovery-chain/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/deregister/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/catalog/register (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/operator/autopilot/configuration (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/pass/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/logout (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/warn/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/connect/ca/configuration (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/snapshot (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/maintenance (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/coordinate/update (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/bootstrap (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/policy (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/event/fire/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/auth-method (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/config (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/register (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/POST_/v1/acl/rules/translate (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/token/ (0.00s) --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/token/ (0.00s) - --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/operator/raft/peer (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/connect/intentions/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/connect/intentions/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/txn (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/policy/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/policy/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/register (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/config/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/join/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/auth-method (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/DELETE_/v1/acl/auth-method/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/check/register (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/acl/token (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/agent/service/maintenance/ (0.00s) + --- PASS: TestHTTPAPI_AllowedNets_OSS/PUT_/v1/catalog/deregister (0.00s) === RUN TestHTTPServer_UnixSocket === PAUSE TestHTTPServer_UnixSocket === RUN TestHTTPServer_UnixSocket_FileExists @@ -18437,123 +18483,123 @@ [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:44.795Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:44.839Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:44.841 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d846596a-64c8-ae42-2307-7b552df634c2 Address:127.0.0.1:17035}]" -12:44.841 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17035 [Follower]" leader= -12:44.854 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d846596a-64c8-ae42-2307-7b552df634c2.dc1 127.0.0.1 -12:44.855 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d846596a-64c8-ae42-2307-7b552df634c2 127.0.0.1 -2021-10-31T23:12:44.855Z [INFO] agent.router: Initializing LAN area manager -12:44.855 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17030 network=udp -12:44.855 [INFO] TestAgent.server: Adding LAN server: server="Node-d846596a-64c8-ae42-2307-7b552df634c2 (Addr: tcp/127.0.0.1:17035) (DC: dc1)" -12:44.855 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d846596a-64c8-ae42-2307-7b552df634c2.dc1 area=wan -12:44.856 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17030 network=tcp -12:44.856 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17031 network=tcp -12:44.856 [INFO] TestAgent: started state syncer -12:44.856 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17036 network=tcp -12:44.893 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:44.894 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17035 [Candidate]" term=2 -12:44.894 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:44.894 [DEBUG] TestAgent.server.raft: vote granted: from=d846596a-64c8-ae42-2307-7b552df634c2 term=2 tally=1 -12:44.894 [INFO] TestAgent.server.raft: election won: tally=1 -12:44.894 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17035 [Leader]" -12:44.894 [INFO] TestAgent.server: cluster leadership acquired -12:44.894 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17035 -12:44.895 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:44.897 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:44.897 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:44.897 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:44.897 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:44.897 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:44.897 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d846596a-64c8-ae42-2307-7b552df634c2 -12:44.897 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d846596a-64c8-ae42-2307-7b552df634c2 -12:44.897 [INFO] TestAgent.server: New leader elected: payload=Node-d846596a-64c8-ae42-2307-7b552df634c2 -12:44.903 [INFO] TestAgent.server: federation state anti-entropy synced -12:45.110 [INFO] TestAgent: Requesting shutdown -12:45.110 [INFO] TestAgent.server: shutting down server -12:45.110 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:45.110 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:45.110 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:45.110 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:45.110 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:45.110 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:45.110 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:45.110 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:45.110 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:45.110 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:45.110 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:45.110 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:45.110Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:45.110Z [INFO] agent.router.manager: shutting down -12:45.110 [INFO] TestAgent: consul server down -12:45.110 [INFO] TestAgent: shutdown complete -12:45.110 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17030 network=tcp -12:45.110 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17030 network=udp -12:45.110 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17031 network=tcp -12:45.613 [INFO] TestAgent: Waiting for endpoints to shut down -12:45.613 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPAPI_Ban_Nonprintable_Characters (0.90s) +2022-12-04T05:48:57.883Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:57.908Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:57.911 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d4a01647-2009-2486-3172-4fce08f4f44e Address:127.0.0.1:24182}]" +48:57.911 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24182 [Follower]" leader= +48:57.911 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d4a01647-2009-2486-3172-4fce08f4f44e.dc1 127.0.0.1 +48:57.912 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d4a01647-2009-2486-3172-4fce08f4f44e 127.0.0.1 +2022-12-04T05:48:57.912Z [INFO] agent.router: Initializing LAN area manager +48:57.912 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d4a01647-2009-2486-3172-4fce08f4f44e.dc1 area=wan +48:57.912 [INFO] TestAgent.server: Adding LAN server: server="Node-d4a01647-2009-2486-3172-4fce08f4f44e (Addr: tcp/127.0.0.1:24182) (DC: dc1)" +48:57.912 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24177 network=udp +48:57.912 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24177 network=tcp +48:57.913 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24178 network=tcp +48:57.913 [INFO] TestAgent: started state syncer +48:57.913 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24183 network=tcp +48:57.979 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:57.979 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24182 [Candidate]" term=2 +48:57.979 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:57.979 [DEBUG] TestAgent.server.raft: vote granted: from=d4a01647-2009-2486-3172-4fce08f4f44e term=2 tally=1 +48:57.979 [INFO] TestAgent.server.raft: election won: tally=1 +48:57.979 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24182 [Leader]" +48:57.979 [INFO] TestAgent.server: cluster leadership acquired +48:57.979 [INFO] TestAgent.server: New leader elected: payload=Node-d4a01647-2009-2486-3172-4fce08f4f44e +48:57.980 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24182 +48:57.980 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:57.982 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:57.982 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:57.982 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:57.982 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:57.982 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:57.982 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d4a01647-2009-2486-3172-4fce08f4f44e +48:57.982 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d4a01647-2009-2486-3172-4fce08f4f44e +48:57.982 [INFO] TestAgent.server: federation state anti-entropy synced +48:58.085 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:58.085 [INFO] TestAgent: Synced node info +48:58.304 [INFO] TestAgent: Requesting shutdown +48:58.304 [INFO] TestAgent.server: shutting down server +48:58.304 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:58.304 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:58.304 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:58.304 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:58.304 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:58.304 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:58.304 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:58.304 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:58.304 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:58.304 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:58.304 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:58.304Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:58.304Z [INFO] agent.router.manager: shutting down +48:58.304 [INFO] TestAgent: consul server down +48:58.304 [INFO] TestAgent: shutdown complete +48:58.304 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24177 network=tcp +48:58.304 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24177 network=udp +48:58.304 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24178 network=tcp +48:58.805 [INFO] TestAgent: Waiting for endpoints to shut down +48:58.805 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPAPI_Ban_Nonprintable_Characters (0.94s) === RUN TestHTTPAPI_Allow_Nonprintable_Characters_With_Flag [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:45.657Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:45.716Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:45.726 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ed5d7671-3c16-d719-5479-387428975deb Address:127.0.0.1:17042}]" -12:45.727 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ed5d7671-3c16-d719-5479-387428975deb.dc1 127.0.0.1 -12:45.728 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ed5d7671-3c16-d719-5479-387428975deb 127.0.0.1 -2021-10-31T23:12:45.728Z [INFO] agent.router: Initializing LAN area manager -12:45.728 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17037 network=udp -12:45.728 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17042 [Follower]" leader= -12:45.728 [INFO] TestAgent.server: Adding LAN server: server="Node-ed5d7671-3c16-d719-5479-387428975deb (Addr: tcp/127.0.0.1:17042) (DC: dc1)" -12:45.728 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ed5d7671-3c16-d719-5479-387428975deb.dc1 area=wan -12:45.728 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17037 network=tcp -12:45.729 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17038 network=tcp -12:45.729 [INFO] TestAgent: started state syncer -12:45.729 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17043 network=tcp -12:45.785 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:45.785 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17042 [Candidate]" term=2 -12:45.786 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:45.786 [DEBUG] TestAgent.server.raft: vote granted: from=ed5d7671-3c16-d719-5479-387428975deb term=2 tally=1 -12:45.786 [INFO] TestAgent.server.raft: election won: tally=1 -12:45.786 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17042 [Leader]" -12:45.786 [INFO] TestAgent.server: cluster leadership acquired -12:45.786 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17042 -12:45.787 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:45.788 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:45.788 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:45.788 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:45.788 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:45.788 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:45.788 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ed5d7671-3c16-d719-5479-387428975deb -12:45.788 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ed5d7671-3c16-d719-5479-387428975deb -12:45.788 [INFO] TestAgent.server: New leader elected: payload=Node-ed5d7671-3c16-d719-5479-387428975deb -12:45.789 [INFO] TestAgent.server: federation state anti-entropy synced -12:45.930 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/bad%00ness from= latency=130.597µs -12:45.930 [INFO] TestAgent: Requesting shutdown -12:45.930 [INFO] TestAgent.server: shutting down server -12:45.930 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:45.930 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:45.930 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:45.930 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:45.930 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:45.930 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:45.930 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:45.930 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:45.930 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:45.930 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:45.930 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:45.930 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:45.930Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:45.930Z [INFO] agent.router.manager: shutting down -12:45.930 [INFO] TestAgent: consul server down -12:45.930 [INFO] TestAgent: shutdown complete -12:45.930 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17037 network=tcp -12:45.930 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17037 network=udp -12:45.930 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17038 network=tcp -12:46.433 [INFO] TestAgent: Waiting for endpoints to shut down -12:46.433 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPAPI_Allow_Nonprintable_Characters_With_Flag (0.81s) +2022-12-04T05:48:58.819Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:58.832Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:58.834 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ff3f4a17-c0a5-368e-c55d-ad54b31fab45 Address:127.0.0.1:24189}]" +48:58.835 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ff3f4a17-c0a5-368e-c55d-ad54b31fab45.dc1 127.0.0.1 +48:58.835 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ff3f4a17-c0a5-368e-c55d-ad54b31fab45 127.0.0.1 +2022-12-04T05:48:58.835Z [INFO] agent.router: Initializing LAN area manager +48:58.835 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24184 network=udp +48:58.836 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24189 [Follower]" leader= +48:58.836 [INFO] TestAgent.server: Adding LAN server: server="Node-ff3f4a17-c0a5-368e-c55d-ad54b31fab45 (Addr: tcp/127.0.0.1:24189) (DC: dc1)" +48:58.836 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ff3f4a17-c0a5-368e-c55d-ad54b31fab45.dc1 area=wan +48:58.836 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24184 network=tcp +48:58.839 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24185 network=tcp +48:58.839 [INFO] TestAgent: started state syncer +48:58.839 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24190 network=tcp +48:58.876 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:58.876 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24189 [Candidate]" term=2 +48:58.877 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:58.877 [DEBUG] TestAgent.server.raft: vote granted: from=ff3f4a17-c0a5-368e-c55d-ad54b31fab45 term=2 tally=1 +48:58.877 [INFO] TestAgent.server.raft: election won: tally=1 +48:58.877 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24189 [Leader]" +48:58.877 [INFO] TestAgent.server: cluster leadership acquired +48:58.877 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24189 +48:58.877 [INFO] TestAgent.server: New leader elected: payload=Node-ff3f4a17-c0a5-368e-c55d-ad54b31fab45 +48:58.878 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:58.879 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:58.879 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:58.879 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:58.879 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:58.879 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:58.879 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ff3f4a17-c0a5-368e-c55d-ad54b31fab45 +48:58.879 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ff3f4a17-c0a5-368e-c55d-ad54b31fab45 +48:58.880 [INFO] TestAgent.server: federation state anti-entropy synced +48:58.910 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/bad%00ness from= latency=107.535µs +48:58.910 [INFO] TestAgent: Requesting shutdown +48:58.911 [INFO] TestAgent.server: shutting down server +48:58.911 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:58.911 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:58.911 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:58.911 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:58.911 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:58.911 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:58.911 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:58.911 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:58.911 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:58.911 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:58.911 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:58.911 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:58.911Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:58.911Z [INFO] agent.router.manager: shutting down +48:58.911 [INFO] TestAgent: consul server down +48:58.911 [INFO] TestAgent: shutdown complete +48:58.911 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24184 network=tcp +48:58.911 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24184 network=udp +48:58.911 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24185 network=tcp +48:59.412 [INFO] TestAgent: Waiting for endpoints to shut down +48:59.412 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPAPI_Allow_Nonprintable_Characters_With_Flag (0.61s) === RUN TestHTTPAPI_TranslateAddrHeader === PAUSE TestHTTPAPI_TranslateAddrHeader === RUN TestHTTPAPIResponseHeaders @@ -18617,63 +18663,62 @@ === RUN TestParseConsistencyAndMaxStale [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:46.462Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:46.492Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:46.506 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cb470eab-e919-0375-80a8-94620e2d1e3d Address:127.0.0.1:17049}]" -12:46.507 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cb470eab-e919-0375-80a8-94620e2d1e3d.dc1 127.0.0.1 -12:46.507 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cb470eab-e919-0375-80a8-94620e2d1e3d 127.0.0.1 -2021-10-31T23:12:46.507Z [INFO] agent.router: Initializing LAN area manager -12:46.507 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17044 network=udp -12:46.508 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17049 [Follower]" leader= -12:46.508 [INFO] TestAgent.server: Adding LAN server: server="Node-cb470eab-e919-0375-80a8-94620e2d1e3d (Addr: tcp/127.0.0.1:17049) (DC: dc1)" -12:46.508 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cb470eab-e919-0375-80a8-94620e2d1e3d.dc1 area=wan -12:46.508 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17044 network=tcp -12:46.508 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17045 network=tcp -12:46.508 [INFO] TestAgent: started state syncer -12:46.508 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17050 network=tcp -12:46.556 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:46.556 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17049 [Candidate]" term=2 -12:46.556 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:46.557 [DEBUG] TestAgent.server.raft: vote granted: from=cb470eab-e919-0375-80a8-94620e2d1e3d term=2 tally=1 -12:46.557 [INFO] TestAgent.server.raft: election won: tally=1 -12:46.557 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17049 [Leader]" -12:46.557 [INFO] TestAgent.server: cluster leadership acquired -12:46.557 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17049 -12:46.557 [INFO] TestAgent.server: New leader elected: payload=Node-cb470eab-e919-0375-80a8-94620e2d1e3d -12:46.557 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:46.559 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:46.559 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:46.559 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:46.559 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:46.559 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:46.559 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cb470eab-e919-0375-80a8-94620e2d1e3d -12:46.559 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cb470eab-e919-0375-80a8-94620e2d1e3d -12:46.559 [INFO] TestAgent.server: federation state anti-entropy synced -12:46.638 [INFO] TestAgent: Requesting shutdown -12:46.638 [INFO] TestAgent.server: shutting down server -12:46.638 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:46.638 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:46.638 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:46.638 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:46.638 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:46.638 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:46.638 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:46.638 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:46.638 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:46.638 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:46.638 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:46.638 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:46.638Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:46.638Z [INFO] agent.router.manager: shutting down -12:46.638 [INFO] TestAgent: consul server down -12:46.638 [INFO] TestAgent: shutdown complete -12:46.638 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17044 network=tcp -12:46.638 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17044 network=udp -12:46.638 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17045 network=tcp -12:47.141 [INFO] TestAgent: Waiting for endpoints to shut down -12:47.141 [INFO] TestAgent: Endpoints down ---- PASS: TestParseConsistencyAndMaxStale (0.71s) +2022-12-04T05:48:59.426Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:59.438Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:59.439 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7b0353ac-4f9e-e5ca-c5a0-29719e3b9293 Address:127.0.0.1:24196}]" +48:59.440 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24196 [Follower]" leader= +48:59.440 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7b0353ac-4f9e-e5ca-c5a0-29719e3b9293.dc1 127.0.0.1 +48:59.440 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7b0353ac-4f9e-e5ca-c5a0-29719e3b9293 127.0.0.1 +2022-12-04T05:48:59.440Z [INFO] agent.router: Initializing LAN area manager +48:59.441 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24191 network=udp +48:59.441 [INFO] TestAgent.server: Adding LAN server: server="Node-7b0353ac-4f9e-e5ca-c5a0-29719e3b9293 (Addr: tcp/127.0.0.1:24196) (DC: dc1)" +48:59.441 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7b0353ac-4f9e-e5ca-c5a0-29719e3b9293.dc1 area=wan +48:59.441 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24191 network=tcp +48:59.442 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24192 network=tcp +48:59.442 [INFO] TestAgent: started state syncer +48:59.442 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24197 network=tcp +48:59.475 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:59.475 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24196 [Candidate]" term=2 +48:59.475 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:59.475 [DEBUG] TestAgent.server.raft: vote granted: from=7b0353ac-4f9e-e5ca-c5a0-29719e3b9293 term=2 tally=1 +48:59.475 [INFO] TestAgent.server.raft: election won: tally=1 +48:59.475 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24196 [Leader]" +48:59.475 [INFO] TestAgent.server: cluster leadership acquired +48:59.476 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24196 +48:59.476 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:59.477 [INFO] TestAgent.server: New leader elected: payload=Node-7b0353ac-4f9e-e5ca-c5a0-29719e3b9293 +48:59.478 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:59.478 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:59.478 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:59.478 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:59.478 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:59.478 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7b0353ac-4f9e-e5ca-c5a0-29719e3b9293 +48:59.478 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7b0353ac-4f9e-e5ca-c5a0-29719e3b9293 +48:59.479 [INFO] TestAgent.server: federation state anti-entropy synced +48:59.492 [INFO] TestAgent: Requesting shutdown +48:59.492 [INFO] TestAgent.server: shutting down server +48:59.492 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:59.492 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:59.492 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:59.492 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:59.492 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:59.492 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:59.492 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:59.492 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:59.492 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:59.492 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:59.492 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:59.492 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:59.492Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:59.492Z [INFO] agent.router.manager: shutting down +48:59.492 [INFO] TestAgent: consul server down +48:59.492 [INFO] TestAgent: shutdown complete +48:59.492 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24191 network=tcp +48:59.492 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24191 network=udp +48:59.492 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24192 network=tcp +48:59.993 [INFO] TestAgent: Waiting for endpoints to shut down +48:59.993 [INFO] TestAgent: Endpoints down +--- PASS: TestParseConsistencyAndMaxStale (0.58s) === RUN TestParseConsistency_Invalid === PAUSE TestParseConsistency_Invalid === RUN TestACLResolution @@ -18765,7 +18810,7 @@ === RUN TestNewNodeIDFromConfig/valid_NodeID_in_config === RUN TestNewNodeIDFromConfig/invalid_NodeID_in_file === RUN TestNewNodeIDFromConfig/valid_NodeID_in_file ---- PASS: TestNewNodeIDFromConfig (0.01s) +--- PASS: TestNewNodeIDFromConfig (0.00s) --- PASS: TestNewNodeIDFromConfig/a_new_ID_is_generated_when_none_is_specified (0.00s) --- PASS: TestNewNodeIDFromConfig/running_again_should_get_the_NodeID_that_was_persisted_to_disk (0.00s) --- PASS: TestNewNodeIDFromConfig/invalid_NodeID_in_config (0.00s) @@ -18775,9 +18820,9 @@ === RUN TestMakeNodeID === RUN TestMakeNodeID/Random_ID_when_HostNodeID_is_disabled === RUN TestMakeNodeID/host-based_ID_when_HostNodeID_is_enabled ---- PASS: TestMakeNodeID (0.03s) +--- PASS: TestMakeNodeID (0.00s) --- PASS: TestMakeNodeID/Random_ID_when_HostNodeID_is_disabled (0.00s) - --- PASS: TestMakeNodeID/host-based_ID_when_HostNodeID_is_enabled (0.03s) + --- PASS: TestMakeNodeID/host-based_ID_when_HostNodeID_is_enabled (0.00s) === RUN TestNotifyGroup --- PASS: TestNotifyGroup (0.00s) === RUN TestNotifyGroup_Clear @@ -18860,246 +18905,251 @@ [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:47.223Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:47.266Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:47.267 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b533d562-0635-87b7-b4dd-1625673a566d Address:127.0.0.1:17056}]" -12:47.268 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b533d562-0635-87b7-b4dd-1625673a566d.dc1 127.0.0.1 -12:47.268 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b533d562-0635-87b7-b4dd-1625673a566d 127.0.0.1 -2021-10-31T23:12:47.268Z [INFO] agent.router: Initializing LAN area manager -12:47.269 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17051 network=udp -12:47.269 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17056 [Follower]" leader= -12:47.269 [INFO] TestAgent.server: Adding LAN server: server="Node-b533d562-0635-87b7-b4dd-1625673a566d (Addr: tcp/127.0.0.1:17056) (DC: dc1)" -12:47.269 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b533d562-0635-87b7-b4dd-1625673a566d.dc1 area=wan -12:47.269 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17051 network=tcp -12:47.274 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17052 network=tcp -12:47.275 [INFO] TestAgent: started state syncer -12:47.275 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17057 network=tcp -12:47.342 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:47.342 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17056 [Candidate]" term=2 -12:47.342 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:47.342 [DEBUG] TestAgent.server.raft: vote granted: from=b533d562-0635-87b7-b4dd-1625673a566d term=2 tally=1 -12:47.342 [INFO] TestAgent.server.raft: election won: tally=1 -12:47.342 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17056 [Leader]" -12:47.342 [INFO] TestAgent.server: cluster leadership acquired -12:47.342 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17056 -12:47.343 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:47.344 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:47.344 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:47.344 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:47.344 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:47.344 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:47.344 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b533d562-0635-87b7-b4dd-1625673a566d -12:47.344 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b533d562-0635-87b7-b4dd-1625673a566d -12:47.345 [INFO] TestAgent.server: New leader elected: payload=Node-b533d562-0635-87b7-b4dd-1625673a566d -12:47.345 [INFO] TestAgent.server: federation state anti-entropy synced -12:47.491 [INFO] TestAgent: Requesting shutdown -12:47.491 [INFO] TestAgent.server: shutting down server -12:47.491 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:47.491 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:47.491 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:47.491 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:47.491 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:47.491 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:47.491 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:47.491 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:47.491 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:47.491 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:47.491 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:47.491 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:47.491Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:47.491Z [INFO] agent.router.manager: shutting down -12:47.491 [INFO] TestAgent: consul server down -12:47.491 [INFO] TestAgent: shutdown complete -12:47.491 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17051 network=tcp -12:47.491 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17051 network=udp -12:47.491 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17052 network=tcp -12:47.993 [INFO] TestAgent: Waiting for endpoints to shut down -12:47.993 [INFO] TestAgent: Endpoints down ---- PASS: TestServiceManager_RegisterService (0.82s) +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:00.014Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:00.025Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:00.026 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:58640fab-7aaa-58c7-5286-405f7c8b522d Address:127.0.0.1:24203}]" +49:00.026 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24203 [Follower]" leader= +49:00.027 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-58640fab-7aaa-58c7-5286-405f7c8b522d.dc1 127.0.0.1 +49:00.027 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-58640fab-7aaa-58c7-5286-405f7c8b522d 127.0.0.1 +2022-12-04T05:49:00.027Z [INFO] agent.router: Initializing LAN area manager +49:00.027 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24198 network=udp +49:00.027 [INFO] TestAgent.server: Adding LAN server: server="Node-58640fab-7aaa-58c7-5286-405f7c8b522d (Addr: tcp/127.0.0.1:24203) (DC: dc1)" +49:00.027 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-58640fab-7aaa-58c7-5286-405f7c8b522d.dc1 area=wan +49:00.027 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24198 network=tcp +49:00.028 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24199 network=tcp +49:00.028 [INFO] TestAgent: started state syncer +49:00.028 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24204 network=tcp +49:00.085 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:00.085 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24203 [Candidate]" term=2 +49:00.085 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:00.085 [DEBUG] TestAgent.server.raft: vote granted: from=58640fab-7aaa-58c7-5286-405f7c8b522d term=2 tally=1 +49:00.085 [INFO] TestAgent.server.raft: election won: tally=1 +49:00.085 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24203 [Leader]" +49:00.085 [INFO] TestAgent.server: cluster leadership acquired +49:00.085 [INFO] TestAgent.server: New leader elected: payload=Node-58640fab-7aaa-58c7-5286-405f7c8b522d +49:00.085 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24203 +49:00.086 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:00.087 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:00.087 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:00.087 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:00.087 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:00.087 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:00.087 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-58640fab-7aaa-58c7-5286-405f7c8b522d +49:00.087 [INFO] TestAgent.server: member joined, marking health alive: member=Node-58640fab-7aaa-58c7-5286-405f7c8b522d +49:00.088 [INFO] TestAgent.server: federation state anti-entropy synced +49:00.229 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:00.229 [INFO] TestAgent: Synced node info +49:00.229 [DEBUG] TestAgent: Node info in sync +49:00.389 [INFO] TestAgent: Requesting shutdown +49:00.389 [INFO] TestAgent.server: shutting down server +49:00.389 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:00.389 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:00.389 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:00.389 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:00.389 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:00.389 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:00.389 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:00.389 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:00.389 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:00.389 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:00.389 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:00.389Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:00.389Z [INFO] agent.router.manager: shutting down +49:00.389 [INFO] TestAgent: consul server down +49:00.389 [INFO] TestAgent: shutdown complete +49:00.389 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24198 network=tcp +49:00.389 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24198 network=udp +49:00.389 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24199 network=tcp +49:00.892 [INFO] TestAgent: Waiting for endpoints to shut down +49:00.892 [INFO] TestAgent: Endpoints down +--- PASS: TestServiceManager_RegisterService (0.89s) === RUN TestServiceManager_RegisterSidecar [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:48.029Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:48.091Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:48.093 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4f83ddfe-25fc-8018-a210-287c01411229 Address:127.0.0.1:17063}]" -12:48.094 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17063 [Follower]" leader= -12:48.098 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4f83ddfe-25fc-8018-a210-287c01411229.dc1 127.0.0.1 -12:48.098 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4f83ddfe-25fc-8018-a210-287c01411229 127.0.0.1 -2021-10-31T23:12:48.098Z [INFO] agent.router: Initializing LAN area manager -12:48.099 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17058 network=udp -12:48.099 [INFO] TestAgent.server: Adding LAN server: server="Node-4f83ddfe-25fc-8018-a210-287c01411229 (Addr: tcp/127.0.0.1:17063) (DC: dc1)" -12:48.099 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4f83ddfe-25fc-8018-a210-287c01411229.dc1 area=wan -12:48.099 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17058 network=tcp -12:48.100 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17059 network=tcp -12:48.100 [INFO] TestAgent: started state syncer -12:48.100 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17064 network=tcp -12:48.159 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:48.159 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17063 [Candidate]" term=2 -12:48.159 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:48.159 [DEBUG] TestAgent.server.raft: vote granted: from=4f83ddfe-25fc-8018-a210-287c01411229 term=2 tally=1 -12:48.159 [INFO] TestAgent.server.raft: election won: tally=1 -12:48.159 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17063 [Leader]" -12:48.159 [INFO] TestAgent.server: cluster leadership acquired -12:48.159 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17063 -12:48.160 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:48.161 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:48.161 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:48.161 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:48.161 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:48.161 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:48.161 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4f83ddfe-25fc-8018-a210-287c01411229 -12:48.161 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4f83ddfe-25fc-8018-a210-287c01411229 -12:48.161 [INFO] TestAgent.server: New leader elected: payload=Node-4f83ddfe-25fc-8018-a210-287c01411229 -12:48.162 [INFO] TestAgent.server: federation state anti-entropy synced -12:48.314 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy -12:48.314 [INFO] TestAgent: Requesting shutdown -12:48.315 [INFO] TestAgent.server: shutting down server -12:48.315 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:48.315 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:48.315 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:48.315 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:48.315 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:48.315 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:48.315 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:48.315 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:48.315 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:48.315 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:48.315 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:48.315 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:48.315Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:48.315Z [INFO] agent.router.manager: shutting down -12:48.315 [INFO] TestAgent: consul server down -12:48.315 [INFO] TestAgent: shutdown complete -12:48.315 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17058 network=tcp -12:48.315 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17058 network=udp -12:48.315 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17059 network=tcp -12:48.817 [INFO] TestAgent: Waiting for endpoints to shut down -12:48.817 [INFO] TestAgent: Endpoints down ---- PASS: TestServiceManager_RegisterSidecar (0.82s) +2022-12-04T05:49:00.914Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:00.945Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:00.946 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5859ef3b-e12e-db3d-a334-dad5b41afcc2 Address:127.0.0.1:24210}]" +49:00.947 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5859ef3b-e12e-db3d-a334-dad5b41afcc2.dc1 127.0.0.1 +49:00.948 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5859ef3b-e12e-db3d-a334-dad5b41afcc2 127.0.0.1 +2022-12-04T05:49:00.948Z [INFO] agent.router: Initializing LAN area manager +49:00.948 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24205 network=udp +49:00.948 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24210 [Follower]" leader= +49:00.948 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5859ef3b-e12e-db3d-a334-dad5b41afcc2.dc1 area=wan +49:00.948 [INFO] TestAgent.server: Adding LAN server: server="Node-5859ef3b-e12e-db3d-a334-dad5b41afcc2 (Addr: tcp/127.0.0.1:24210) (DC: dc1)" +49:00.948 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24205 network=tcp +49:00.953 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24206 network=tcp +49:00.953 [INFO] TestAgent: started state syncer +49:00.953 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24211 network=tcp +49:00.997 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:00.997 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24210 [Candidate]" term=2 +49:00.997 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:00.997 [DEBUG] TestAgent.server.raft: vote granted: from=5859ef3b-e12e-db3d-a334-dad5b41afcc2 term=2 tally=1 +49:00.997 [INFO] TestAgent.server.raft: election won: tally=1 +49:00.997 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24210 [Leader]" +49:00.997 [INFO] TestAgent.server: cluster leadership acquired +49:00.998 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24210 +49:00.998 [INFO] TestAgent.server: New leader elected: payload=Node-5859ef3b-e12e-db3d-a334-dad5b41afcc2 +49:00.998 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:01.000 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:01.000 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:01.000 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:01.000 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:01.000 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:01.000 [INFO] TestAgent.server: federation state anti-entropy synced +49:01.001 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5859ef3b-e12e-db3d-a334-dad5b41afcc2 +49:01.001 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5859ef3b-e12e-db3d-a334-dad5b41afcc2 +49:01.017 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy +49:01.017 [INFO] TestAgent: Requesting shutdown +49:01.018 [INFO] TestAgent.server: shutting down server +49:01.018 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:01.018 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:01.018 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:01.018 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:01.018 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:01.018 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:01.018 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:01.018 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:01.018 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:01.018 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:01.018 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:01.018 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:01.018Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:01.018Z [INFO] agent.router.manager: shutting down +49:01.018 [INFO] TestAgent: consul server down +49:01.018 [INFO] TestAgent: shutdown complete +49:01.018 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24205 network=tcp +49:01.018 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24205 network=udp +49:01.018 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24206 network=tcp +49:01.519 [INFO] TestAgent: Waiting for endpoints to shut down +49:01.519 [INFO] TestAgent: Endpoints down +--- PASS: TestServiceManager_RegisterSidecar (0.63s) === RUN TestServiceManager_RegisterMeshGateway [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:48.836Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:48.861Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:48.863 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e9517960-b0f9-ff9a-6597-ed029675d22f Address:127.0.0.1:17070}]" -12:48.864 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e9517960-b0f9-ff9a-6597-ed029675d22f.dc1 127.0.0.1 -12:48.864 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e9517960-b0f9-ff9a-6597-ed029675d22f 127.0.0.1 -2021-10-31T23:12:48.864Z [INFO] agent.router: Initializing LAN area manager -12:48.864 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17065 network=udp -12:48.864 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17070 [Follower]" leader= -12:48.865 [INFO] TestAgent.server: Adding LAN server: server="Node-e9517960-b0f9-ff9a-6597-ed029675d22f (Addr: tcp/127.0.0.1:17070) (DC: dc1)" -12:48.865 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e9517960-b0f9-ff9a-6597-ed029675d22f.dc1 area=wan -12:48.874 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17065 network=tcp -12:48.874 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17066 network=tcp -12:48.874 [INFO] TestAgent: started state syncer -12:48.874 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17071 network=tcp -12:48.910 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:48.910 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17070 [Candidate]" term=2 -12:48.910 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:48.910 [DEBUG] TestAgent.server.raft: vote granted: from=e9517960-b0f9-ff9a-6597-ed029675d22f term=2 tally=1 -12:48.910 [INFO] TestAgent.server.raft: election won: tally=1 -12:48.910 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17070 [Leader]" -12:48.910 [INFO] TestAgent.server: cluster leadership acquired -12:48.910 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17070 -12:48.911 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:48.911 [INFO] TestAgent.server: New leader elected: payload=Node-e9517960-b0f9-ff9a-6597-ed029675d22f -12:48.912 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:48.912 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:48.912 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:48.912 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:48.912 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:48.912 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e9517960-b0f9-ff9a-6597-ed029675d22f -12:48.913 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e9517960-b0f9-ff9a-6597-ed029675d22f -12:48.913 [INFO] TestAgent.server: federation state anti-entropy synced -12:49.047 [DEBUG] TestAgent: added local registration for service: service=mesh-gateway -12:49.047 [INFO] TestAgent: Requesting shutdown -12:49.048 [INFO] TestAgent.server: shutting down server -12:49.048 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:49.048 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:49.048 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:49.048 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:49.048 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:49.048 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:49.048 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:49.048 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:49.048 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:49.048 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:49.048 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:49.048 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:49.048Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:49.048Z [INFO] agent.router.manager: shutting down -12:49.048 [INFO] TestAgent: consul server down -12:49.048 [INFO] TestAgent: shutdown complete -12:49.048 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17065 network=tcp -12:49.048 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17065 network=udp -12:49.048 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17066 network=tcp -12:49.549 [INFO] TestAgent: Waiting for endpoints to shut down -12:49.549 [INFO] TestAgent: Endpoints down ---- PASS: TestServiceManager_RegisterMeshGateway (0.73s) +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:01.547Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:01.561Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:01.563 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ba2ab95a-1fa0-9cbd-c5bc-ee9eacc06155 Address:127.0.0.1:24217}]" +49:01.563 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24217 [Follower]" leader= +49:01.563 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ba2ab95a-1fa0-9cbd-c5bc-ee9eacc06155.dc1 127.0.0.1 +49:01.564 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ba2ab95a-1fa0-9cbd-c5bc-ee9eacc06155 127.0.0.1 +2022-12-04T05:49:01.564Z [INFO] agent.router: Initializing LAN area manager +49:01.564 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24212 network=udp +49:01.564 [INFO] TestAgent.server: Adding LAN server: server="Node-ba2ab95a-1fa0-9cbd-c5bc-ee9eacc06155 (Addr: tcp/127.0.0.1:24217) (DC: dc1)" +49:01.564 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ba2ab95a-1fa0-9cbd-c5bc-ee9eacc06155.dc1 area=wan +49:01.564 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24212 network=tcp +49:01.565 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24213 network=tcp +49:01.565 [INFO] TestAgent: started state syncer +49:01.565 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24218 network=tcp +49:01.633 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:01.633 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24217 [Candidate]" term=2 +49:01.633 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:01.633 [DEBUG] TestAgent.server.raft: vote granted: from=ba2ab95a-1fa0-9cbd-c5bc-ee9eacc06155 term=2 tally=1 +49:01.633 [INFO] TestAgent.server.raft: election won: tally=1 +49:01.633 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24217 [Leader]" +49:01.633 [INFO] TestAgent.server: cluster leadership acquired +49:01.633 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24217 +49:01.633 [INFO] TestAgent.server: New leader elected: payload=Node-ba2ab95a-1fa0-9cbd-c5bc-ee9eacc06155 +49:01.634 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:01.635 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:01.635 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:01.635 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:01.635 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:01.635 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:01.635 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba2ab95a-1fa0-9cbd-c5bc-ee9eacc06155 +49:01.635 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ba2ab95a-1fa0-9cbd-c5bc-ee9eacc06155 +49:01.636 [INFO] TestAgent.server: federation state anti-entropy synced +49:01.770 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:01.770 [INFO] TestAgent: Synced node info +49:01.928 [DEBUG] TestAgent: added local registration for service: service=mesh-gateway +49:01.928 [INFO] TestAgent: Requesting shutdown +49:01.928 [INFO] TestAgent.server: shutting down server +49:01.928 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:01.928 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:01.928 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:01.928 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:01.928 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:01.929 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:01.929 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:01.929 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:01.929 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:01.929 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:01.929 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:01.929Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:01.929Z [INFO] agent.router.manager: shutting down +49:01.929 [INFO] TestAgent: consul server down +49:01.929 [INFO] TestAgent: shutdown complete +49:01.929 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24212 network=tcp +49:01.929 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24212 network=udp +49:01.929 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24213 network=tcp +49:02.429 [INFO] TestAgent: Waiting for endpoints to shut down +49:02.429 [INFO] TestAgent: Endpoints down +--- PASS: TestServiceManager_RegisterMeshGateway (0.91s) === RUN TestServiceManager_RegisterTerminatingGateway [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:49.575Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:49.618Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:49.620 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4cca3667-573c-e443-5436-ca8ba9572f87 Address:127.0.0.1:17077}]" -12:49.620 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4cca3667-573c-e443-5436-ca8ba9572f87.dc1 127.0.0.1 -12:49.621 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4cca3667-573c-e443-5436-ca8ba9572f87 127.0.0.1 -2021-10-31T23:12:49.621Z [INFO] agent.router: Initializing LAN area manager -12:49.621 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17072 network=udp -12:49.621 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17077 [Follower]" leader= -12:49.621 [INFO] TestAgent.server: Adding LAN server: server="Node-4cca3667-573c-e443-5436-ca8ba9572f87 (Addr: tcp/127.0.0.1:17077) (DC: dc1)" -12:49.621 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4cca3667-573c-e443-5436-ca8ba9572f87.dc1 area=wan -12:49.621 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17072 network=tcp -12:49.630 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17073 network=tcp -12:49.630 [INFO] TestAgent: started state syncer -12:49.631 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17078 network=tcp -12:49.673 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:49.673 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17077 [Candidate]" term=2 -12:49.674 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:49.674 [DEBUG] TestAgent.server.raft: vote granted: from=4cca3667-573c-e443-5436-ca8ba9572f87 term=2 tally=1 -12:49.674 [INFO] TestAgent.server.raft: election won: tally=1 -12:49.674 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17077 [Leader]" -12:49.674 [INFO] TestAgent.server: cluster leadership acquired -12:49.674 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17077 -12:49.675 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:49.676 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:49.676 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:49.676 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:49.676 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:49.676 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:49.676 [INFO] TestAgent.server: New leader elected: payload=Node-4cca3667-573c-e443-5436-ca8ba9572f87 -12:49.676 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4cca3667-573c-e443-5436-ca8ba9572f87 -12:49.676 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4cca3667-573c-e443-5436-ca8ba9572f87 -12:49.677 [INFO] TestAgent.server: federation state anti-entropy synced -12:49.915 [DEBUG] TestAgent: added local registration for service: service=terminating-gateway -12:49.915 [INFO] TestAgent: Requesting shutdown -12:49.915 [INFO] TestAgent.server: shutting down server -12:49.915 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:49.915 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:49.915 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:49.915 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:49.915 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:49.916 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:49.916 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:49.916 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:49.916 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:49.916 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:49.916 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:49.916 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:49.916Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:49.916Z [INFO] agent.router.manager: shutting down -12:49.916 [INFO] TestAgent: consul server down -12:49.916 [INFO] TestAgent: shutdown complete -12:49.916 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17072 network=tcp -12:49.916 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17072 network=udp -12:49.916 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17073 network=tcp -12:50.418 [INFO] TestAgent: Waiting for endpoints to shut down -12:50.418 [INFO] TestAgent: Endpoints down ---- PASS: TestServiceManager_RegisterTerminatingGateway (0.87s) +2022-12-04T05:49:02.440Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:02.456Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:02.458 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9a73b15f-8fef-58fb-bb93-2787e47a05ea Address:127.0.0.1:24224}]" +49:02.458 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9a73b15f-8fef-58fb-bb93-2787e47a05ea.dc1 127.0.0.1 +49:02.458 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9a73b15f-8fef-58fb-bb93-2787e47a05ea 127.0.0.1 +2022-12-04T05:49:02.459Z [INFO] agent.router: Initializing LAN area manager +49:02.459 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24219 network=udp +49:02.459 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24224 [Follower]" leader= +49:02.459 [INFO] TestAgent.server: Adding LAN server: server="Node-9a73b15f-8fef-58fb-bb93-2787e47a05ea (Addr: tcp/127.0.0.1:24224) (DC: dc1)" +49:02.459 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9a73b15f-8fef-58fb-bb93-2787e47a05ea.dc1 area=wan +49:02.459 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24219 network=tcp +49:02.460 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24220 network=tcp +49:02.460 [INFO] TestAgent: started state syncer +49:02.460 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24225 network=tcp +49:02.518 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:02.518 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24224 [Candidate]" term=2 +49:02.518 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:02.518 [DEBUG] TestAgent.server.raft: vote granted: from=9a73b15f-8fef-58fb-bb93-2787e47a05ea term=2 tally=1 +49:02.518 [INFO] TestAgent.server.raft: election won: tally=1 +49:02.518 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24224 [Leader]" +49:02.518 [INFO] TestAgent.server: cluster leadership acquired +49:02.519 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24224 +49:02.519 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:02.520 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:02.520 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:02.520 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:02.520 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:02.520 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:02.520 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9a73b15f-8fef-58fb-bb93-2787e47a05ea +49:02.520 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9a73b15f-8fef-58fb-bb93-2787e47a05ea +49:02.521 [INFO] TestAgent.server: New leader elected: payload=Node-9a73b15f-8fef-58fb-bb93-2787e47a05ea +49:02.521 [INFO] TestAgent.server: federation state anti-entropy synced +49:02.581 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:02.581 [INFO] TestAgent: Synced node info +49:02.581 [DEBUG] TestAgent: Node info in sync +49:02.638 [DEBUG] TestAgent: added local registration for service: service=terminating-gateway +49:02.638 [INFO] TestAgent: Requesting shutdown +49:02.638 [INFO] TestAgent.server: shutting down server +49:02.638 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:02.638 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:02.638 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:02.638 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:02.638 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:02.638 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:02.638 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:02.639 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:02.639 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:02.639 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:02.639 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:02.639Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:02.639Z [INFO] agent.router.manager: shutting down +49:02.639 [INFO] TestAgent: consul server down +49:02.639 [INFO] TestAgent: shutdown complete +49:02.639 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24219 network=tcp +49:02.639 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24219 network=udp +49:02.639 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24220 network=tcp +49:03.140 [INFO] TestAgent: Waiting for endpoints to shut down +49:03.140 [INFO] TestAgent: Endpoints down +--- PASS: TestServiceManager_RegisterTerminatingGateway (0.71s) === RUN TestServiceManager_PersistService_API service_manager_test.go:264: DM-disabled --- SKIP: TestServiceManager_PersistService_API (0.00s) @@ -19109,63 +19159,65 @@ [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:50.471Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:50.504Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:50.519 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c3732b27-cb95-f827-1b58-466bcdf61226 Address:127.0.0.1:17084}]" -12:50.520 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c3732b27-cb95-f827-1b58-466bcdf61226.dc1 127.0.0.1 -12:50.520 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c3732b27-cb95-f827-1b58-466bcdf61226 127.0.0.1 -2021-10-31T23:12:50.520Z [INFO] agent.router: Initializing LAN area manager -12:50.521 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17079 network=udp -12:50.521 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17084 [Follower]" leader= -12:50.521 [INFO] TestAgent.server: Adding LAN server: server="Node-c3732b27-cb95-f827-1b58-466bcdf61226 (Addr: tcp/127.0.0.1:17084) (DC: dc1)" -12:50.521 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c3732b27-cb95-f827-1b58-466bcdf61226.dc1 area=wan -12:50.521 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17079 network=tcp -12:50.522 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17080 network=tcp -12:50.522 [INFO] TestAgent: started state syncer -12:50.522 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17085 network=tcp -12:50.566 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:50.566 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17084 [Candidate]" term=2 -12:50.566 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:50.566 [DEBUG] TestAgent.server.raft: vote granted: from=c3732b27-cb95-f827-1b58-466bcdf61226 term=2 tally=1 -12:50.566 [INFO] TestAgent.server.raft: election won: tally=1 -12:50.566 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17084 [Leader]" -12:50.566 [INFO] TestAgent.server: cluster leadership acquired -12:50.566 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17084 -12:50.567 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:50.569 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:50.569 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:50.569 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:50.569 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:50.569 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:50.569 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c3732b27-cb95-f827-1b58-466bcdf61226 -12:50.569 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c3732b27-cb95-f827-1b58-466bcdf61226 -12:50.569 [INFO] TestAgent.server: New leader elected: payload=Node-c3732b27-cb95-f827-1b58-466bcdf61226 -12:50.570 [INFO] TestAgent.server: federation state anti-entropy synced -12:50.770 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:50.770 [INFO] TestAgent: Synced node info -12:50.847 [INFO] TestAgent: Requesting shutdown -12:50.847 [INFO] TestAgent.server: shutting down server -12:50.847 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:50.847 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:50.847 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:50.847 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:50.847 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:50.847 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:50.847 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:50.847 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:50.847 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:50.847 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:50.847 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:50.847Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:50.847Z [INFO] agent.router.manager: shutting down -12:50.847 [INFO] TestAgent: consul server down -12:50.847 [INFO] TestAgent: shutdown complete -12:50.847 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17079 network=tcp -12:50.848 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17079 network=udp -12:50.848 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17080 network=tcp -12:51.349 [INFO] TestAgent: Waiting for endpoints to shut down -12:51.349 [INFO] TestAgent: Endpoints down ---- PASS: TestServiceManager_Disabled (0.93s) +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:03.163Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:03.179Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:03.181 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2e05734c-adb0-97ee-16bf-55a49e0f03cc Address:127.0.0.1:24231}]" +49:03.181 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2e05734c-adb0-97ee-16bf-55a49e0f03cc.dc1 127.0.0.1 +49:03.182 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2e05734c-adb0-97ee-16bf-55a49e0f03cc 127.0.0.1 +2022-12-04T05:49:03.182Z [INFO] agent.router: Initializing LAN area manager +49:03.182 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24226 network=udp +49:03.182 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24231 [Follower]" leader= +49:03.182 [INFO] TestAgent.server: Adding LAN server: server="Node-2e05734c-adb0-97ee-16bf-55a49e0f03cc (Addr: tcp/127.0.0.1:24231) (DC: dc1)" +49:03.182 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2e05734c-adb0-97ee-16bf-55a49e0f03cc.dc1 area=wan +49:03.182 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24226 network=tcp +49:03.183 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24227 network=tcp +49:03.183 [INFO] TestAgent: started state syncer +49:03.183 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24232 network=tcp +49:03.224 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:03.224 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24231 [Candidate]" term=2 +49:03.224 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:03.224 [DEBUG] TestAgent.server.raft: vote granted: from=2e05734c-adb0-97ee-16bf-55a49e0f03cc term=2 tally=1 +49:03.224 [INFO] TestAgent.server.raft: election won: tally=1 +49:03.224 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24231 [Leader]" +49:03.224 [INFO] TestAgent.server: cluster leadership acquired +49:03.224 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24231 +49:03.225 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:03.226 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:03.226 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:03.226 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:03.226 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:03.226 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:03.227 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2e05734c-adb0-97ee-16bf-55a49e0f03cc +49:03.227 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2e05734c-adb0-97ee-16bf-55a49e0f03cc +49:03.227 [INFO] TestAgent.server: New leader elected: payload=Node-2e05734c-adb0-97ee-16bf-55a49e0f03cc +49:03.227 [INFO] TestAgent.server: federation state anti-entropy synced +49:03.428 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:03.428 [INFO] TestAgent: Synced node info +49:03.428 [DEBUG] TestAgent: Node info in sync +49:03.601 [INFO] TestAgent: Requesting shutdown +49:03.601 [INFO] TestAgent.server: shutting down server +49:03.601 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:03.601 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:03.601 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:03.601 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:03.601 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:03.601 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:03.601 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:03.601 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:03.601 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:03.601 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:03.601 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:03.601Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:03.601Z [INFO] agent.router.manager: shutting down +49:03.601 [INFO] TestAgent: consul server down +49:03.601 [INFO] TestAgent: shutdown complete +49:03.601 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24226 network=tcp +49:03.601 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24226 network=udp +49:03.601 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24227 network=tcp +49:04.102 [INFO] TestAgent: Waiting for endpoints to shut down +49:04.102 [INFO] TestAgent: Endpoints down +--- PASS: TestServiceManager_Disabled (0.96s) === RUN TestSessionCreate === PAUSE TestSessionCreate === RUN TestSessionCreate_NodeChecks @@ -19190,125 +19242,125 @@ [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:51.392Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:51.416Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:51.422 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b02d1f05-416b-4835-5d9e-45005a26f487 Address:127.0.0.1:17091}]" -12:51.423 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b02d1f05-416b-4835-5d9e-45005a26f487.dc1 127.0.0.1 -12:51.423 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b02d1f05-416b-4835-5d9e-45005a26f487 127.0.0.1 -2021-10-31T23:12:51.423Z [INFO] agent.router: Initializing LAN area manager -12:51.424 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17086 network=udp -12:51.424 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17091 [Follower]" leader= -12:51.424 [INFO] TestAgent.server: Adding LAN server: server="Node-b02d1f05-416b-4835-5d9e-45005a26f487 (Addr: tcp/127.0.0.1:17091) (DC: dc1)" -12:51.424 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b02d1f05-416b-4835-5d9e-45005a26f487.dc1 area=wan -12:51.424 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17086 network=tcp -12:51.425 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17087 network=tcp -12:51.425 [INFO] TestAgent: started state syncer -12:51.425 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17092 network=tcp -12:51.480 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:51.480 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17091 [Candidate]" term=2 -12:51.480 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:51.480 [DEBUG] TestAgent.server.raft: vote granted: from=b02d1f05-416b-4835-5d9e-45005a26f487 term=2 tally=1 -12:51.480 [INFO] TestAgent.server.raft: election won: tally=1 -12:51.480 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17091 [Leader]" -12:51.480 [INFO] TestAgent.server: cluster leadership acquired -12:51.480 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17091 -12:51.481 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:51.481 [INFO] TestAgent.server: New leader elected: payload=Node-b02d1f05-416b-4835-5d9e-45005a26f487 -12:51.483 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:51.483 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:51.483 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:51.483 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:51.483 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:51.483 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b02d1f05-416b-4835-5d9e-45005a26f487 -12:51.483 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b02d1f05-416b-4835-5d9e-45005a26f487 -12:51.484 [INFO] TestAgent.server: federation state anti-entropy synced -12:51.586 [INFO] TestAgent: Requesting shutdown -12:51.586 [INFO] TestAgent.server: shutting down server -12:51.586 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:51.586 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:51.586 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:51.586 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:51.586 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:51.586 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:51.586 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:51.586 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:51.586 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:51.586 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:51.586 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:51.586 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:51.586Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:51.586Z [INFO] agent.router.manager: shutting down -12:51.586 [INFO] TestAgent: consul server down -12:51.586 [INFO] TestAgent: shutdown complete -12:51.586 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17086 network=tcp -12:51.586 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17086 network=udp -12:51.586 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17087 network=tcp -12:52.090 [INFO] TestAgent: Waiting for endpoints to shut down -12:52.090 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:04.122Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:04.134Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:04.136 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:69e4770a-f744-9776-ac1c-40dd07c6d95d Address:127.0.0.1:24238}]" +49:04.136 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24238 [Follower]" leader= +49:04.136 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-69e4770a-f744-9776-ac1c-40dd07c6d95d.dc1 127.0.0.1 +49:04.137 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-69e4770a-f744-9776-ac1c-40dd07c6d95d 127.0.0.1 +2022-12-04T05:49:04.137Z [INFO] agent.router: Initializing LAN area manager +49:04.137 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24233 network=udp +49:04.137 [INFO] TestAgent.server: Adding LAN server: server="Node-69e4770a-f744-9776-ac1c-40dd07c6d95d (Addr: tcp/127.0.0.1:24238) (DC: dc1)" +49:04.137 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-69e4770a-f744-9776-ac1c-40dd07c6d95d.dc1 area=wan +49:04.137 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24233 network=tcp +49:04.138 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24234 network=tcp +49:04.138 [INFO] TestAgent: started state syncer +49:04.138 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24239 network=tcp +49:04.180 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:04.180 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24238 [Candidate]" term=2 +49:04.180 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:04.180 [DEBUG] TestAgent.server.raft: vote granted: from=69e4770a-f744-9776-ac1c-40dd07c6d95d term=2 tally=1 +49:04.180 [INFO] TestAgent.server.raft: election won: tally=1 +49:04.180 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24238 [Leader]" +49:04.180 [INFO] TestAgent.server: cluster leadership acquired +49:04.180 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24238 +49:04.181 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:04.183 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:04.183 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:04.183 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:04.183 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:04.183 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:04.183 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-69e4770a-f744-9776-ac1c-40dd07c6d95d +49:04.183 [INFO] TestAgent.server: member joined, marking health alive: member=Node-69e4770a-f744-9776-ac1c-40dd07c6d95d +49:04.183 [INFO] TestAgent.server: New leader elected: payload=Node-69e4770a-f744-9776-ac1c-40dd07c6d95d +49:04.184 [INFO] TestAgent.server: federation state anti-entropy synced +49:04.238 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:04.239 [INFO] TestAgent: Synced node info +49:04.239 [DEBUG] TestAgent: Node info in sync +49:04.434 [INFO] TestAgent: Requesting shutdown +49:04.434 [INFO] TestAgent.server: shutting down server +49:04.434 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:04.434 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:04.434 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:04.434 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:04.434 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:04.434 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:04.434 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:04.434 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:04.434 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:04.434 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:04.434 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:04.434Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:04.434Z [INFO] agent.router.manager: shutting down +49:04.434 [INFO] TestAgent: consul server down +49:04.434 [INFO] TestAgent: shutdown complete +49:04.434 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24233 network=tcp +49:04.434 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24233 network=udp +49:04.434 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24234 network=tcp +49:04.935 [INFO] TestAgent: Waiting for endpoints to shut down +49:04.935 [INFO] TestAgent: Endpoints down === RUN TestSessionList/#01 [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:52.111Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:52.125Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:52.126 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:357614b8-4343-b686-3a52-050fc81a0769 Address:127.0.0.1:17098}]" -12:52.126 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-357614b8-4343-b686-3a52-050fc81a0769.dc1 127.0.0.1 -12:52.127 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-357614b8-4343-b686-3a52-050fc81a0769 127.0.0.1 -2021-10-31T23:12:52.127Z [INFO] agent.router: Initializing LAN area manager -12:52.127 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17098 [Follower]" leader= -12:52.127 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17093 network=udp -12:52.127 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-357614b8-4343-b686-3a52-050fc81a0769.dc1 area=wan -12:52.127 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17093 network=tcp -12:52.127 [INFO] TestAgent.server: Adding LAN server: server="Node-357614b8-4343-b686-3a52-050fc81a0769 (Addr: tcp/127.0.0.1:17098) (DC: dc1)" -12:52.128 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17094 network=tcp -12:52.128 [INFO] TestAgent: started state syncer -12:52.128 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17099 network=tcp -12:52.193 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:52.193 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17098 [Candidate]" term=2 -12:52.194 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:52.194 [DEBUG] TestAgent.server.raft: vote granted: from=357614b8-4343-b686-3a52-050fc81a0769 term=2 tally=1 -12:52.194 [INFO] TestAgent.server.raft: election won: tally=1 -12:52.194 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17098 [Leader]" -12:52.194 [INFO] TestAgent.server: cluster leadership acquired -12:52.194 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17098 -12:52.195 [INFO] TestAgent.server: New leader elected: payload=Node-357614b8-4343-b686-3a52-050fc81a0769 -12:52.195 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:52.196 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:52.196 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:52.196 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:52.196 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:52.196 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:52.196 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-357614b8-4343-b686-3a52-050fc81a0769 -12:52.196 [INFO] TestAgent.server: member joined, marking health alive: member=Node-357614b8-4343-b686-3a52-050fc81a0769 -12:52.197 [INFO] TestAgent.server: federation state anti-entropy synced -12:52.270 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:52.270 [INFO] TestAgent: Synced node info -12:52.270 [DEBUG] TestAgent: Node info in sync -12:52.313 [INFO] TestAgent: Requesting shutdown -12:52.313 [INFO] TestAgent.server: shutting down server -12:52.313 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:52.313 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:52.313 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:52.313 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:52.313 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:52.313 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:52.313 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:52.313 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:52.313 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:52.313 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:52.313 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:52.313Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:52.313Z [INFO] agent.router.manager: shutting down -12:52.327 [INFO] TestAgent: consul server down -12:52.327 [INFO] TestAgent: shutdown complete -12:52.327 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17093 network=tcp -12:52.327 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17093 network=udp -12:52.327 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17094 network=tcp -12:52.829 [INFO] TestAgent: Waiting for endpoints to shut down -12:52.830 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:04.972Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:04.983Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:04.985 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d5245139-e5e2-8680-5401-ad19c578df79 Address:127.0.0.1:24245}]" +49:04.985 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d5245139-e5e2-8680-5401-ad19c578df79.dc1 127.0.0.1 +49:04.986 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d5245139-e5e2-8680-5401-ad19c578df79 127.0.0.1 +2022-12-04T05:49:04.986Z [INFO] agent.router: Initializing LAN area manager +49:04.986 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24245 [Follower]" leader= +49:04.986 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24240 network=udp +49:04.986 [INFO] TestAgent.server: Adding LAN server: server="Node-d5245139-e5e2-8680-5401-ad19c578df79 (Addr: tcp/127.0.0.1:24245) (DC: dc1)" +49:04.986 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d5245139-e5e2-8680-5401-ad19c578df79.dc1 area=wan +49:04.987 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24240 network=tcp +49:04.987 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24241 network=tcp +49:04.987 [INFO] TestAgent: started state syncer +49:04.987 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24246 network=tcp +49:05.024 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:05.024 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24245 [Candidate]" term=2 +49:05.024 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:05.024 [DEBUG] TestAgent.server.raft: vote granted: from=d5245139-e5e2-8680-5401-ad19c578df79 term=2 tally=1 +49:05.024 [INFO] TestAgent.server.raft: election won: tally=1 +49:05.024 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24245 [Leader]" +49:05.024 [INFO] TestAgent.server: cluster leadership acquired +49:05.024 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24245 +49:05.025 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:05.026 [INFO] TestAgent.server: New leader elected: payload=Node-d5245139-e5e2-8680-5401-ad19c578df79 +49:05.027 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:05.027 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:05.027 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:05.027 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:05.027 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:05.027 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d5245139-e5e2-8680-5401-ad19c578df79 +49:05.027 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d5245139-e5e2-8680-5401-ad19c578df79 +49:05.027 [INFO] TestAgent.server: federation state anti-entropy synced +49:05.082 [INFO] TestAgent: Requesting shutdown +49:05.082 [INFO] TestAgent.server: shutting down server +49:05.082 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:05.082 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:05.082 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:05.082 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:05.082 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:05.082 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:05.082 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:05.082 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:05.082 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:05.082 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:05.082 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:05.083 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:05.083Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:05.083Z [INFO] agent.router.manager: shutting down +49:05.083 [INFO] TestAgent: consul server down +49:05.083 [INFO] TestAgent: shutdown complete +49:05.083 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24240 network=tcp +49:05.083 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24240 network=udp +49:05.083 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24241 network=tcp +49:05.584 [INFO] TestAgent: Waiting for endpoints to shut down +49:05.584 [INFO] TestAgent: Endpoints down --- PASS: TestSessionList (1.48s) - --- PASS: TestSessionList/#00 (0.74s) - --- PASS: TestSessionList/#01 (0.74s) + --- PASS: TestSessionList/#00 (0.84s) + --- PASS: TestSessionList/#01 (0.64s) === RUN TestSessionsForNode session_endpoint_test.go:677: DM-skipped --- SKIP: TestSessionsForNode (0.00s) @@ -19318,552 +19370,559 @@ === RUN TestAgent_sidecarServiceFromNodeService/no_sidecar [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:52.865Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:52.930Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:52.932 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8bbb770d-9b3a-502c-20d5-8f9dd5ab9ad8 Address:127.0.0.1:17105}]" -12:52.933 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-8bbb770d-9b3a-502c-20d5-8f9dd5ab9ad8.dc1 127.0.0.1 -12:52.933 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:17105 [Follower]" leader= -12:52.938 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-8bbb770d-9b3a-502c-20d5-8f9dd5ab9ad8 127.0.0.1 -2021-10-31T23:12:52.938Z [INFO] agent.router: Initializing LAN area manager -12:52.938 [INFO] jones: Started DNS server: address=127.0.0.1:17100 network=udp -12:52.938 [INFO] jones.server: Adding LAN server: server="Node-8bbb770d-9b3a-502c-20d5-8f9dd5ab9ad8 (Addr: tcp/127.0.0.1:17105) (DC: dc1)" -12:52.938 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-8bbb770d-9b3a-502c-20d5-8f9dd5ab9ad8.dc1 area=wan -12:52.938 [INFO] jones: Started DNS server: address=127.0.0.1:17100 network=tcp -12:52.939 [INFO] jones: Started HTTP server: address=127.0.0.1:17101 network=tcp -12:52.939 [INFO] jones: started state syncer -12:52.939 [INFO] jones: Started gRPC server: address=127.0.0.1:17106 network=tcp -12:52.985 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -12:52.985 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:17105 [Candidate]" term=2 -12:52.986 [DEBUG] jones.server.raft: votes: needed=1 -12:52.986 [DEBUG] jones.server.raft: vote granted: from=8bbb770d-9b3a-502c-20d5-8f9dd5ab9ad8 term=2 tally=1 -12:52.986 [INFO] jones.server.raft: election won: tally=1 -12:52.986 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:17105 [Leader]" -12:52.986 [INFO] jones.server: cluster leadership acquired -12:52.986 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17105 -12:52.987 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:52.988 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -12:52.988 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -12:52.988 [INFO] jones.leader: started routine: routine="federation state pruning" -12:52.988 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -12:52.988 [INFO] jones.leader: started routine: routine="CA root pruning" -12:52.988 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-8bbb770d-9b3a-502c-20d5-8f9dd5ab9ad8 -12:52.988 [INFO] jones.server: member joined, marking health alive: member=Node-8bbb770d-9b3a-502c-20d5-8f9dd5ab9ad8 -12:52.989 [INFO] jones.server: New leader elected: payload=Node-8bbb770d-9b3a-502c-20d5-8f9dd5ab9ad8 -12:52.989 [INFO] jones.server: federation state anti-entropy synced -12:53.120 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth -12:53.121 [INFO] jones: Synced node info -12:53.121 [DEBUG] jones: Node info in sync -12:53.350 [INFO] jones: Requesting shutdown -12:53.350 [INFO] jones.server: shutting down server -12:53.350 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -12:53.350 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -12:53.350 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -12:53.350 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -12:53.350 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -12:53.350 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -12:53.350 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -12:53.350 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -12:53.350 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -12:53.350 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -12:53.350 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:53.350Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:53.350Z [INFO] agent.router.manager: shutting down -12:53.350 [INFO] jones: consul server down -12:53.350 [INFO] jones: shutdown complete -12:53.350 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:17100 network=tcp -12:53.350 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:17100 network=udp -12:53.350 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:17101 network=tcp -12:53.853 [INFO] jones: Waiting for endpoints to shut down -12:53.853 [INFO] jones: Endpoints down +2022-12-04T05:49:05.597Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:05.611Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:05.615 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:24db99a2-747e-83a2-c6f3-a9a02bbed21c Address:127.0.0.1:24252}]" +49:05.616 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:24252 [Follower]" leader= +49:05.616 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-24db99a2-747e-83a2-c6f3-a9a02bbed21c.dc1 127.0.0.1 +49:05.616 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-24db99a2-747e-83a2-c6f3-a9a02bbed21c 127.0.0.1 +2022-12-04T05:49:05.616Z [INFO] agent.router: Initializing LAN area manager +49:05.617 [INFO] jones: Started DNS server: address=127.0.0.1:24247 network=udp +49:05.617 [INFO] jones.server: Adding LAN server: server="Node-24db99a2-747e-83a2-c6f3-a9a02bbed21c (Addr: tcp/127.0.0.1:24252) (DC: dc1)" +49:05.617 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-24db99a2-747e-83a2-c6f3-a9a02bbed21c.dc1 area=wan +49:05.617 [INFO] jones: Started DNS server: address=127.0.0.1:24247 network=tcp +49:05.618 [INFO] jones: Started HTTP server: address=127.0.0.1:24248 network=tcp +49:05.618 [INFO] jones: started state syncer +49:05.618 [INFO] jones: Started gRPC server: address=127.0.0.1:24253 network=tcp +49:05.686 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +49:05.686 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:24252 [Candidate]" term=2 +49:05.686 [DEBUG] jones.server.raft: votes: needed=1 +49:05.686 [DEBUG] jones.server.raft: vote granted: from=24db99a2-747e-83a2-c6f3-a9a02bbed21c term=2 tally=1 +49:05.686 [INFO] jones.server.raft: election won: tally=1 +49:05.686 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:24252 [Leader]" +49:05.686 [INFO] jones.server: cluster leadership acquired +49:05.687 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24252 +49:05.687 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:05.687 [INFO] jones.server: New leader elected: payload=Node-24db99a2-747e-83a2-c6f3-a9a02bbed21c +49:05.689 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +49:05.689 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +49:05.689 [INFO] jones.leader: started routine: routine="federation state pruning" +49:05.689 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +49:05.689 [INFO] jones.leader: started routine: routine="CA root pruning" +49:05.689 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-24db99a2-747e-83a2-c6f3-a9a02bbed21c +49:05.689 [INFO] jones.server: member joined, marking health alive: member=Node-24db99a2-747e-83a2-c6f3-a9a02bbed21c +49:05.690 [INFO] jones.server: federation state anti-entropy synced +49:05.752 [INFO] jones: Requesting shutdown +49:05.752 [INFO] jones.server: shutting down server +49:05.752 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +49:05.752 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +49:05.752 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +49:05.752 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +49:05.752 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +49:05.752 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +49:05.752 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +49:05.752 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +49:05.752 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" +49:05.752 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +49:05.752 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +49:05.753 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:49:05.753Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:05.753Z [INFO] agent.router.manager: shutting down +49:05.753 [INFO] jones: consul server down +49:05.753 [INFO] jones: shutdown complete +49:05.753 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24247 network=tcp +49:05.753 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24247 network=udp +49:05.753 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:24248 network=tcp +49:06.253 [INFO] jones: Waiting for endpoints to shut down +49:06.253 [INFO] jones: Endpoints down === RUN TestAgent_sidecarServiceFromNodeService/all_the_defaults [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:53.919Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:53.961Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:53.963 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aec45322-cb1e-4686-56aa-8076e205733d Address:127.0.0.1:17112}]" -12:53.964 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-aec45322-cb1e-4686-56aa-8076e205733d.dc1 127.0.0.1 -12:53.965 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:17112 [Follower]" leader= -12:53.965 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-aec45322-cb1e-4686-56aa-8076e205733d 127.0.0.1 -2021-10-31T23:12:53.965Z [INFO] agent.router: Initializing LAN area manager -12:53.965 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-aec45322-cb1e-4686-56aa-8076e205733d.dc1 area=wan -12:53.965 [INFO] jones.server: Adding LAN server: server="Node-aec45322-cb1e-4686-56aa-8076e205733d (Addr: tcp/127.0.0.1:17112) (DC: dc1)" -12:53.965 [INFO] jones: Started DNS server: address=127.0.0.1:17107 network=udp -12:53.965 [INFO] jones: Started DNS server: address=127.0.0.1:17107 network=tcp -12:53.974 [INFO] jones: Started HTTP server: address=127.0.0.1:17108 network=tcp -12:53.974 [INFO] jones: started state syncer -12:53.974 [INFO] jones: Started gRPC server: address=127.0.0.1:17113 network=tcp -12:54.013 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -12:54.013 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:17112 [Candidate]" term=2 -12:54.014 [DEBUG] jones.server.raft: votes: needed=1 -12:54.014 [DEBUG] jones.server.raft: vote granted: from=aec45322-cb1e-4686-56aa-8076e205733d term=2 tally=1 -12:54.014 [INFO] jones.server.raft: election won: tally=1 -12:54.014 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:17112 [Leader]" -12:54.014 [INFO] jones.server: cluster leadership acquired -12:54.014 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17112 -12:54.015 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:54.016 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -12:54.016 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -12:54.016 [INFO] jones.leader: started routine: routine="federation state pruning" -12:54.016 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -12:54.016 [INFO] jones.leader: started routine: routine="CA root pruning" -12:54.016 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-aec45322-cb1e-4686-56aa-8076e205733d -12:54.016 [INFO] jones.server: member joined, marking health alive: member=Node-aec45322-cb1e-4686-56aa-8076e205733d -12:54.016 [INFO] jones.server: New leader elected: payload=Node-aec45322-cb1e-4686-56aa-8076e205733d -12:54.016 [INFO] jones.server: federation state anti-entropy synced -12:54.226 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth -12:54.226 [INFO] jones: Synced node info -12:54.226 [DEBUG] jones: Node info in sync -12:54.374 [INFO] jones: Requesting shutdown -12:54.374 [INFO] jones.server: shutting down server -12:54.374 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -12:54.374 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -12:54.374 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -12:54.374 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -12:54.374 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -12:54.374 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -12:54.374 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -12:54.374 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -12:54.374 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -12:54.374 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -12:54.374 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:54.374Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:54.374Z [INFO] agent.router.manager: shutting down -12:54.374 [INFO] jones: consul server down -12:54.374 [INFO] jones: shutdown complete -12:54.374 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:17107 network=tcp -12:54.374 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:17107 network=udp -12:54.374 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:17108 network=tcp -12:54.875 [INFO] jones: Waiting for endpoints to shut down -12:54.875 [INFO] jones: Endpoints down +2022-12-04T05:49:06.264Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:06.275Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:06.276 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:339b2ce9-fa14-8cd4-d03d-8d0056f3f01e Address:127.0.0.1:24259}]" +49:06.277 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:24259 [Follower]" leader= +49:06.277 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-339b2ce9-fa14-8cd4-d03d-8d0056f3f01e.dc1 127.0.0.1 +49:06.277 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-339b2ce9-fa14-8cd4-d03d-8d0056f3f01e 127.0.0.1 +2022-12-04T05:49:06.277Z [INFO] agent.router: Initializing LAN area manager +49:06.277 [INFO] jones.server: Adding LAN server: server="Node-339b2ce9-fa14-8cd4-d03d-8d0056f3f01e (Addr: tcp/127.0.0.1:24259) (DC: dc1)" +49:06.277 [INFO] jones: Started DNS server: address=127.0.0.1:24254 network=udp +49:06.277 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-339b2ce9-fa14-8cd4-d03d-8d0056f3f01e.dc1 area=wan +49:06.278 [INFO] jones: Started DNS server: address=127.0.0.1:24254 network=tcp +49:06.278 [INFO] jones: Started HTTP server: address=127.0.0.1:24255 network=tcp +49:06.278 [INFO] jones: started state syncer +49:06.278 [INFO] jones: Started gRPC server: address=127.0.0.1:24260 network=tcp +49:06.318 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +49:06.318 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:24259 [Candidate]" term=2 +49:06.318 [DEBUG] jones.server.raft: votes: needed=1 +49:06.318 [DEBUG] jones.server.raft: vote granted: from=339b2ce9-fa14-8cd4-d03d-8d0056f3f01e term=2 tally=1 +49:06.318 [INFO] jones.server.raft: election won: tally=1 +49:06.318 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:24259 [Leader]" +49:06.318 [INFO] jones.server: cluster leadership acquired +49:06.318 [INFO] jones.server: New leader elected: payload=Node-339b2ce9-fa14-8cd4-d03d-8d0056f3f01e +49:06.318 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24259 +49:06.319 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:06.320 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +49:06.320 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +49:06.320 [INFO] jones.leader: started routine: routine="federation state pruning" +49:06.320 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +49:06.320 [INFO] jones.leader: started routine: routine="CA root pruning" +49:06.320 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-339b2ce9-fa14-8cd4-d03d-8d0056f3f01e +49:06.320 [INFO] jones.server: member joined, marking health alive: member=Node-339b2ce9-fa14-8cd4-d03d-8d0056f3f01e +49:06.321 [INFO] jones.server: federation state anti-entropy synced +49:06.325 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth +49:06.325 [INFO] jones: Synced node info +49:06.340 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth +49:06.340 [DEBUG] jones: Node info in sync +49:06.340 [DEBUG] jones: Node info in sync +49:06.715 [INFO] jones: Requesting shutdown +49:06.715 [INFO] jones.server: shutting down server +49:06.715 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +49:06.715 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +49:06.715 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +49:06.715 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +49:06.715 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +49:06.715 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +49:06.715 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +49:06.715 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +49:06.715 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +49:06.715 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +49:06.715 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:49:06.715Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:06.715Z [INFO] agent.router.manager: shutting down +49:06.715 [INFO] jones: consul server down +49:06.715 [INFO] jones: shutdown complete +49:06.715 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24254 network=tcp +49:06.715 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24254 network=udp +49:06.715 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:24255 network=tcp +49:07.216 [INFO] jones: Waiting for endpoints to shut down +49:07.216 [INFO] jones: Endpoints down === RUN TestAgent_sidecarServiceFromNodeService/all_the_allowed_overrides [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:54.907Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:54.939Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:54.941 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c33a1390-bfa7-6e3e-bc65-15bfae06912d Address:127.0.0.1:17119}]" -12:54.941 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-c33a1390-bfa7-6e3e-bc65-15bfae06912d.dc1 127.0.0.1 -12:54.941 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:17119 [Follower]" leader= -12:54.946 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-c33a1390-bfa7-6e3e-bc65-15bfae06912d 127.0.0.1 -2021-10-31T23:12:54.946Z [INFO] agent.router: Initializing LAN area manager -12:54.947 [INFO] jones: Started DNS server: address=127.0.0.1:17114 network=udp -12:54.947 [INFO] jones.server: Adding LAN server: server="Node-c33a1390-bfa7-6e3e-bc65-15bfae06912d (Addr: tcp/127.0.0.1:17119) (DC: dc1)" -12:54.947 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-c33a1390-bfa7-6e3e-bc65-15bfae06912d.dc1 area=wan -12:54.947 [INFO] jones: Started DNS server: address=127.0.0.1:17114 network=tcp -12:54.947 [INFO] jones: Started HTTP server: address=127.0.0.1:17115 network=tcp -12:54.948 [INFO] jones: started state syncer -12:54.948 [INFO] jones: Started gRPC server: address=127.0.0.1:17120 network=tcp -12:54.985 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -12:54.985 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:17119 [Candidate]" term=2 -12:54.985 [DEBUG] jones.server.raft: votes: needed=1 -12:54.985 [DEBUG] jones.server.raft: vote granted: from=c33a1390-bfa7-6e3e-bc65-15bfae06912d term=2 tally=1 -12:54.985 [INFO] jones.server.raft: election won: tally=1 -12:54.985 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:17119 [Leader]" -12:54.985 [INFO] jones.server: cluster leadership acquired -12:54.986 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17119 -12:54.986 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:54.988 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -12:54.988 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -12:54.988 [INFO] jones.leader: started routine: routine="federation state pruning" -12:54.988 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -12:54.988 [INFO] jones.leader: started routine: routine="CA root pruning" -12:54.988 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-c33a1390-bfa7-6e3e-bc65-15bfae06912d -12:54.988 [INFO] jones.server: member joined, marking health alive: member=Node-c33a1390-bfa7-6e3e-bc65-15bfae06912d -12:54.988 [INFO] jones.server: New leader elected: payload=Node-c33a1390-bfa7-6e3e-bc65-15bfae06912d -12:54.989 [INFO] jones.server: federation state anti-entropy synced -12:55.170 [INFO] jones: Requesting shutdown -12:55.170 [INFO] jones.server: shutting down server -12:55.170 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -12:55.170 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -12:55.170 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -12:55.170 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -12:55.170 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -12:55.170 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -12:55.170 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -12:55.170 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" -12:55.170 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -12:55.170 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -12:55.170 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -12:55.170 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:55.170Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:55.170Z [INFO] agent.router.manager: shutting down -12:55.170 [INFO] jones: consul server down -12:55.170 [INFO] jones: shutdown complete -12:55.170 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:17114 network=tcp -12:55.170 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:17114 network=udp -12:55.170 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:17115 network=tcp -12:55.671 [INFO] jones: Waiting for endpoints to shut down -12:55.671 [INFO] jones: Endpoints down +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:07.238Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:07.252Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:07.254 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c0f4e397-5da7-b768-4d1a-2dd72ea9470f Address:127.0.0.1:24266}]" +49:07.254 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-c0f4e397-5da7-b768-4d1a-2dd72ea9470f.dc1 127.0.0.1 +49:07.255 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-c0f4e397-5da7-b768-4d1a-2dd72ea9470f 127.0.0.1 +2022-12-04T05:49:07.255Z [INFO] agent.router: Initializing LAN area manager +49:07.255 [INFO] jones: Started DNS server: address=127.0.0.1:24261 network=udp +49:07.255 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:24266 [Follower]" leader= +49:07.255 [INFO] jones.server: Adding LAN server: server="Node-c0f4e397-5da7-b768-4d1a-2dd72ea9470f (Addr: tcp/127.0.0.1:24266) (DC: dc1)" +49:07.255 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-c0f4e397-5da7-b768-4d1a-2dd72ea9470f.dc1 area=wan +49:07.256 [INFO] jones: Started DNS server: address=127.0.0.1:24261 network=tcp +49:07.257 [INFO] jones: Started HTTP server: address=127.0.0.1:24262 network=tcp +49:07.257 [INFO] jones: started state syncer +49:07.257 [INFO] jones: Started gRPC server: address=127.0.0.1:24267 network=tcp +49:07.316 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +49:07.316 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:24266 [Candidate]" term=2 +49:07.316 [DEBUG] jones.server.raft: votes: needed=1 +49:07.316 [DEBUG] jones.server.raft: vote granted: from=c0f4e397-5da7-b768-4d1a-2dd72ea9470f term=2 tally=1 +49:07.316 [INFO] jones.server.raft: election won: tally=1 +49:07.316 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:24266 [Leader]" +49:07.316 [INFO] jones.server: cluster leadership acquired +49:07.316 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24266 +49:07.328 [INFO] jones.server: New leader elected: payload=Node-c0f4e397-5da7-b768-4d1a-2dd72ea9470f +49:07.329 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:07.331 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +49:07.331 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +49:07.331 [INFO] jones.leader: started routine: routine="federation state pruning" +49:07.331 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +49:07.331 [INFO] jones.leader: started routine: routine="CA root pruning" +49:07.331 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-c0f4e397-5da7-b768-4d1a-2dd72ea9470f +49:07.332 [INFO] jones.server: member joined, marking health alive: member=Node-c0f4e397-5da7-b768-4d1a-2dd72ea9470f +49:07.333 [INFO] jones.server: federation state anti-entropy synced +49:07.566 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth +49:07.566 [INFO] jones: Synced node info +49:07.566 [DEBUG] jones: Node info in sync +49:07.592 [INFO] jones: Requesting shutdown +49:07.592 [INFO] jones.server: shutting down server +49:07.592 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +49:07.592 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +49:07.592 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +49:07.592 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +49:07.592 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +49:07.592 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +49:07.592 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +49:07.592 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +49:07.592 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +49:07.592 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +49:07.592 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:07.592Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:07.592Z [INFO] agent.router.manager: shutting down +49:07.592 [INFO] jones: consul server down +49:07.592 [INFO] jones: shutdown complete +49:07.593 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24261 network=tcp +49:07.593 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24261 network=udp +49:07.593 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:24262 network=tcp +49:08.093 [INFO] jones: Waiting for endpoints to shut down +49:08.093 [INFO] jones: Endpoints down === RUN TestAgent_sidecarServiceFromNodeService/no_auto_ports_available [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:55.699Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:55.774Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:55.776 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bef55d0c-c168-ca57-6e7e-ec69a8b2ea50 Address:127.0.0.1:17126}]" -12:55.776 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:17126 [Follower]" leader= -12:55.792 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-bef55d0c-c168-ca57-6e7e-ec69a8b2ea50.dc1 127.0.0.1 -12:55.793 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-bef55d0c-c168-ca57-6e7e-ec69a8b2ea50 127.0.0.1 -2021-10-31T23:12:55.793Z [INFO] agent.router: Initializing LAN area manager -12:55.793 [INFO] jones: Started DNS server: address=127.0.0.1:17121 network=udp -12:55.793 [INFO] jones.server: Adding LAN server: server="Node-bef55d0c-c168-ca57-6e7e-ec69a8b2ea50 (Addr: tcp/127.0.0.1:17126) (DC: dc1)" -12:55.793 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-bef55d0c-c168-ca57-6e7e-ec69a8b2ea50.dc1 area=wan -12:55.793 [INFO] jones: Started DNS server: address=127.0.0.1:17121 network=tcp -12:55.794 [INFO] jones: Started HTTP server: address=127.0.0.1:17122 network=tcp -12:55.794 [INFO] jones: started state syncer -12:55.794 [INFO] jones: Started gRPC server: address=127.0.0.1:17127 network=tcp -12:55.838 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -12:55.838 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:17126 [Candidate]" term=2 -12:55.838 [DEBUG] jones.server.raft: votes: needed=1 -12:55.838 [DEBUG] jones.server.raft: vote granted: from=bef55d0c-c168-ca57-6e7e-ec69a8b2ea50 term=2 tally=1 -12:55.838 [INFO] jones.server.raft: election won: tally=1 -12:55.838 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:17126 [Leader]" -12:55.838 [INFO] jones.server: cluster leadership acquired -12:55.838 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17126 -12:55.839 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:55.840 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -12:55.840 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -12:55.840 [INFO] jones.leader: started routine: routine="federation state pruning" -12:55.840 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -12:55.840 [INFO] jones.leader: started routine: routine="CA root pruning" -12:55.840 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-bef55d0c-c168-ca57-6e7e-ec69a8b2ea50 -12:55.840 [INFO] jones.server: member joined, marking health alive: member=Node-bef55d0c-c168-ca57-6e7e-ec69a8b2ea50 -12:55.841 [INFO] jones.server: New leader elected: payload=Node-bef55d0c-c168-ca57-6e7e-ec69a8b2ea50 -12:55.841 [INFO] jones.server: federation state anti-entropy synced -12:55.850 [INFO] jones: Requesting shutdown -12:55.850 [INFO] jones.server: shutting down server -12:55.850 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -12:55.850 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -12:55.850 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -12:55.850 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -12:55.850 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -12:55.851 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -12:55.851 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -12:55.851 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" -12:55.851 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -12:55.851 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -12:55.851 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -12:55.851 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:55.851Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:55.851Z [INFO] agent.router.manager: shutting down -12:55.851 [INFO] jones: consul server down -12:55.851 [INFO] jones: shutdown complete -12:55.851 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:17121 network=tcp -12:55.851 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:17121 network=udp -12:55.851 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:17122 network=tcp -12:56.354 [INFO] jones: Waiting for endpoints to shut down -12:56.354 [INFO] jones: Endpoints down +2022-12-04T05:49:08.108Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:08.122Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:08.124 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9dc87505-a4d3-e3b2-9885-ba2fc6904dec Address:127.0.0.1:24273}]" +49:08.124 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:24273 [Follower]" leader= +49:08.125 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-9dc87505-a4d3-e3b2-9885-ba2fc6904dec.dc1 127.0.0.1 +49:08.125 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-9dc87505-a4d3-e3b2-9885-ba2fc6904dec 127.0.0.1 +2022-12-04T05:49:08.125Z [INFO] agent.router: Initializing LAN area manager +49:08.126 [INFO] jones: Started DNS server: address=127.0.0.1:24268 network=udp +49:08.126 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-9dc87505-a4d3-e3b2-9885-ba2fc6904dec.dc1 area=wan +49:08.126 [INFO] jones.server: Adding LAN server: server="Node-9dc87505-a4d3-e3b2-9885-ba2fc6904dec (Addr: tcp/127.0.0.1:24273) (DC: dc1)" +49:08.126 [INFO] jones: Started DNS server: address=127.0.0.1:24268 network=tcp +49:08.126 [INFO] jones: Started HTTP server: address=127.0.0.1:24269 network=tcp +49:08.127 [INFO] jones: started state syncer +49:08.127 [INFO] jones: Started gRPC server: address=127.0.0.1:24274 network=tcp +49:08.180 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +49:08.180 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:24273 [Candidate]" term=2 +49:08.180 [DEBUG] jones.server.raft: votes: needed=1 +49:08.180 [DEBUG] jones.server.raft: vote granted: from=9dc87505-a4d3-e3b2-9885-ba2fc6904dec term=2 tally=1 +49:08.180 [INFO] jones.server.raft: election won: tally=1 +49:08.180 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:24273 [Leader]" +49:08.180 [INFO] jones.server: cluster leadership acquired +49:08.180 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24273 +49:08.181 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:08.182 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +49:08.182 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +49:08.182 [INFO] jones.leader: started routine: routine="federation state pruning" +49:08.182 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +49:08.182 [INFO] jones.leader: started routine: routine="CA root pruning" +49:08.182 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-9dc87505-a4d3-e3b2-9885-ba2fc6904dec +49:08.182 [INFO] jones.server: member joined, marking health alive: member=Node-9dc87505-a4d3-e3b2-9885-ba2fc6904dec +49:08.183 [INFO] jones.server: New leader elected: payload=Node-9dc87505-a4d3-e3b2-9885-ba2fc6904dec +49:08.183 [INFO] jones.server: federation state anti-entropy synced +49:08.195 [INFO] jones: Requesting shutdown +49:08.196 [INFO] jones.server: shutting down server +49:08.196 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +49:08.196 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +49:08.196 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +49:08.196 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +49:08.196 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +49:08.196 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" +49:08.196 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +49:08.196 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +49:08.196 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +49:08.196 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +49:08.196 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +49:08.196 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:08.196Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:08.196Z [INFO] agent.router.manager: shutting down +49:08.196 [INFO] jones: consul server down +49:08.196 [INFO] jones: shutdown complete +49:08.196 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24268 network=tcp +49:08.196 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24268 network=udp +49:08.196 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:24269 network=tcp +49:08.700 [INFO] jones: Waiting for endpoints to shut down +49:08.700 [INFO] jones: Endpoints down === RUN TestAgent_sidecarServiceFromNodeService/auto_ports_disabled [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:56.385Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:56.428Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:56.434 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4e3e4264-3b2e-36ca-bda4-c42a9e6bed92 Address:127.0.0.1:17133}]" -12:56.435 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-4e3e4264-3b2e-36ca-bda4-c42a9e6bed92.dc1 127.0.0.1 -12:56.435 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:17133 [Follower]" leader= -12:56.436 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-4e3e4264-3b2e-36ca-bda4-c42a9e6bed92 127.0.0.1 -2021-10-31T23:12:56.436Z [INFO] agent.router: Initializing LAN area manager -12:56.436 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-4e3e4264-3b2e-36ca-bda4-c42a9e6bed92.dc1 area=wan -12:56.436 [INFO] jones.server: Adding LAN server: server="Node-4e3e4264-3b2e-36ca-bda4-c42a9e6bed92 (Addr: tcp/127.0.0.1:17133) (DC: dc1)" -12:56.436 [INFO] jones: Started DNS server: address=127.0.0.1:17128 network=udp -12:56.436 [INFO] jones: Started DNS server: address=127.0.0.1:17128 network=tcp -12:56.437 [INFO] jones: Started HTTP server: address=127.0.0.1:17129 network=tcp -12:56.437 [INFO] jones: started state syncer -12:56.437 [INFO] jones: Started gRPC server: address=127.0.0.1:17134 network=tcp -12:56.500 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -12:56.501 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:17133 [Candidate]" term=2 -12:56.501 [DEBUG] jones.server.raft: votes: needed=1 -12:56.501 [DEBUG] jones.server.raft: vote granted: from=4e3e4264-3b2e-36ca-bda4-c42a9e6bed92 term=2 tally=1 -12:56.501 [INFO] jones.server.raft: election won: tally=1 -12:56.501 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:17133 [Leader]" -12:56.501 [INFO] jones.server: cluster leadership acquired -12:56.501 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17133 -12:56.502 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:56.504 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -12:56.504 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -12:56.504 [INFO] jones.leader: started routine: routine="federation state pruning" -12:56.504 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -12:56.504 [INFO] jones.leader: started routine: routine="CA root pruning" -12:56.504 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-4e3e4264-3b2e-36ca-bda4-c42a9e6bed92 -12:56.504 [INFO] jones.server: member joined, marking health alive: member=Node-4e3e4264-3b2e-36ca-bda4-c42a9e6bed92 -12:56.504 [INFO] jones.server: New leader elected: payload=Node-4e3e4264-3b2e-36ca-bda4-c42a9e6bed92 -12:56.504 [INFO] jones.server: federation state anti-entropy synced -12:56.609 [INFO] jones: Requesting shutdown -12:56.609 [INFO] jones.server: shutting down server -12:56.609 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -12:56.609 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -12:56.609 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -12:56.609 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -12:56.609 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -12:56.609 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -12:56.609 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -12:56.609 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" -12:56.609 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -12:56.609 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -12:56.609 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -12:56.609 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:56.610Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:56.610Z [INFO] agent.router.manager: shutting down -12:56.610 [INFO] jones: consul server down -12:56.610 [INFO] jones: shutdown complete -12:56.610 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:17128 network=tcp -12:56.610 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:17128 network=udp -12:56.610 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:17129 network=tcp -12:57.110 [INFO] jones: Waiting for endpoints to shut down -12:57.110 [INFO] jones: Endpoints down +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:08.727Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:08.760Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:08.762 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cb71a143-998f-7877-9f3f-7813b62431cd Address:127.0.0.1:24280}]" +49:08.762 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-cb71a143-998f-7877-9f3f-7813b62431cd.dc1 127.0.0.1 +49:08.763 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-cb71a143-998f-7877-9f3f-7813b62431cd 127.0.0.1 +2022-12-04T05:49:08.763Z [INFO] agent.router: Initializing LAN area manager +49:08.763 [INFO] jones: Started DNS server: address=127.0.0.1:24275 network=udp +49:08.763 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:24280 [Follower]" leader= +49:08.763 [INFO] jones.server: Adding LAN server: server="Node-cb71a143-998f-7877-9f3f-7813b62431cd (Addr: tcp/127.0.0.1:24280) (DC: dc1)" +49:08.763 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-cb71a143-998f-7877-9f3f-7813b62431cd.dc1 area=wan +49:08.763 [INFO] jones: Started DNS server: address=127.0.0.1:24275 network=tcp +49:08.764 [INFO] jones: Started HTTP server: address=127.0.0.1:24276 network=tcp +49:08.764 [INFO] jones: started state syncer +49:08.764 [INFO] jones: Started gRPC server: address=127.0.0.1:24281 network=tcp +49:08.812 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +49:08.812 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:24280 [Candidate]" term=2 +49:08.812 [DEBUG] jones.server.raft: votes: needed=1 +49:08.812 [DEBUG] jones.server.raft: vote granted: from=cb71a143-998f-7877-9f3f-7813b62431cd term=2 tally=1 +49:08.812 [INFO] jones.server.raft: election won: tally=1 +49:08.812 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:24280 [Leader]" +49:08.812 [INFO] jones.server: cluster leadership acquired +49:08.813 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24280 +49:08.813 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:08.815 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +49:08.815 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +49:08.815 [INFO] jones.leader: started routine: routine="federation state pruning" +49:08.815 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +49:08.815 [INFO] jones.leader: started routine: routine="CA root pruning" +49:08.815 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-cb71a143-998f-7877-9f3f-7813b62431cd +49:08.815 [INFO] jones.server: member joined, marking health alive: member=Node-cb71a143-998f-7877-9f3f-7813b62431cd +49:08.815 [INFO] jones.server: New leader elected: payload=Node-cb71a143-998f-7877-9f3f-7813b62431cd +49:08.816 [INFO] jones.server: federation state anti-entropy synced +49:08.956 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth +49:08.956 [INFO] jones: Synced node info +49:08.968 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth +49:08.968 [DEBUG] jones: Node info in sync +49:08.968 [DEBUG] jones: Node info in sync +49:09.074 [INFO] jones: Requesting shutdown +49:09.074 [INFO] jones.server: shutting down server +49:09.074 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +49:09.074 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +49:09.074 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +49:09.074 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +49:09.074 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +49:09.074 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +49:09.074 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +49:09.074 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +49:09.074 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +49:09.074 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +49:09.074 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:09.075Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:09.075Z [INFO] agent.router.manager: shutting down +49:09.075 [INFO] jones: consul server down +49:09.075 [INFO] jones: shutdown complete +49:09.075 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24275 network=tcp +49:09.075 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24275 network=udp +49:09.075 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:24276 network=tcp +49:09.575 [INFO] jones: Waiting for endpoints to shut down +49:09.575 [INFO] jones: Endpoints down === RUN TestAgent_sidecarServiceFromNodeService/inherit_tags_and_meta [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:57.133Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:57.177Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:57.191 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9aa6dc1f-17d4-0a48-1b38-0d71c32689c7 Address:127.0.0.1:17140}]" -12:57.191 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:17140 [Follower]" leader= -12:57.192 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-9aa6dc1f-17d4-0a48-1b38-0d71c32689c7.dc1 127.0.0.1 -12:57.193 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-9aa6dc1f-17d4-0a48-1b38-0d71c32689c7 127.0.0.1 -2021-10-31T23:12:57.193Z [INFO] agent.router: Initializing LAN area manager -12:57.193 [INFO] jones.server: Adding LAN server: server="Node-9aa6dc1f-17d4-0a48-1b38-0d71c32689c7 (Addr: tcp/127.0.0.1:17140) (DC: dc1)" -12:57.193 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-9aa6dc1f-17d4-0a48-1b38-0d71c32689c7.dc1 area=wan -12:57.193 [INFO] jones: Started DNS server: address=127.0.0.1:17135 network=tcp -12:57.193 [INFO] jones: Started DNS server: address=127.0.0.1:17135 network=udp -12:57.194 [INFO] jones: Started HTTP server: address=127.0.0.1:17136 network=tcp -12:57.194 [INFO] jones: started state syncer -12:57.194 [INFO] jones: Started gRPC server: address=127.0.0.1:17141 network=tcp -12:57.243 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -12:57.243 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:17140 [Candidate]" term=2 -12:57.243 [DEBUG] jones.server.raft: votes: needed=1 -12:57.243 [DEBUG] jones.server.raft: vote granted: from=9aa6dc1f-17d4-0a48-1b38-0d71c32689c7 term=2 tally=1 -12:57.243 [INFO] jones.server.raft: election won: tally=1 -12:57.243 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:17140 [Leader]" -12:57.243 [INFO] jones.server: cluster leadership acquired -12:57.244 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17140 -12:57.244 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:57.245 [INFO] jones.server: New leader elected: payload=Node-9aa6dc1f-17d4-0a48-1b38-0d71c32689c7 -12:57.246 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -12:57.246 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -12:57.246 [INFO] jones.leader: started routine: routine="federation state pruning" -12:57.246 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -12:57.246 [INFO] jones.leader: started routine: routine="CA root pruning" -12:57.246 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-9aa6dc1f-17d4-0a48-1b38-0d71c32689c7 -12:57.246 [INFO] jones.server: member joined, marking health alive: member=Node-9aa6dc1f-17d4-0a48-1b38-0d71c32689c7 -12:57.247 [INFO] jones.server: federation state anti-entropy synced -12:57.302 [INFO] jones: Requesting shutdown -12:57.302 [INFO] jones.server: shutting down server -12:57.302 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -12:57.302 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -12:57.302 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -12:57.302 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -12:57.302 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -12:57.302 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -12:57.302 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -12:57.302 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" -12:57.302 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -12:57.302 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -12:57.302 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -12:57.302 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:57.302Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:57.302Z [INFO] agent.router.manager: shutting down -12:57.302 [INFO] jones: consul server down -12:57.302 [INFO] jones: shutdown complete -12:57.302 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:17135 network=tcp -12:57.302 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:17135 network=udp -12:57.302 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:17136 network=tcp -12:57.805 [INFO] jones: Waiting for endpoints to shut down -12:57.805 [INFO] jones: Endpoints down +2022-12-04T05:49:09.587Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:09.598Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:09.599 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2557bb4b-698a-8224-81a5-c37ecc60bada Address:127.0.0.1:24287}]" +49:09.600 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:24287 [Follower]" leader= +49:09.600 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-2557bb4b-698a-8224-81a5-c37ecc60bada.dc1 127.0.0.1 +49:09.600 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-2557bb4b-698a-8224-81a5-c37ecc60bada 127.0.0.1 +2022-12-04T05:49:09.600Z [INFO] agent.router: Initializing LAN area manager +49:09.601 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-2557bb4b-698a-8224-81a5-c37ecc60bada.dc1 area=wan +49:09.601 [INFO] jones.server: Adding LAN server: server="Node-2557bb4b-698a-8224-81a5-c37ecc60bada (Addr: tcp/127.0.0.1:24287) (DC: dc1)" +49:09.601 [INFO] jones: Started DNS server: address=127.0.0.1:24282 network=udp +49:09.601 [INFO] jones: Started DNS server: address=127.0.0.1:24282 network=tcp +49:09.601 [INFO] jones: Started HTTP server: address=127.0.0.1:24283 network=tcp +49:09.601 [INFO] jones: started state syncer +49:09.601 [INFO] jones: Started gRPC server: address=127.0.0.1:24288 network=tcp +49:09.639 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +49:09.639 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:24287 [Candidate]" term=2 +49:09.639 [DEBUG] jones.server.raft: votes: needed=1 +49:09.639 [DEBUG] jones.server.raft: vote granted: from=2557bb4b-698a-8224-81a5-c37ecc60bada term=2 tally=1 +49:09.639 [INFO] jones.server.raft: election won: tally=1 +49:09.639 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:24287 [Leader]" +49:09.639 [INFO] jones.server: cluster leadership acquired +49:09.639 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24287 +49:09.640 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:09.641 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +49:09.641 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +49:09.641 [INFO] jones.leader: started routine: routine="federation state pruning" +49:09.641 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +49:09.641 [INFO] jones.leader: started routine: routine="CA root pruning" +49:09.641 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-2557bb4b-698a-8224-81a5-c37ecc60bada +49:09.641 [INFO] jones.server: member joined, marking health alive: member=Node-2557bb4b-698a-8224-81a5-c37ecc60bada +49:09.642 [INFO] jones.server: New leader elected: payload=Node-2557bb4b-698a-8224-81a5-c37ecc60bada +49:09.642 [INFO] jones.server: federation state anti-entropy synced +49:09.678 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth +49:09.678 [INFO] jones: Synced node info +49:09.678 [DEBUG] jones: Node info in sync +49:09.714 [INFO] jones: Requesting shutdown +49:09.714 [INFO] jones.server: shutting down server +49:09.714 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +49:09.714 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +49:09.714 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +49:09.714 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +49:09.714 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +49:09.714 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +49:09.714 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +49:09.714 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +49:09.714 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +49:09.714 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +49:09.714 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:09.714Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:09.714Z [INFO] agent.router.manager: shutting down +49:09.714 [INFO] jones: consul server down +49:09.714 [INFO] jones: shutdown complete +49:09.714 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24282 network=tcp +49:09.714 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24282 network=udp +49:09.714 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:24283 network=tcp +49:10.216 [INFO] jones: Waiting for endpoints to shut down +49:10.216 [INFO] jones: Endpoints down === RUN TestAgent_sidecarServiceFromNodeService/invalid_check_type [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:57.835Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:57.867Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:57.869 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6750a36b-2490-42c7-1342-83c53dfbf74c Address:127.0.0.1:16127}]" -12:57.869 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-6750a36b-2490-42c7-1342-83c53dfbf74c.dc1 127.0.0.1 -12:57.869 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:16127 [Follower]" leader= -12:57.890 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-6750a36b-2490-42c7-1342-83c53dfbf74c 127.0.0.1 -2021-10-31T23:12:57.890Z [INFO] agent.router: Initializing LAN area manager -12:57.890 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-6750a36b-2490-42c7-1342-83c53dfbf74c.dc1 area=wan -12:57.890 [INFO] jones: Started DNS server: address=127.0.0.1:17142 network=udp -12:57.890 [INFO] jones.server: Adding LAN server: server="Node-6750a36b-2490-42c7-1342-83c53dfbf74c (Addr: tcp/127.0.0.1:16127) (DC: dc1)" -12:57.890 [INFO] jones: Started DNS server: address=127.0.0.1:17142 network=tcp -12:57.891 [INFO] jones: Started HTTP server: address=127.0.0.1:17143 network=tcp -12:57.891 [INFO] jones: started state syncer -12:57.891 [INFO] jones: Started gRPC server: address=127.0.0.1:16128 network=tcp -12:57.910 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -12:57.910 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:16127 [Candidate]" term=2 -12:57.910 [DEBUG] jones.server.raft: votes: needed=1 -12:57.910 [DEBUG] jones.server.raft: vote granted: from=6750a36b-2490-42c7-1342-83c53dfbf74c term=2 tally=1 -12:57.910 [INFO] jones.server.raft: election won: tally=1 -12:57.910 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:16127 [Leader]" -12:57.910 [INFO] jones.server: cluster leadership acquired -12:57.910 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16127 -12:57.929 [INFO] jones.server: New leader elected: payload=Node-6750a36b-2490-42c7-1342-83c53dfbf74c -12:57.932 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:57.933 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -12:57.933 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -12:57.933 [INFO] jones.leader: started routine: routine="federation state pruning" -12:57.933 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -12:57.933 [INFO] jones.leader: started routine: routine="CA root pruning" -12:57.934 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-6750a36b-2490-42c7-1342-83c53dfbf74c -12:57.934 [INFO] jones.server: member joined, marking health alive: member=Node-6750a36b-2490-42c7-1342-83c53dfbf74c -12:57.934 [INFO] jones.server: federation state anti-entropy synced -12:58.078 [INFO] jones: Requesting shutdown -12:58.078 [INFO] jones.server: shutting down server -12:58.078 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -12:58.078 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -12:58.078 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -12:58.078 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -12:58.078 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -12:58.078 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -12:58.078 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -12:58.078 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" -12:58.078 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -12:58.078 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -12:58.078 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -12:58.078 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:58.078Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:58.078Z [INFO] agent.router.manager: shutting down -12:58.078 [INFO] jones: consul server down -12:58.078 [INFO] jones: shutdown complete -12:58.078 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:17142 network=tcp -12:58.078 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:17142 network=udp -12:58.078 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:17143 network=tcp -12:58.579 [INFO] jones: Waiting for endpoints to shut down -12:58.579 [INFO] jones: Endpoints down +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:10.246Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:10.272Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:10.278 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6b57a78a-fbd9-a733-4ff2-8bd8d360b6f7 Address:127.0.0.1:23274}]" +49:10.278 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-6b57a78a-fbd9-a733-4ff2-8bd8d360b6f7.dc1 127.0.0.1 +49:10.279 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-6b57a78a-fbd9-a733-4ff2-8bd8d360b6f7 127.0.0.1 +2022-12-04T05:49:10.279Z [INFO] agent.router: Initializing LAN area manager +49:10.279 [INFO] jones: Started DNS server: address=127.0.0.1:24289 network=udp +49:10.279 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:23274 [Follower]" leader= +49:10.279 [INFO] jones.server: Adding LAN server: server="Node-6b57a78a-fbd9-a733-4ff2-8bd8d360b6f7 (Addr: tcp/127.0.0.1:23274) (DC: dc1)" +49:10.279 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-6b57a78a-fbd9-a733-4ff2-8bd8d360b6f7.dc1 area=wan +49:10.279 [INFO] jones: Started DNS server: address=127.0.0.1:24289 network=tcp +49:10.288 [INFO] jones: Started HTTP server: address=127.0.0.1:24290 network=tcp +49:10.288 [INFO] jones: started state syncer +49:10.288 [INFO] jones: Started gRPC server: address=127.0.0.1:23275 network=tcp +49:10.332 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +49:10.332 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:23274 [Candidate]" term=2 +49:10.332 [DEBUG] jones.server.raft: votes: needed=1 +49:10.332 [DEBUG] jones.server.raft: vote granted: from=6b57a78a-fbd9-a733-4ff2-8bd8d360b6f7 term=2 tally=1 +49:10.332 [INFO] jones.server.raft: election won: tally=1 +49:10.332 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:23274 [Leader]" +49:10.332 [INFO] jones.server: cluster leadership acquired +49:10.332 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23274 +49:10.333 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:10.356 [INFO] jones.server: New leader elected: payload=Node-6b57a78a-fbd9-a733-4ff2-8bd8d360b6f7 +49:10.356 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +49:10.356 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +49:10.356 [INFO] jones.leader: started routine: routine="federation state pruning" +49:10.356 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +49:10.356 [INFO] jones.leader: started routine: routine="CA root pruning" +49:10.357 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-6b57a78a-fbd9-a733-4ff2-8bd8d360b6f7 +49:10.357 [INFO] jones.server: member joined, marking health alive: member=Node-6b57a78a-fbd9-a733-4ff2-8bd8d360b6f7 +49:10.358 [INFO] jones.server: federation state anti-entropy synced +49:10.492 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth +49:10.492 [INFO] jones: Synced node info +49:10.492 [DEBUG] jones: Node info in sync +49:10.609 [INFO] jones: Requesting shutdown +49:10.609 [INFO] jones.server: shutting down server +49:10.610 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +49:10.610 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +49:10.610 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +49:10.610 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +49:10.610 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +49:10.610 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +49:10.610 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +49:10.610 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +49:10.610 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +49:10.610 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +49:10.610 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +2022-12-04T05:49:10.610Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:10.610Z [INFO] agent.router.manager: shutting down +49:10.610 [INFO] jones: consul server down +49:10.610 [INFO] jones: shutdown complete +49:10.610 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24289 network=tcp +49:10.610 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:24289 network=udp +49:10.610 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:24290 network=tcp +49:11.110 [INFO] jones: Waiting for endpoints to shut down +49:11.110 [INFO] jones: Endpoints down === RUN TestAgent_sidecarServiceFromNodeService/invalid_meta [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:58.618Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:58.663Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:58.665 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fdaf0e83-aec4-7cc0-3109-6d60c561c455 Address:127.0.0.1:16134}]" -12:58.665 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:16134 [Follower]" leader= -12:58.666 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-fdaf0e83-aec4-7cc0-3109-6d60c561c455.dc1 127.0.0.1 -12:58.666 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-fdaf0e83-aec4-7cc0-3109-6d60c561c455 127.0.0.1 -2021-10-31T23:12:58.666Z [INFO] agent.router: Initializing LAN area manager -12:58.667 [INFO] jones: Started DNS server: address=127.0.0.1:16129 network=udp -12:58.667 [INFO] jones.server: Adding LAN server: server="Node-fdaf0e83-aec4-7cc0-3109-6d60c561c455 (Addr: tcp/127.0.0.1:16134) (DC: dc1)" -12:58.667 [INFO] jones: Started DNS server: address=127.0.0.1:16129 network=tcp -12:58.667 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-fdaf0e83-aec4-7cc0-3109-6d60c561c455.dc1 area=wan -12:58.667 [INFO] jones: Started HTTP server: address=127.0.0.1:16130 network=tcp -12:58.667 [INFO] jones: started state syncer -12:58.667 [INFO] jones: Started gRPC server: address=127.0.0.1:16135 network=tcp -12:58.702 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -12:58.702 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:16134 [Candidate]" term=2 -12:58.702 [DEBUG] jones.server.raft: votes: needed=1 -12:58.702 [DEBUG] jones.server.raft: vote granted: from=fdaf0e83-aec4-7cc0-3109-6d60c561c455 term=2 tally=1 -12:58.702 [INFO] jones.server.raft: election won: tally=1 -12:58.702 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:16134 [Leader]" -12:58.703 [INFO] jones.server: cluster leadership acquired -12:58.703 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16134 -12:58.703 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:58.704 [INFO] jones.server: New leader elected: payload=Node-fdaf0e83-aec4-7cc0-3109-6d60c561c455 -12:58.705 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -12:58.705 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -12:58.705 [INFO] jones.leader: started routine: routine="federation state pruning" -12:58.705 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -12:58.705 [INFO] jones.leader: started routine: routine="CA root pruning" -12:58.705 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-fdaf0e83-aec4-7cc0-3109-6d60c561c455 -12:58.705 [INFO] jones.server: member joined, marking health alive: member=Node-fdaf0e83-aec4-7cc0-3109-6d60c561c455 -12:58.711 [INFO] jones.server: federation state anti-entropy synced -12:58.954 [DEBUG] jones: Skipping remote check since it is managed automatically: check=serfHealth -12:58.954 [INFO] jones: Synced node info -12:59.026 [INFO] jones: Requesting shutdown -12:59.026 [INFO] jones.server: shutting down server -12:59.026 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -12:59.026 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -12:59.026 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -12:59.026 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -12:59.026 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -12:59.026 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -12:59.026 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -12:59.026 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -12:59.026 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -12:59.026 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -12:59.026 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:59.026Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:59.026Z [INFO] agent.router.manager: shutting down -12:59.026 [INFO] jones: consul server down -12:59.026 [INFO] jones: shutdown complete -12:59.026 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:16129 network=tcp -12:59.026 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:16129 network=udp -12:59.026 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:16130 network=tcp -12:59.529 [INFO] jones: Waiting for endpoints to shut down -12:59.529 [INFO] jones: Endpoints down +2022-12-04T05:49:11.121Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:11.130Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:11.132 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bf271242-2166-9eda-da3a-1377960d004e Address:127.0.0.1:23288}]" +49:11.132 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:23288 [Follower]" leader= +49:11.133 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-bf271242-2166-9eda-da3a-1377960d004e.dc1 127.0.0.1 +49:11.133 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-bf271242-2166-9eda-da3a-1377960d004e 127.0.0.1 +2022-12-04T05:49:11.133Z [INFO] agent.router: Initializing LAN area manager +49:11.133 [INFO] jones.server: Adding LAN server: server="Node-bf271242-2166-9eda-da3a-1377960d004e (Addr: tcp/127.0.0.1:23288) (DC: dc1)" +49:11.133 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-bf271242-2166-9eda-da3a-1377960d004e.dc1 area=wan +49:11.133 [INFO] jones: Started DNS server: address=127.0.0.1:23276 network=udp +49:11.133 [INFO] jones: Started DNS server: address=127.0.0.1:23276 network=tcp +49:11.134 [INFO] jones: Started HTTP server: address=127.0.0.1:23277 network=tcp +49:11.134 [INFO] jones: started state syncer +49:11.134 [INFO] jones: Started gRPC server: address=127.0.0.1:23289 network=tcp +49:11.171 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +49:11.171 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:23288 [Candidate]" term=2 +49:11.171 [DEBUG] jones.server.raft: votes: needed=1 +49:11.171 [DEBUG] jones.server.raft: vote granted: from=bf271242-2166-9eda-da3a-1377960d004e term=2 tally=1 +49:11.171 [INFO] jones.server.raft: election won: tally=1 +49:11.171 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:23288 [Leader]" +49:11.171 [INFO] jones.server: cluster leadership acquired +49:11.171 [INFO] jones.server: New leader elected: payload=Node-bf271242-2166-9eda-da3a-1377960d004e +49:11.171 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23288 +49:11.172 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:11.173 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +49:11.173 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +49:11.173 [INFO] jones.leader: started routine: routine="federation state pruning" +49:11.173 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +49:11.173 [INFO] jones.leader: started routine: routine="CA root pruning" +49:11.173 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-bf271242-2166-9eda-da3a-1377960d004e +49:11.173 [INFO] jones.server: member joined, marking health alive: member=Node-bf271242-2166-9eda-da3a-1377960d004e +49:11.174 [INFO] jones.server: federation state anti-entropy synced +49:11.354 [INFO] jones: Requesting shutdown +49:11.354 [INFO] jones.server: shutting down server +49:11.354 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +49:11.354 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +49:11.354 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +49:11.354 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +49:11.354 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +49:11.354 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +49:11.354 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +49:11.354 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" +49:11.354 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +49:11.354 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +49:11.354 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +49:11.354 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:11.354Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:11.354Z [INFO] agent.router.manager: shutting down +49:11.354 [INFO] jones: consul server down +49:11.354 [INFO] jones: shutdown complete +49:11.354 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:23276 network=tcp +49:11.354 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:23276 network=udp +49:11.354 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:23277 network=tcp +49:11.856 [INFO] jones: Waiting for endpoints to shut down +49:11.856 [INFO] jones: Endpoints down === RUN TestAgent_sidecarServiceFromNodeService/re-registering_same_sidecar_with_no_port_should_pick_same_one [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:12:59.568Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:59.619Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:59.621 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f4a8710a-5eb3-236e-8c27-da3bf0e601a0 Address:127.0.0.1:16141}]" -12:59.621 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:16141 [Follower]" leader= -12:59.630 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-f4a8710a-5eb3-236e-8c27-da3bf0e601a0.dc1 127.0.0.1 -12:59.631 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-f4a8710a-5eb3-236e-8c27-da3bf0e601a0 127.0.0.1 -2021-10-31T23:12:59.631Z [INFO] agent.router: Initializing LAN area manager -12:59.631 [INFO] jones: Started DNS server: address=127.0.0.1:16136 network=udp -12:59.631 [INFO] jones.server: Adding LAN server: server="Node-f4a8710a-5eb3-236e-8c27-da3bf0e601a0 (Addr: tcp/127.0.0.1:16141) (DC: dc1)" -12:59.631 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-f4a8710a-5eb3-236e-8c27-da3bf0e601a0.dc1 area=wan -12:59.631 [INFO] jones: Started DNS server: address=127.0.0.1:16136 network=tcp -12:59.632 [INFO] jones: Started HTTP server: address=127.0.0.1:16137 network=tcp -12:59.632 [INFO] jones: started state syncer -12:59.632 [INFO] jones: Started gRPC server: address=127.0.0.1:16142 network=tcp -12:59.690 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= -12:59.690 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:16141 [Candidate]" term=2 -12:59.690 [DEBUG] jones.server.raft: votes: needed=1 -12:59.690 [DEBUG] jones.server.raft: vote granted: from=f4a8710a-5eb3-236e-8c27-da3bf0e601a0 term=2 tally=1 -12:59.690 [INFO] jones.server.raft: election won: tally=1 -12:59.690 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:16141 [Leader]" -12:59.690 [INFO] jones.server: cluster leadership acquired -12:59.690 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16141 -12:59.690 [INFO] jones.server: New leader elected: payload=Node-f4a8710a-5eb3-236e-8c27-da3bf0e601a0 -12:59.691 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:59.692 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul -12:59.692 [INFO] jones.leader: started routine: routine="federation state anti-entropy" -12:59.692 [INFO] jones.leader: started routine: routine="federation state pruning" -12:59.692 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" -12:59.692 [INFO] jones.leader: started routine: routine="CA root pruning" -12:59.692 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-f4a8710a-5eb3-236e-8c27-da3bf0e601a0 -12:59.692 [INFO] jones.server: member joined, marking health alive: member=Node-f4a8710a-5eb3-236e-8c27-da3bf0e601a0 -12:59.693 [INFO] jones.server: federation state anti-entropy synced -12:59.815 [INFO] jones: Requesting shutdown -12:59.815 [INFO] jones.server: shutting down server -12:59.815 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" -12:59.815 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" -12:59.815 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" -12:59.815 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" -12:59.815 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave -12:59.816 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" -12:59.816 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" -12:59.816 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" -12:59.816 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" -12:59.816 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" -12:59.816 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" -12:59.816 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:59.816Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:59.816Z [INFO] agent.router.manager: shutting down -12:59.816 [INFO] jones: consul server down -12:59.816 [INFO] jones: shutdown complete -12:59.816 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:16136 network=tcp -12:59.816 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:16136 network=udp -12:59.816 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:16137 network=tcp -13:00.317 [INFO] jones: Waiting for endpoints to shut down -13:00.317 [INFO] jones: Endpoints down ---- PASS: TestAgent_sidecarServiceFromNodeService (7.49s) - --- PASS: TestAgent_sidecarServiceFromNodeService/no_sidecar (1.02s) - --- PASS: TestAgent_sidecarServiceFromNodeService/all_the_defaults (1.02s) - --- PASS: TestAgent_sidecarServiceFromNodeService/all_the_allowed_overrides (0.80s) - --- PASS: TestAgent_sidecarServiceFromNodeService/no_auto_ports_available (0.68s) - --- PASS: TestAgent_sidecarServiceFromNodeService/auto_ports_disabled (0.76s) - --- PASS: TestAgent_sidecarServiceFromNodeService/inherit_tags_and_meta (0.70s) - --- PASS: TestAgent_sidecarServiceFromNodeService/invalid_check_type (0.77s) - --- PASS: TestAgent_sidecarServiceFromNodeService/invalid_meta (0.95s) - --- PASS: TestAgent_sidecarServiceFromNodeService/re-registering_same_sidecar_with_no_port_should_pick_same_one (0.79s) +2022-12-04T05:49:11.874Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:11.887Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:11.889 [INFO] jones.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c6d14fbd-ceb7-a203-f4f6-42460de86e86 Address:127.0.0.1:23281}]" +49:11.890 [INFO] jones.server.serf.wan: serf: EventMemberJoin: Node-c6d14fbd-ceb7-a203-f4f6-42460de86e86.dc1 127.0.0.1 +49:11.891 [INFO] jones.server.serf.lan: serf: EventMemberJoin: Node-c6d14fbd-ceb7-a203-f4f6-42460de86e86 127.0.0.1 +2022-12-04T05:49:11.891Z [INFO] agent.router: Initializing LAN area manager +49:11.891 [INFO] jones: Started DNS server: address=127.0.0.1:23290 network=udp +49:11.891 [INFO] jones.server.raft: entering follower state: follower="Node at 127.0.0.1:23281 [Follower]" leader= +49:11.891 [INFO] jones.server: Adding LAN server: server="Node-c6d14fbd-ceb7-a203-f4f6-42460de86e86 (Addr: tcp/127.0.0.1:23281) (DC: dc1)" +49:11.891 [INFO] jones.server: Handled event for server in area: event=member-join server=Node-c6d14fbd-ceb7-a203-f4f6-42460de86e86.dc1 area=wan +49:11.891 [INFO] jones: Started DNS server: address=127.0.0.1:23290 network=tcp +49:11.896 [INFO] jones: Started HTTP server: address=127.0.0.1:23291 network=tcp +49:11.896 [INFO] jones: started state syncer +49:11.896 [INFO] jones: Started gRPC server: address=127.0.0.1:23282 network=tcp +49:11.936 [WARN] jones.server.raft: heartbeat timeout reached, starting election: last-leader= +49:11.936 [INFO] jones.server.raft: entering candidate state: node="Node at 127.0.0.1:23281 [Candidate]" term=2 +49:11.936 [DEBUG] jones.server.raft: votes: needed=1 +49:11.936 [DEBUG] jones.server.raft: vote granted: from=c6d14fbd-ceb7-a203-f4f6-42460de86e86 term=2 tally=1 +49:11.936 [INFO] jones.server.raft: election won: tally=1 +49:11.936 [INFO] jones.server.raft: entering leader state: leader="Node at 127.0.0.1:23281 [Leader]" +49:11.936 [INFO] jones.server: cluster leadership acquired +49:11.937 [DEBUG] jones.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23281 +49:11.937 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:11.939 [INFO] jones.server.connect: initialized primary datacenter CA with provider: provider=consul +49:11.939 [INFO] jones.leader: started routine: routine="federation state anti-entropy" +49:11.939 [INFO] jones.leader: started routine: routine="federation state pruning" +49:11.939 [INFO] jones.leader: started routine: routine="intermediate cert renew watch" +49:11.939 [INFO] jones.leader: started routine: routine="CA root pruning" +49:11.939 [DEBUG] jones.server: Skipping self join check for node since the cluster is too small: node=Node-c6d14fbd-ceb7-a203-f4f6-42460de86e86 +49:11.939 [INFO] jones.server: member joined, marking health alive: member=Node-c6d14fbd-ceb7-a203-f4f6-42460de86e86 +49:11.940 [INFO] jones.server: New leader elected: payload=Node-c6d14fbd-ceb7-a203-f4f6-42460de86e86 +49:11.940 [INFO] jones.server: federation state anti-entropy synced +49:12.060 [INFO] jones: Requesting shutdown +49:12.060 [INFO] jones.server: shutting down server +49:12.060 [DEBUG] jones.leader: stopping routine: routine="federation state anti-entropy" +49:12.060 [DEBUG] jones.leader: stopping routine: routine="federation state pruning" +49:12.060 [DEBUG] jones.leader: stopping routine: routine="intermediate cert renew watch" +49:12.060 [DEBUG] jones.leader: stopping routine: routine="CA root pruning" +49:12.060 [WARN] jones.server.serf.lan: serf: Shutdown without a Leave +49:12.061 [ERROR] jones.server: error performing anti-entropy sync of federation state: error="context canceled" +49:12.061 [DEBUG] jones.leader: stopped routine: routine="federation state anti-entropy" +49:12.061 [ERROR] jones.anti_entropy: failed to sync remote state: error="No cluster leader" +49:12.061 [DEBUG] jones.leader: stopped routine: routine="federation state pruning" +49:12.061 [DEBUG] jones.leader: stopped routine: routine="intermediate cert renew watch" +49:12.061 [DEBUG] jones.leader: stopped routine: routine="CA root pruning" +49:12.061 [WARN] jones.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:12.061Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:12.061Z [INFO] agent.router.manager: shutting down +49:12.061 [INFO] jones: consul server down +49:12.061 [INFO] jones: shutdown complete +49:12.061 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:23290 network=tcp +49:12.061 [INFO] jones: Stopping server: protocol=DNS address=127.0.0.1:23290 network=udp +49:12.061 [INFO] jones: Stopping server: protocol=HTTP address=127.0.0.1:23291 network=tcp +49:12.561 [INFO] jones: Waiting for endpoints to shut down +49:12.561 [INFO] jones: Endpoints down +--- PASS: TestAgent_sidecarServiceFromNodeService (6.98s) + --- PASS: TestAgent_sidecarServiceFromNodeService/no_sidecar (0.67s) + --- PASS: TestAgent_sidecarServiceFromNodeService/all_the_defaults (0.96s) + --- PASS: TestAgent_sidecarServiceFromNodeService/all_the_allowed_overrides (0.88s) + --- PASS: TestAgent_sidecarServiceFromNodeService/no_auto_ports_available (0.61s) + --- PASS: TestAgent_sidecarServiceFromNodeService/auto_ports_disabled (0.87s) + --- PASS: TestAgent_sidecarServiceFromNodeService/inherit_tags_and_meta (0.64s) + --- PASS: TestAgent_sidecarServiceFromNodeService/invalid_check_type (0.89s) + --- PASS: TestAgent_sidecarServiceFromNodeService/invalid_meta (0.75s) + --- PASS: TestAgent_sidecarServiceFromNodeService/re-registering_same_sidecar_with_no_port_should_pick_same_one (0.71s) === RUN TestSnapshot snapshot_endpoint_test.go:16: DM-skipped --- SKIP: TestSnapshot (0.00s) @@ -20880,62 +20939,40 @@ === PAUSE TestDefaultConfig/#499 === CONT TestDefaultConfig/#00 === CONT TestDefaultConfig/#499 -=== CONT TestDefaultConfig/#481 -=== CONT TestDefaultConfig/#464 -=== CONT TestDefaultConfig/#449 -=== CONT TestDefaultConfig/#435 +=== CONT TestDefaultConfig/#482 +=== CONT TestDefaultConfig/#466 +=== CONT TestDefaultConfig/#451 +=== CONT TestDefaultConfig/#437 +=== CONT TestDefaultConfig/#424 +=== CONT TestDefaultConfig/#412 +=== CONT TestDefaultConfig/#401 === CONT TestDefaultConfig/#498 -=== CONT TestDefaultConfig/#434 === CONT TestDefaultConfig/#497 === CONT TestDefaultConfig/#496 +=== CONT TestDefaultConfig/#490 === CONT TestDefaultConfig/#495 -=== CONT TestDefaultConfig/#422 -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#463 -=== CONT TestDefaultConfig/#462 -=== CONT TestDefaultConfig/#461 -=== CONT TestDefaultConfig/#460 -=== CONT TestDefaultConfig/#459 -=== CONT TestDefaultConfig/#458 -=== CONT TestDefaultConfig/#457 -=== CONT TestDefaultConfig/#456 -=== CONT TestDefaultConfig/#455 -=== CONT TestDefaultConfig/#454 -=== CONT TestDefaultConfig/#453 -=== CONT TestDefaultConfig/#452 -=== CONT TestDefaultConfig/#403 -=== CONT TestDefaultConfig/#451 -=== CONT TestDefaultConfig/#402 -=== CONT TestDefaultConfig/#401 -=== CONT TestDefaultConfig/#400 -=== CONT TestDefaultConfig/#450 -=== CONT TestDefaultConfig/#409 -=== CONT TestDefaultConfig/#408 -=== CONT TestDefaultConfig/#407 -=== CONT TestDefaultConfig/#406 -=== CONT TestDefaultConfig/#405 -=== CONT TestDefaultConfig/#404 -=== CONT TestDefaultConfig/#399 -=== CONT TestDefaultConfig/#398 -=== CONT TestDefaultConfig/#397 -=== CONT TestDefaultConfig/#394 -=== CONT TestDefaultConfig/#396 -=== CONT TestDefaultConfig/#393 -=== CONT TestDefaultConfig/#392 -=== CONT TestDefaultConfig/#395 -=== CONT TestDefaultConfig/#391 +=== CONT TestDefaultConfig/#494 +=== CONT TestDefaultConfig/#493 +=== CONT TestDefaultConfig/#492 +=== CONT TestDefaultConfig/#491 +=== CONT TestDefaultConfig/#489 +=== CONT TestDefaultConfig/#488 +=== CONT TestDefaultConfig/#487 +=== CONT TestDefaultConfig/#486 +=== CONT TestDefaultConfig/#485 +=== CONT TestDefaultConfig/#484 +=== CONT TestDefaultConfig/#483 === CONT TestDefaultConfig/#390 === CONT TestDefaultConfig/#389 === CONT TestDefaultConfig/#388 === CONT TestDefaultConfig/#387 === CONT TestDefaultConfig/#386 === CONT TestDefaultConfig/#385 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== CONT TestDefaultConfig/#380 === CONT TestDefaultConfig/#384 === CONT TestDefaultConfig/#383 === CONT TestDefaultConfig/#382 === CONT TestDefaultConfig/#381 -=== CONT TestDefaultConfig/#380 === CONT TestDefaultConfig/#379 === CONT TestDefaultConfig/#378 === CONT TestDefaultConfig/#377 @@ -20947,22 +20984,13 @@ === CONT TestDefaultConfig/#243 === CONT TestDefaultConfig/#371 === CONT TestDefaultConfig/#370 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#366 === CONT TestDefaultConfig/#369 === CONT TestDefaultConfig/#368 === CONT TestDefaultConfig/#367 +=== CONT TestDefaultConfig/#366 === CONT TestDefaultConfig/#365 -=== CONT TestDefaultConfig/#363 === CONT TestDefaultConfig/#364 -=== CONT TestDefaultConfig/#362 -=== CONT TestDefaultConfig/#361 -=== CONT TestDefaultConfig/#360 -=== CONT TestDefaultConfig/#359 -=== CONT TestDefaultConfig/#358 -=== CONT TestDefaultConfig/#357 -=== CONT TestDefaultConfig/#356 -=== CONT TestDefaultConfig/#355 +=== CONT TestDefaultConfig/#363 === CONT TestDefaultConfig/#354 === CONT TestDefaultConfig/#353 === CONT TestDefaultConfig/#352 @@ -20975,928 +21003,947 @@ === CONT TestDefaultConfig/#345 === CONT TestDefaultConfig/#344 === CONT TestDefaultConfig/#343 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === CONT TestDefaultConfig/#342 === CONT TestDefaultConfig/#341 === CONT TestDefaultConfig/#340 === CONT TestDefaultConfig/#339 === CONT TestDefaultConfig/#338 === CONT TestDefaultConfig/#337 +=== CONT TestDefaultConfig/#362 === CONT TestDefaultConfig/#336 +=== CONT TestDefaultConfig/#361 === CONT TestDefaultConfig/#335 +=== CONT TestDefaultConfig/#360 === CONT TestDefaultConfig/#334 +=== CONT TestDefaultConfig/#359 +=== CONT TestDefaultConfig/#358 === CONT TestDefaultConfig/#333 +=== CONT TestDefaultConfig/#357 +=== CONT TestDefaultConfig/#356 === CONT TestDefaultConfig/#332 === CONT TestDefaultConfig/#331 +=== CONT TestDefaultConfig/#355 === CONT TestDefaultConfig/#330 === CONT TestDefaultConfig/#329 -=== CONT TestDefaultConfig/#328 -=== CONT TestDefaultConfig/#327 -=== CONT TestDefaultConfig/#325 === CONT TestDefaultConfig/#326 +=== CONT TestDefaultConfig/#325 +=== CONT TestDefaultConfig/#328 === CONT TestDefaultConfig/#324 === CONT TestDefaultConfig/#323 === CONT TestDefaultConfig/#322 +=== CONT TestDefaultConfig/#327 === CONT TestDefaultConfig/#321 === CONT TestDefaultConfig/#320 === CONT TestDefaultConfig/#319 -=== CONT TestDefaultConfig/#318 -=== CONT TestDefaultConfig/#317 -=== CONT TestDefaultConfig/#316 === CONT TestDefaultConfig/#315 +=== CONT TestDefaultConfig/#318 +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! === CONT TestDefaultConfig/#314 === CONT TestDefaultConfig/#313 +=== CONT TestDefaultConfig/#317 +=== CONT TestDefaultConfig/#316 === CONT TestDefaultConfig/#312 -=== CONT TestDefaultConfig/#311 === CONT TestDefaultConfig/#310 === CONT TestDefaultConfig/#309 +=== CONT TestDefaultConfig/#311 === CONT TestDefaultConfig/#308 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === CONT TestDefaultConfig/#307 === CONT TestDefaultConfig/#306 === CONT TestDefaultConfig/#305 +=== CONT TestDefaultConfig/#301 === CONT TestDefaultConfig/#304 === CONT TestDefaultConfig/#303 -=== CONT TestDefaultConfig/#302 -=== CONT TestDefaultConfig/#301 === CONT TestDefaultConfig/#300 +=== CONT TestDefaultConfig/#302 === CONT TestDefaultConfig/#299 === CONT TestDefaultConfig/#298 -=== CONT TestDefaultConfig/#297 === CONT TestDefaultConfig/#296 === CONT TestDefaultConfig/#295 +=== CONT TestDefaultConfig/#297 === CONT TestDefaultConfig/#294 -=== CONT TestDefaultConfig/#293 === CONT TestDefaultConfig/#292 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== CONT TestDefaultConfig/#293 === CONT TestDefaultConfig/#291 === CONT TestDefaultConfig/#290 -=== CONT TestDefaultConfig/#289 === CONT TestDefaultConfig/#288 +=== CONT TestDefaultConfig/#289 === CONT TestDefaultConfig/#287 -=== CONT TestDefaultConfig/#286 === CONT TestDefaultConfig/#285 === CONT TestDefaultConfig/#284 +=== CONT TestDefaultConfig/#286 === CONT TestDefaultConfig/#283 === CONT TestDefaultConfig/#282 === CONT TestDefaultConfig/#281 === CONT TestDefaultConfig/#280 === CONT TestDefaultConfig/#279 === CONT TestDefaultConfig/#278 -=== CONT TestDefaultConfig/#277 === CONT TestDefaultConfig/#276 +=== CONT TestDefaultConfig/#277 === CONT TestDefaultConfig/#275 -=== CONT TestDefaultConfig/#274 === CONT TestDefaultConfig/#273 +=== CONT TestDefaultConfig/#274 === CONT TestDefaultConfig/#272 === CONT TestDefaultConfig/#271 -=== CONT TestDefaultConfig/#270 === CONT TestDefaultConfig/#269 +=== CONT TestDefaultConfig/#270 === CONT TestDefaultConfig/#268 === CONT TestDefaultConfig/#267 -=== CONT TestDefaultConfig/#266 === CONT TestDefaultConfig/#265 +=== CONT TestDefaultConfig/#266 === CONT TestDefaultConfig/#264 === CONT TestDefaultConfig/#263 -=== CONT TestDefaultConfig/#262 === CONT TestDefaultConfig/#261 +=== CONT TestDefaultConfig/#262 === CONT TestDefaultConfig/#260 -=== CONT TestDefaultConfig/#259 -=== CONT TestDefaultConfig/#258 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === CONT TestDefaultConfig/#257 === CONT TestDefaultConfig/#256 +=== CONT TestDefaultConfig/#259 === CONT TestDefaultConfig/#255 +=== CONT TestDefaultConfig/#258 === CONT TestDefaultConfig/#254 -=== CONT TestDefaultConfig/#253 === CONT TestDefaultConfig/#252 === CONT TestDefaultConfig/#251 +=== CONT TestDefaultConfig/#253 === CONT TestDefaultConfig/#250 === CONT TestDefaultConfig/#249 === CONT TestDefaultConfig/#248 -=== CONT TestDefaultConfig/#246 === CONT TestDefaultConfig/#247 +=== CONT TestDefaultConfig/#246 === CONT TestDefaultConfig/#245 === CONT TestDefaultConfig/#244 === CONT TestDefaultConfig/#114 -=== CONT TestDefaultConfig/#480 -=== CONT TestDefaultConfig/#479 -=== CONT TestDefaultConfig/#478 -=== CONT TestDefaultConfig/#477 -=== CONT TestDefaultConfig/#476 -=== CONT TestDefaultConfig/#433 -=== CONT TestDefaultConfig/#475 -=== CONT TestDefaultConfig/#432 -=== CONT TestDefaultConfig/#474 -=== CONT TestDefaultConfig/#431 -=== CONT TestDefaultConfig/#473 -=== CONT TestDefaultConfig/#430 -=== CONT TestDefaultConfig/#472 -=== CONT TestDefaultConfig/#429 -=== CONT TestDefaultConfig/#471 -=== CONT TestDefaultConfig/#428 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#470 -=== CONT TestDefaultConfig/#427 -=== CONT TestDefaultConfig/#426 -=== CONT TestDefaultConfig/#469 -=== CONT TestDefaultConfig/#425 -=== CONT TestDefaultConfig/#424 -=== CONT TestDefaultConfig/#423 -=== CONT TestDefaultConfig/#467 -=== CONT TestDefaultConfig/#466 === CONT TestDefaultConfig/#122 -=== CONT TestDefaultConfig/#465 === CONT TestDefaultConfig/#242 +=== CONT TestDefaultConfig/#431 +=== CONT TestDefaultConfig/#436 === CONT TestDefaultConfig/#241 -=== CONT TestDefaultConfig/#468 +=== CONT TestDefaultConfig/#435 === CONT TestDefaultConfig/#240 +=== CONT TestDefaultConfig/#434 +=== CONT TestDefaultConfig/#433 +=== CONT TestDefaultConfig/#432 === CONT TestDefaultConfig/#239 === CONT TestDefaultConfig/#238 === CONT TestDefaultConfig/#237 === CONT TestDefaultConfig/#236 -=== CONT TestDefaultConfig/#60 +=== CONT TestDefaultConfig/#418 +=== CONT TestDefaultConfig/#423 +=== CONT TestDefaultConfig/#422 +=== CONT TestDefaultConfig/#421 === CONT TestDefaultConfig/#235 +=== CONT TestDefaultConfig/#420 +=== CONT TestDefaultConfig/#419 === CONT TestDefaultConfig/#234 === CONT TestDefaultConfig/#233 -=== CONT TestDefaultConfig/#121 -=== CONT TestDefaultConfig/#120 -=== CONT TestDefaultConfig/#119 -=== CONT TestDefaultConfig/#118 === CONT TestDefaultConfig/#232 -=== CONT TestDefaultConfig/#117 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#115 -=== CONT TestDefaultConfig/#113 -=== CONT TestDefaultConfig/#112 -=== CONT TestDefaultConfig/#111 +=== CONT TestDefaultConfig/#428 === CONT TestDefaultConfig/#231 -=== CONT TestDefaultConfig/#110 +=== CONT TestDefaultConfig/#430 === CONT TestDefaultConfig/#230 +=== CONT TestDefaultConfig/#429 === CONT TestDefaultConfig/#229 === CONT TestDefaultConfig/#228 +=== CONT TestDefaultConfig/#444 === CONT TestDefaultConfig/#227 === CONT TestDefaultConfig/#226 +=== CONT TestDefaultConfig/#450 === CONT TestDefaultConfig/#225 +=== CONT TestDefaultConfig/#449 +=== CONT TestDefaultConfig/#448 === CONT TestDefaultConfig/#224 -=== CONT TestDefaultConfig/#116 -=== CONT TestDefaultConfig/#109 -=== CONT TestDefaultConfig/#108 +=== CONT TestDefaultConfig/#447 +=== CONT TestDefaultConfig/#446 === CONT TestDefaultConfig/#223 +=== CONT TestDefaultConfig/#445 === CONT TestDefaultConfig/#222 -=== CONT TestDefaultConfig/#107 -=== CONT TestDefaultConfig/#106 === CONT TestDefaultConfig/#221 === CONT TestDefaultConfig/#220 -=== CONT TestDefaultConfig/#105 -=== CONT TestDefaultConfig/#104 -=== CONT TestDefaultConfig/#103 -=== CONT TestDefaultConfig/#102 -=== CONT TestDefaultConfig/#101 === CONT TestDefaultConfig/#219 -=== CONT TestDefaultConfig/#100 +=== CONT TestDefaultConfig/#396 === CONT TestDefaultConfig/#218 +=== CONT TestDefaultConfig/#400 === CONT TestDefaultConfig/#217 === CONT TestDefaultConfig/#216 [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#98 -=== CONT TestDefaultConfig/#99 -=== CONT TestDefaultConfig/#215 +=== CONT TestDefaultConfig/#399 === CONT TestDefaultConfig/#214 +=== CONT TestDefaultConfig/#215 +=== CONT TestDefaultConfig/#398 === CONT TestDefaultConfig/#213 +=== CONT TestDefaultConfig/#397 === CONT TestDefaultConfig/#212 === CONT TestDefaultConfig/#211 === CONT TestDefaultConfig/#210 +=== CONT TestDefaultConfig/#441 === CONT TestDefaultConfig/#209 +=== CONT TestDefaultConfig/#443 +=== CONT TestDefaultConfig/#442 === CONT TestDefaultConfig/#208 -=== CONT TestDefaultConfig/#97 === CONT TestDefaultConfig/#207 -=== CONT TestDefaultConfig/#165 -=== CONT TestDefaultConfig/#96 +=== CONT TestDefaultConfig/#393 +=== CONT TestDefaultConfig/#395 === CONT TestDefaultConfig/#206 -=== CONT TestDefaultConfig/#164 -=== CONT TestDefaultConfig/#95 +=== CONT TestDefaultConfig/#394 === CONT TestDefaultConfig/#205 -=== CONT TestDefaultConfig/#94 -=== CONT TestDefaultConfig/#163 -=== CONT TestDefaultConfig/#162 === CONT TestDefaultConfig/#204 -=== CONT TestDefaultConfig/#161 +=== CONT TestDefaultConfig/#439 === CONT TestDefaultConfig/#203 -=== CONT TestDefaultConfig/#93 -=== CONT TestDefaultConfig/#160 -=== CONT TestDefaultConfig/#92 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== CONT TestDefaultConfig/#440 === CONT TestDefaultConfig/#202 -=== CONT TestDefaultConfig/#159 -=== CONT TestDefaultConfig/#91 -=== CONT TestDefaultConfig/#158 -=== CONT TestDefaultConfig/#157 === CONT TestDefaultConfig/#201 -=== CONT TestDefaultConfig/#156 +=== CONT TestDefaultConfig/#459 === CONT TestDefaultConfig/#200 -=== CONT TestDefaultConfig/#90 +=== CONT TestDefaultConfig/#465 === CONT TestDefaultConfig/#199 -=== CONT TestDefaultConfig/#155 +=== CONT TestDefaultConfig/#464 === CONT TestDefaultConfig/#198 -=== CONT TestDefaultConfig/#89 -=== CONT TestDefaultConfig/#154 +=== CONT TestDefaultConfig/#463 === CONT TestDefaultConfig/#197 -=== CONT TestDefaultConfig/#88 -=== CONT TestDefaultConfig/#87 +=== CONT TestDefaultConfig/#462 === CONT TestDefaultConfig/#196 -=== CONT TestDefaultConfig/#153 -=== CONT TestDefaultConfig/#152 -=== CONT TestDefaultConfig/#151 -=== CONT TestDefaultConfig/#150 +=== CONT TestDefaultConfig/#461 +=== CONT TestDefaultConfig/#460 === CONT TestDefaultConfig/#195 -=== CONT TestDefaultConfig/#149 -=== CONT TestDefaultConfig/#148 -=== CONT TestDefaultConfig/#147 -=== CONT TestDefaultConfig/#194 [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#145 -=== CONT TestDefaultConfig/#193 -=== CONT TestDefaultConfig/#144 +=== CONT TestDefaultConfig/#194 === CONT TestDefaultConfig/#192 -=== CONT TestDefaultConfig/#143 +=== CONT TestDefaultConfig/#193 +=== CONT TestDefaultConfig/#157 +=== CONT TestDefaultConfig/#156 === CONT TestDefaultConfig/#191 -=== CONT TestDefaultConfig/#142 === CONT TestDefaultConfig/#190 === CONT TestDefaultConfig/#189 +=== CONT TestDefaultConfig/#155 === CONT TestDefaultConfig/#188 -=== CONT TestDefaultConfig/#146 +=== CONT TestDefaultConfig/#154 === CONT TestDefaultConfig/#187 +=== CONT TestDefaultConfig/#186 +=== CONT TestDefaultConfig/#153 === CONT TestDefaultConfig/#185 +=== CONT TestDefaultConfig/#152 === CONT TestDefaultConfig/#184 +=== CONT TestDefaultConfig/#151 === CONT TestDefaultConfig/#183 +=== CONT TestDefaultConfig/#150 === CONT TestDefaultConfig/#182 +=== CONT TestDefaultConfig/#149 +=== CONT TestDefaultConfig/#148 === CONT TestDefaultConfig/#181 === CONT TestDefaultConfig/#180 +=== CONT TestDefaultConfig/#147 === CONT TestDefaultConfig/#179 === CONT TestDefaultConfig/#178 === CONT TestDefaultConfig/#177 +=== CONT TestDefaultConfig/#146 === CONT TestDefaultConfig/#176 +=== CONT TestDefaultConfig/#145 +=== CONT TestDefaultConfig/#144 === CONT TestDefaultConfig/#175 -=== CONT TestDefaultConfig/#186 +=== CONT TestDefaultConfig/#143 +=== CONT TestDefaultConfig/#142 === CONT TestDefaultConfig/#174 === CONT TestDefaultConfig/#173 +=== CONT TestDefaultConfig/#141 === CONT TestDefaultConfig/#172 +=== CONT TestDefaultConfig/#140 === CONT TestDefaultConfig/#171 +=== CONT TestDefaultConfig/#139 === CONT TestDefaultConfig/#170 +=== CONT TestDefaultConfig/#138 === CONT TestDefaultConfig/#169 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== CONT TestDefaultConfig/#137 === CONT TestDefaultConfig/#168 === CONT TestDefaultConfig/#167 +=== CONT TestDefaultConfig/#136 +=== CONT TestDefaultConfig/#135 === CONT TestDefaultConfig/#166 -=== CONT TestDefaultConfig/#488 -=== CONT TestDefaultConfig/#494 -=== CONT TestDefaultConfig/#493 -=== CONT TestDefaultConfig/#492 -=== CONT TestDefaultConfig/#491 -=== CONT TestDefaultConfig/#490 -=== CONT TestDefaultConfig/#489 -=== CONT TestDefaultConfig/#416 -=== CONT TestDefaultConfig/#421 -=== CONT TestDefaultConfig/#420 -=== CONT TestDefaultConfig/#419 -=== CONT TestDefaultConfig/#418 +=== CONT TestDefaultConfig/#134 +=== CONT TestDefaultConfig/#133 +=== CONT TestDefaultConfig/#165 +=== CONT TestDefaultConfig/#164 +=== CONT TestDefaultConfig/#132 +=== CONT TestDefaultConfig/#163 +=== CONT TestDefaultConfig/#131 +=== CONT TestDefaultConfig/#162 +=== CONT TestDefaultConfig/#161 +=== CONT TestDefaultConfig/#130 +=== CONT TestDefaultConfig/#160 +=== CONT TestDefaultConfig/#129 +=== CONT TestDefaultConfig/#128 +=== CONT TestDefaultConfig/#159 +=== CONT TestDefaultConfig/#158 +=== CONT TestDefaultConfig/#126 +=== CONT TestDefaultConfig/#127 +=== CONT TestDefaultConfig/#407 +=== CONT TestDefaultConfig/#411 +=== CONT TestDefaultConfig/#125 +=== CONT TestDefaultConfig/#410 +=== CONT TestDefaultConfig/#409 +=== CONT TestDefaultConfig/#408 +=== CONT TestDefaultConfig/#124 +=== CONT TestDefaultConfig/#123 +=== CONT TestDefaultConfig/#415 === CONT TestDefaultConfig/#417 -=== CONT TestDefaultConfig/#30 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#59 -=== CONT TestDefaultConfig/#58 -=== CONT TestDefaultConfig/#57 -=== CONT TestDefaultConfig/#56 -=== CONT TestDefaultConfig/#55 -=== CONT TestDefaultConfig/#54 -=== CONT TestDefaultConfig/#53 -=== CONT TestDefaultConfig/#52 -=== CONT TestDefaultConfig/#51 -=== CONT TestDefaultConfig/#50 -=== CONT TestDefaultConfig/#49 -=== CONT TestDefaultConfig/#48 -=== CONT TestDefaultConfig/#47 -=== CONT TestDefaultConfig/#46 +=== CONT TestDefaultConfig/#416 +=== CONT TestDefaultConfig/#392 +=== CONT TestDefaultConfig/#414 +=== CONT TestDefaultConfig/#406 +=== CONT TestDefaultConfig/#404 +=== CONT TestDefaultConfig/#405 +=== CONT TestDefaultConfig/#403 +=== CONT TestDefaultConfig/#474 +=== CONT TestDefaultConfig/#477 +=== CONT TestDefaultConfig/#391 +=== CONT TestDefaultConfig/#476 +=== CONT TestDefaultConfig/#478 +=== CONT TestDefaultConfig/#481 +=== CONT TestDefaultConfig/#480 +=== CONT TestDefaultConfig/#475 +=== CONT TestDefaultConfig/#479 +=== CONT TestDefaultConfig/#413 +=== CONT TestDefaultConfig/#61 +=== CONT TestDefaultConfig/#91 +=== CONT TestDefaultConfig/#121 +=== CONT TestDefaultConfig/#90 +=== CONT TestDefaultConfig/#89 +=== CONT TestDefaultConfig/#120 +=== CONT TestDefaultConfig/#119 [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#45 -=== CONT TestDefaultConfig/#44 -=== CONT TestDefaultConfig/#43 -=== CONT TestDefaultConfig/#42 -=== CONT TestDefaultConfig/#41 -=== CONT TestDefaultConfig/#40 -=== CONT TestDefaultConfig/#39 -=== CONT TestDefaultConfig/#38 -=== CONT TestDefaultConfig/#37 -=== CONT TestDefaultConfig/#36 -=== CONT TestDefaultConfig/#35 -=== CONT TestDefaultConfig/#34 -=== CONT TestDefaultConfig/#33 -=== CONT TestDefaultConfig/#32 -=== CONT TestDefaultConfig/#31 -=== CONT TestDefaultConfig/#485 -=== CONT TestDefaultConfig/#487 -=== CONT TestDefaultConfig/#486 -=== CONT TestDefaultConfig/#73 +=== CONT TestDefaultConfig/#88 +=== CONT TestDefaultConfig/#87 +=== CONT TestDefaultConfig/#86 === CONT TestDefaultConfig/#85 +=== CONT TestDefaultConfig/#117 === CONT TestDefaultConfig/#84 === CONT TestDefaultConfig/#83 +=== CONT TestDefaultConfig/#116 === CONT TestDefaultConfig/#82 +=== CONT TestDefaultConfig/#115 === CONT TestDefaultConfig/#81 +=== CONT TestDefaultConfig/#113 === CONT TestDefaultConfig/#80 +=== CONT TestDefaultConfig/#112 +=== CONT TestDefaultConfig/#118 === CONT TestDefaultConfig/#79 +=== CONT TestDefaultConfig/#111 +=== CONT TestDefaultConfig/#110 === CONT TestDefaultConfig/#78 +=== CONT TestDefaultConfig/#109 === CONT TestDefaultConfig/#77 === CONT TestDefaultConfig/#76 === CONT TestDefaultConfig/#75 === CONT TestDefaultConfig/#74 -=== CONT TestDefaultConfig/#132 -=== CONT TestDefaultConfig/#140 -=== CONT TestDefaultConfig/#139 -=== CONT TestDefaultConfig/#138 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#137 -=== CONT TestDefaultConfig/#136 -=== CONT TestDefaultConfig/#135 -=== CONT TestDefaultConfig/#134 -=== CONT TestDefaultConfig/#133 -=== CONT TestDefaultConfig/#483 -=== CONT TestDefaultConfig/#484 -=== CONT TestDefaultConfig/#67 +=== CONT TestDefaultConfig/#108 +=== CONT TestDefaultConfig/#107 +=== CONT TestDefaultConfig/#73 === CONT TestDefaultConfig/#72 +=== CONT TestDefaultConfig/#106 +=== CONT TestDefaultConfig/#105 +=== CONT TestDefaultConfig/#104 === CONT TestDefaultConfig/#71 +=== CONT TestDefaultConfig/#103 === CONT TestDefaultConfig/#70 +=== CONT TestDefaultConfig/#102 === CONT TestDefaultConfig/#69 +=== CONT TestDefaultConfig/#66 +=== CONT TestDefaultConfig/#101 === CONT TestDefaultConfig/#68 -=== CONT TestDefaultConfig/#15 +=== CONT TestDefaultConfig/#100 +=== CONT TestDefaultConfig/#67 +=== CONT TestDefaultConfig/#98 +=== CONT TestDefaultConfig/#65 +=== CONT TestDefaultConfig/#64 +=== CONT TestDefaultConfig/#97 +=== CONT TestDefaultConfig/#63 +=== CONT TestDefaultConfig/#96 +=== CONT TestDefaultConfig/#62 +=== CONT TestDefaultConfig/#95 +=== CONT TestDefaultConfig/#94 +=== CONT TestDefaultConfig/#99 +=== CONT TestDefaultConfig/#31 +=== CONT TestDefaultConfig/#60 +=== CONT TestDefaultConfig/#93 +=== CONT TestDefaultConfig/#59 +=== CONT TestDefaultConfig/#58 +=== CONT TestDefaultConfig/#56 +=== CONT TestDefaultConfig/#57 +=== CONT TestDefaultConfig/#55 +=== CONT TestDefaultConfig/#54 +=== CONT TestDefaultConfig/#43 +=== CONT TestDefaultConfig/#42 +=== CONT TestDefaultConfig/#41 +=== CONT TestDefaultConfig/#53 +=== CONT TestDefaultConfig/#40 +=== CONT TestDefaultConfig/#39 +=== CONT TestDefaultConfig/#52 +=== CONT TestDefaultConfig/#38 +=== CONT TestDefaultConfig/#51 +=== CONT TestDefaultConfig/#50 +=== CONT TestDefaultConfig/#37 +=== CONT TestDefaultConfig/#49 +=== CONT TestDefaultConfig/#92 +=== CONT TestDefaultConfig/#36 +=== CONT TestDefaultConfig/#48 +=== CONT TestDefaultConfig/#35 +=== CONT TestDefaultConfig/#47 +=== CONT TestDefaultConfig/#34 +=== CONT TestDefaultConfig/#46 +=== CONT TestDefaultConfig/#33 +=== CONT TestDefaultConfig/#45 +=== CONT TestDefaultConfig/#32 +=== CONT TestDefaultConfig/#44 +=== CONT TestDefaultConfig/#426 +=== CONT TestDefaultConfig/#427 +=== CONT TestDefaultConfig/#438 +=== CONT TestDefaultConfig/#470 +=== CONT TestDefaultConfig/#16 +=== CONT TestDefaultConfig/#30 +=== CONT TestDefaultConfig/#473 === CONT TestDefaultConfig/#29 === CONT TestDefaultConfig/#28 +=== CONT TestDefaultConfig/#472 +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== CONT TestDefaultConfig/#471 === CONT TestDefaultConfig/#27 === CONT TestDefaultConfig/#26 +=== CONT TestDefaultConfig/#458 +=== CONT TestDefaultConfig/#457 === CONT TestDefaultConfig/#25 === CONT TestDefaultConfig/#24 === CONT TestDefaultConfig/#23 +=== CONT TestDefaultConfig/#456 === CONT TestDefaultConfig/#22 +=== CONT TestDefaultConfig/#08 === CONT TestDefaultConfig/#21 +=== CONT TestDefaultConfig/#455 === CONT TestDefaultConfig/#20 === CONT TestDefaultConfig/#19 +=== CONT TestDefaultConfig/#15 === CONT TestDefaultConfig/#18 -=== CONT TestDefaultConfig/#17 -=== CONT TestDefaultConfig/#16 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#07 === CONT TestDefaultConfig/#14 +=== CONT TestDefaultConfig/#17 === CONT TestDefaultConfig/#13 === CONT TestDefaultConfig/#12 +=== CONT TestDefaultConfig/#468 +=== CONT TestDefaultConfig/#469 === CONT TestDefaultConfig/#11 === CONT TestDefaultConfig/#10 === CONT TestDefaultConfig/#09 -=== CONT TestDefaultConfig/#08 -=== CONT TestDefaultConfig/#127 -=== CONT TestDefaultConfig/#130 -=== CONT TestDefaultConfig/#129 -=== CONT TestDefaultConfig/#128 -=== CONT TestDefaultConfig/#64 -=== CONT TestDefaultConfig/#66 -=== CONT TestDefaultConfig/#65 -=== CONT TestDefaultConfig/#03 +=== CONT TestDefaultConfig/#402 +=== CONT TestDefaultConfig/#467 +=== CONT TestDefaultConfig/#453 +=== CONT TestDefaultConfig/#454 +=== CONT TestDefaultConfig/#425 +=== CONT TestDefaultConfig/#04 +=== CONT TestDefaultConfig/#07 === CONT TestDefaultConfig/#06 -=== CONT TestDefaultConfig/#131 +=== CONT TestDefaultConfig/#03 === CONT TestDefaultConfig/#05 -=== CONT TestDefaultConfig/#04 -=== CONT TestDefaultConfig/#01 === CONT TestDefaultConfig/#02 -=== CONT TestDefaultConfig/#413 -=== CONT TestDefaultConfig/#415 -=== CONT TestDefaultConfig/#414 -=== CONT TestDefaultConfig/#125 -=== CONT TestDefaultConfig/#126 -=== CONT TestDefaultConfig/#124 -=== CONT TestDefaultConfig/#411 -=== CONT TestDefaultConfig/#412 -=== CONT TestDefaultConfig/#62 -=== CONT TestDefaultConfig/#63 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== CONT TestDefaultConfig/#482 -=== CONT TestDefaultConfig/#448 -=== CONT TestDefaultConfig/#447 -=== CONT TestDefaultConfig/#442 -=== CONT TestDefaultConfig/#446 -=== CONT TestDefaultConfig/#445 -=== CONT TestDefaultConfig/#444 -=== CONT TestDefaultConfig/#443 -=== CONT TestDefaultConfig/#438 -=== CONT TestDefaultConfig/#441 -=== CONT TestDefaultConfig/#440 -=== CONT TestDefaultConfig/#123 -=== CONT TestDefaultConfig/#61 -=== CONT TestDefaultConfig/#410 -=== CONT TestDefaultConfig/#439 -=== CONT TestDefaultConfig/#436 -=== CONT TestDefaultConfig/#437 -=== CONT TestDefaultConfig/#86 -=== CONT TestDefaultConfig/#141 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! ---- PASS: TestDefaultConfig (0.02s) - --- PASS: TestDefaultConfig/#481 (0.08s) - --- PASS: TestDefaultConfig/#462 (0.05s) - --- PASS: TestDefaultConfig/#422 (0.05s) - --- PASS: TestDefaultConfig/#499 (0.09s) - --- PASS: TestDefaultConfig/#434 (0.08s) - --- PASS: TestDefaultConfig/#498 (0.08s) - --- PASS: TestDefaultConfig/#460 (0.02s) - --- PASS: TestDefaultConfig/#457 (0.01s) - --- PASS: TestDefaultConfig/#00 (0.18s) - --- PASS: TestDefaultConfig/#455 (0.09s) - --- PASS: TestDefaultConfig/#497 (0.18s) - --- PASS: TestDefaultConfig/#458 (0.10s) - --- PASS: TestDefaultConfig/#453 (0.09s) - --- PASS: TestDefaultConfig/#463 (0.16s) - --- PASS: TestDefaultConfig/#454 (0.09s) - --- PASS: TestDefaultConfig/#435 (0.19s) - --- PASS: TestDefaultConfig/#461 (0.15s) - --- PASS: TestDefaultConfig/#459 (0.12s) - --- PASS: TestDefaultConfig/#449 (0.19s) - --- PASS: TestDefaultConfig/#456 (0.11s) - --- PASS: TestDefaultConfig/#452 (0.01s) - --- PASS: TestDefaultConfig/#450 (0.07s) - --- PASS: TestDefaultConfig/#406 (0.05s) - --- PASS: TestDefaultConfig/#409 (0.08s) - --- PASS: TestDefaultConfig/#451 (0.09s) - --- PASS: TestDefaultConfig/#399 (0.02s) - --- PASS: TestDefaultConfig/#407 (0.09s) - --- PASS: TestDefaultConfig/#403 (0.11s) - --- PASS: TestDefaultConfig/#401 (0.10s) - --- PASS: TestDefaultConfig/#397 (0.04s) - --- PASS: TestDefaultConfig/#402 (0.17s) - --- PASS: TestDefaultConfig/#398 (0.09s) - --- PASS: TestDefaultConfig/#395 (0.07s) - --- PASS: TestDefaultConfig/#464 (0.38s) - --- PASS: TestDefaultConfig/#394 (0.08s) - --- PASS: TestDefaultConfig/#393 (0.08s) +=== CONT TestDefaultConfig/#452 +=== CONT TestDefaultConfig/#01 +--- PASS: TestDefaultConfig (0.01s) + --- PASS: TestDefaultConfig/#499 (0.01s) + --- PASS: TestDefaultConfig/#482 (0.02s) + --- PASS: TestDefaultConfig/#00 (0.04s) + --- PASS: TestDefaultConfig/#466 (0.03s) + --- PASS: TestDefaultConfig/#497 (0.02s) + --- PASS: TestDefaultConfig/#491 (0.01s) + --- PASS: TestDefaultConfig/#496 (0.02s) + --- PASS: TestDefaultConfig/#487 (0.01s) + --- PASS: TestDefaultConfig/#494 (0.02s) + --- PASS: TestDefaultConfig/#498 (0.03s) + --- PASS: TestDefaultConfig/#495 (0.02s) + --- PASS: TestDefaultConfig/#412 (0.06s) + --- PASS: TestDefaultConfig/#486 (0.03s) + --- PASS: TestDefaultConfig/#489 (0.04s) + --- PASS: TestDefaultConfig/#493 (0.05s) + --- PASS: TestDefaultConfig/#390 (0.03s) + --- PASS: TestDefaultConfig/#484 (0.04s) + --- PASS: TestDefaultConfig/#485 (0.04s) + --- PASS: TestDefaultConfig/#490 (0.06s) --- PASS: TestDefaultConfig/#388 (0.01s) - --- PASS: TestDefaultConfig/#392 (0.08s) - --- PASS: TestDefaultConfig/#495 (0.38s) - --- PASS: TestDefaultConfig/#404 (0.15s) - --- PASS: TestDefaultConfig/#389 (0.05s) - --- PASS: TestDefaultConfig/#408 (0.22s) - --- PASS: TestDefaultConfig/#396 (0.17s) - --- PASS: TestDefaultConfig/#385 (0.09s) - --- PASS: TestDefaultConfig/#390 (0.11s) - --- PASS: TestDefaultConfig/#382 (0.09s) - --- PASS: TestDefaultConfig/#387 (0.11s) - --- PASS: TestDefaultConfig/#376 (0.01s) - --- PASS: TestDefaultConfig/#391 (0.16s) - --- PASS: TestDefaultConfig/#400 (0.29s) - --- PASS: TestDefaultConfig/#496 (0.49s) - --- PASS: TestDefaultConfig/#378 (0.03s) - --- PASS: TestDefaultConfig/#383 (0.17s) - --- PASS: TestDefaultConfig/#371 (0.06s) - --- PASS: TestDefaultConfig/#381 (0.18s) - --- PASS: TestDefaultConfig/#386 (0.19s) - --- PASS: TestDefaultConfig/#375 (0.09s) - --- PASS: TestDefaultConfig/#379 (0.15s) - --- PASS: TestDefaultConfig/#368 (0.02s) - --- PASS: TestDefaultConfig/#366 (0.03s) - --- PASS: TestDefaultConfig/#372 (0.13s) - --- PASS: TestDefaultConfig/#384 (0.24s) - --- PASS: TestDefaultConfig/#380 (0.22s) - --- PASS: TestDefaultConfig/#370 (0.15s) - --- PASS: TestDefaultConfig/#369 (0.09s) - --- PASS: TestDefaultConfig/#359 (0.01s) - --- PASS: TestDefaultConfig/#364 (0.07s) - --- PASS: TestDefaultConfig/#363 (0.08s) - --- PASS: TestDefaultConfig/#405 (0.43s) - --- PASS: TestDefaultConfig/#357 (0.02s) - --- PASS: TestDefaultConfig/#361 (0.05s) - --- PASS: TestDefaultConfig/#373 (0.18s) - --- PASS: TestDefaultConfig/#356 (0.05s) - --- PASS: TestDefaultConfig/#362 (0.16s) - --- PASS: TestDefaultConfig/#360 (0.11s) - --- PASS: TestDefaultConfig/#353 (0.09s) - --- PASS: TestDefaultConfig/#365 (0.17s) + --- PASS: TestDefaultConfig/#492 (0.06s) + --- PASS: TestDefaultConfig/#401 (0.07s) + --- PASS: TestDefaultConfig/#386 (0.01s) + --- PASS: TestDefaultConfig/#380 (0.01s) + --- PASS: TestDefaultConfig/#451 (0.08s) + --- PASS: TestDefaultConfig/#387 (0.02s) + --- PASS: TestDefaultConfig/#424 (0.10s) + --- PASS: TestDefaultConfig/#374 (0.02s) + --- PASS: TestDefaultConfig/#381 (0.03s) + --- PASS: TestDefaultConfig/#383 (0.03s) + --- PASS: TestDefaultConfig/#375 (0.03s) + --- PASS: TestDefaultConfig/#373 (0.01s) + --- PASS: TestDefaultConfig/#385 (0.04s) + --- PASS: TestDefaultConfig/#379 (0.03s) + --- PASS: TestDefaultConfig/#376 (0.03s) + --- PASS: TestDefaultConfig/#382 (0.04s) + --- PASS: TestDefaultConfig/#377 (0.03s) + --- PASS: TestDefaultConfig/#372 (0.01s) + --- PASS: TestDefaultConfig/#243 (0.02s) + --- PASS: TestDefaultConfig/#437 (0.13s) + --- PASS: TestDefaultConfig/#384 (0.06s) + --- PASS: TestDefaultConfig/#378 (0.05s) + --- PASS: TestDefaultConfig/#369 (0.02s) + --- PASS: TestDefaultConfig/#350 (0.01s) + --- PASS: TestDefaultConfig/#351 (0.01s) + --- PASS: TestDefaultConfig/#364 (0.03s) + --- PASS: TestDefaultConfig/#370 (0.04s) + --- PASS: TestDefaultConfig/#365 (0.03s) + --- PASS: TestDefaultConfig/#368 (0.04s) + --- PASS: TestDefaultConfig/#353 (0.02s) + --- PASS: TestDefaultConfig/#348 (0.01s) + --- PASS: TestDefaultConfig/#363 (0.04s) --- PASS: TestDefaultConfig/#347 (0.01s) - --- PASS: TestDefaultConfig/#243 (0.26s) - --- PASS: TestDefaultConfig/#355 (0.11s) - --- PASS: TestDefaultConfig/#348 (0.02s) - --- PASS: TestDefaultConfig/#377 (0.31s) - --- PASS: TestDefaultConfig/#346 (0.03s) - --- PASS: TestDefaultConfig/#342 (0.02s) - --- PASS: TestDefaultConfig/#367 (0.24s) - --- PASS: TestDefaultConfig/#352 (0.16s) - --- PASS: TestDefaultConfig/#358 (0.20s) - --- PASS: TestDefaultConfig/#341 (0.08s) - --- PASS: TestDefaultConfig/#345 (0.10s) - --- PASS: TestDefaultConfig/#344 (0.11s) - --- PASS: TestDefaultConfig/#343 (0.11s) - --- PASS: TestDefaultConfig/#350 (0.20s) + --- PASS: TestDefaultConfig/#371 (0.05s) + --- PASS: TestDefaultConfig/#483 (0.12s) + --- PASS: TestDefaultConfig/#349 (0.02s) + --- PASS: TestDefaultConfig/#488 (0.14s) + --- PASS: TestDefaultConfig/#389 (0.13s) + --- PASS: TestDefaultConfig/#346 (0.02s) + --- PASS: TestDefaultConfig/#339 (0.02s) + --- PASS: TestDefaultConfig/#341 (0.03s) + --- PASS: TestDefaultConfig/#336 (0.01s) + --- PASS: TestDefaultConfig/#362 (0.01s) + --- PASS: TestDefaultConfig/#337 (0.01s) + --- PASS: TestDefaultConfig/#361 (0.01s) + --- PASS: TestDefaultConfig/#342 (0.04s) + --- PASS: TestDefaultConfig/#344 (0.05s) + --- PASS: TestDefaultConfig/#338 (0.03s) + --- PASS: TestDefaultConfig/#354 (0.07s) + --- PASS: TestDefaultConfig/#360 (0.02s) + --- PASS: TestDefaultConfig/#340 (0.05s) + --- PASS: TestDefaultConfig/#359 (0.02s) --- PASS: TestDefaultConfig/#333 (0.02s) + --- PASS: TestDefaultConfig/#343 (0.05s) + --- PASS: TestDefaultConfig/#366 (0.09s) + --- PASS: TestDefaultConfig/#357 (0.01s) + --- PASS: TestDefaultConfig/#352 (0.07s) + --- PASS: TestDefaultConfig/#332 (0.01s) + --- PASS: TestDefaultConfig/#330 (0.01s) + --- PASS: TestDefaultConfig/#358 (0.03s) + --- PASS: TestDefaultConfig/#345 (0.06s) + --- PASS: TestDefaultConfig/#355 (0.01s) --- PASS: TestDefaultConfig/#335 (0.04s) - --- PASS: TestDefaultConfig/#331 (0.01s) - --- PASS: TestDefaultConfig/#334 (0.06s) - --- PASS: TestDefaultConfig/#336 (0.08s) - --- PASS: TestDefaultConfig/#338 (0.16s) - --- PASS: TestDefaultConfig/#330 (0.08s) - --- PASS: TestDefaultConfig/#374 (0.47s) - --- PASS: TestDefaultConfig/#332 (0.09s) + --- PASS: TestDefaultConfig/#367 (0.10s) + --- PASS: TestDefaultConfig/#331 (0.02s) + --- PASS: TestDefaultConfig/#329 (0.02s) + --- PASS: TestDefaultConfig/#328 (0.02s) + --- PASS: TestDefaultConfig/#334 (0.05s) + --- PASS: TestDefaultConfig/#326 (0.02s) + --- PASS: TestDefaultConfig/#323 (0.02s) + --- PASS: TestDefaultConfig/#322 (0.02s) --- PASS: TestDefaultConfig/#324 (0.02s) - --- PASS: TestDefaultConfig/#325 (0.06s) - --- PASS: TestDefaultConfig/#351 (0.30s) - --- PASS: TestDefaultConfig/#337 (0.16s) - --- PASS: TestDefaultConfig/#323 (0.03s) - --- PASS: TestDefaultConfig/#327 (0.10s) - --- PASS: TestDefaultConfig/#329 (0.10s) - --- PASS: TestDefaultConfig/#322 (0.05s) - --- PASS: TestDefaultConfig/#315 (0.08s) - --- PASS: TestDefaultConfig/#320 (0.09s) - --- PASS: TestDefaultConfig/#354 (0.40s) - --- PASS: TestDefaultConfig/#328 (0.19s) - --- PASS: TestDefaultConfig/#317 (0.10s) + --- PASS: TestDefaultConfig/#315 (0.01s) + --- PASS: TestDefaultConfig/#325 (0.03s) + --- PASS: TestDefaultConfig/#318 (0.01s) + --- PASS: TestDefaultConfig/#327 (0.02s) + --- PASS: TestDefaultConfig/#321 (0.02s) + --- PASS: TestDefaultConfig/#320 (0.04s) --- PASS: TestDefaultConfig/#311 (0.02s) - --- PASS: TestDefaultConfig/#316 (0.11s) - --- PASS: TestDefaultConfig/#339 (0.30s) - --- PASS: TestDefaultConfig/#313 (0.08s) - --- PASS: TestDefaultConfig/#310 (0.02s) - --- PASS: TestDefaultConfig/#321 (0.21s) - --- PASS: TestDefaultConfig/#340 (0.41s) - --- PASS: TestDefaultConfig/#326 (0.27s) - --- PASS: TestDefaultConfig/#319 (0.22s) - --- PASS: TestDefaultConfig/#318 (0.23s) - --- PASS: TestDefaultConfig/#312 (0.15s) - --- PASS: TestDefaultConfig/#309 (0.13s) - --- PASS: TestDefaultConfig/#349 (0.50s) - --- PASS: TestDefaultConfig/#303 (0.13s) - --- PASS: TestDefaultConfig/#308 (0.16s) - --- PASS: TestDefaultConfig/#314 (0.27s) - --- PASS: TestDefaultConfig/#296 (0.08s) - --- PASS: TestDefaultConfig/#295 (0.08s) - --- PASS: TestDefaultConfig/#300 (0.10s) - --- PASS: TestDefaultConfig/#298 (0.10s) - --- PASS: TestDefaultConfig/#305 (0.21s) - --- PASS: TestDefaultConfig/#293 (0.06s) + --- PASS: TestDefaultConfig/#308 (0.02s) + --- PASS: TestDefaultConfig/#313 (0.03s) + --- PASS: TestDefaultConfig/#319 (0.04s) + --- PASS: TestDefaultConfig/#317 (0.02s) + --- PASS: TestDefaultConfig/#304 (0.01s) + --- PASS: TestDefaultConfig/#316 (0.03s) + --- PASS: TestDefaultConfig/#312 (0.03s) + --- PASS: TestDefaultConfig/#305 (0.03s) + --- PASS: TestDefaultConfig/#303 (0.01s) + --- PASS: TestDefaultConfig/#356 (0.06s) + --- PASS: TestDefaultConfig/#310 (0.04s) + --- PASS: TestDefaultConfig/#302 (0.02s) + --- PASS: TestDefaultConfig/#314 (0.04s) + --- PASS: TestDefaultConfig/#299 (0.02s) + --- PASS: TestDefaultConfig/#300 (0.02s) + --- PASS: TestDefaultConfig/#309 (0.04s) + --- PASS: TestDefaultConfig/#288 (0.01s) + --- PASS: TestDefaultConfig/#298 (0.03s) + --- PASS: TestDefaultConfig/#296 (0.03s) + --- PASS: TestDefaultConfig/#294 (0.02s) + --- PASS: TestDefaultConfig/#306 (0.05s) --- PASS: TestDefaultConfig/#289 (0.01s) - --- PASS: TestDefaultConfig/#302 (0.13s) - --- PASS: TestDefaultConfig/#292 (0.06s) - --- PASS: TestDefaultConfig/#306 (0.22s) --- PASS: TestDefaultConfig/#287 (0.01s) - --- PASS: TestDefaultConfig/#291 (0.05s) - --- PASS: TestDefaultConfig/#288 (0.04s) - --- PASS: TestDefaultConfig/#299 (0.18s) - --- PASS: TestDefaultConfig/#301 (0.19s) - --- PASS: TestDefaultConfig/#280 (0.05s) - --- PASS: TestDefaultConfig/#278 (0.01s) - --- PASS: TestDefaultConfig/#286 (0.09s) - --- PASS: TestDefaultConfig/#290 (0.11s) - --- PASS: TestDefaultConfig/#294 (0.18s) - --- PASS: TestDefaultConfig/#283 (0.09s) - --- PASS: TestDefaultConfig/#275 (0.01s) - --- PASS: TestDefaultConfig/#282 (0.09s) - --- PASS: TestDefaultConfig/#281 (0.12s) - --- PASS: TestDefaultConfig/#307 (0.34s) - --- PASS: TestDefaultConfig/#268 (0.04s) - --- PASS: TestDefaultConfig/#273 (0.07s) - --- PASS: TestDefaultConfig/#284 (0.17s) - --- PASS: TestDefaultConfig/#304 (0.39s) - --- PASS: TestDefaultConfig/#274 (0.09s) - --- PASS: TestDefaultConfig/#279 (0.15s) - --- PASS: TestDefaultConfig/#276 (0.10s) - --- PASS: TestDefaultConfig/#266 (0.03s) - --- PASS: TestDefaultConfig/#265 (0.06s) - --- PASS: TestDefaultConfig/#272 (0.16s) - --- PASS: TestDefaultConfig/#270 (0.16s) - --- PASS: TestDefaultConfig/#285 (0.27s) - --- PASS: TestDefaultConfig/#271 (0.17s) - --- PASS: TestDefaultConfig/#267 (0.15s) - --- PASS: TestDefaultConfig/#277 (0.19s) - --- PASS: TestDefaultConfig/#264 (0.10s) - --- PASS: TestDefaultConfig/#256 (0.02s) - --- PASS: TestDefaultConfig/#254 (0.02s) - --- PASS: TestDefaultConfig/#255 (0.02s) - --- PASS: TestDefaultConfig/#269 (0.21s) - --- PASS: TestDefaultConfig/#257 (0.05s) - --- PASS: TestDefaultConfig/#253 (0.05s) - --- PASS: TestDefaultConfig/#260 (0.16s) - --- PASS: TestDefaultConfig/#248 (0.06s) - --- PASS: TestDefaultConfig/#263 (0.17s) - --- PASS: TestDefaultConfig/#262 (0.17s) - --- PASS: TestDefaultConfig/#114 (0.01s) - --- PASS: TestDefaultConfig/#252 (0.09s) - --- PASS: TestDefaultConfig/#297 (0.46s) - --- PASS: TestDefaultConfig/#259 (0.18s) - --- PASS: TestDefaultConfig/#478 (0.01s) - --- PASS: TestDefaultConfig/#479 (0.02s) - --- PASS: TestDefaultConfig/#246 (0.09s) - --- PASS: TestDefaultConfig/#477 (0.05s) - --- PASS: TestDefaultConfig/#474 (0.04s) - --- PASS: TestDefaultConfig/#261 (0.26s) - --- PASS: TestDefaultConfig/#249 (0.16s) - --- PASS: TestDefaultConfig/#244 (0.11s) - --- PASS: TestDefaultConfig/#247 (0.13s) - --- PASS: TestDefaultConfig/#473 (0.01s) - --- PASS: TestDefaultConfig/#476 (0.10s) - --- PASS: TestDefaultConfig/#433 (0.10s) - --- PASS: TestDefaultConfig/#429 (0.04s) - --- PASS: TestDefaultConfig/#432 (0.14s) - --- PASS: TestDefaultConfig/#472 (0.06s) - --- PASS: TestDefaultConfig/#471 (0.05s) - --- PASS: TestDefaultConfig/#258 (0.29s) - --- PASS: TestDefaultConfig/#428 (0.06s) - --- PASS: TestDefaultConfig/#424 (0.03s) - --- PASS: TestDefaultConfig/#423 (0.01s) - --- PASS: TestDefaultConfig/#475 (0.19s) - --- PASS: TestDefaultConfig/#470 (0.10s) - --- PASS: TestDefaultConfig/#480 (0.21s) - --- PASS: TestDefaultConfig/#469 (0.06s) - --- PASS: TestDefaultConfig/#431 (0.15s) - --- PASS: TestDefaultConfig/#465 (0.04s) - --- PASS: TestDefaultConfig/#430 (0.19s) - --- PASS: TestDefaultConfig/#426 (0.16s) - --- PASS: TestDefaultConfig/#122 (0.10s) - --- PASS: TestDefaultConfig/#427 (0.19s) - --- PASS: TestDefaultConfig/#241 (0.09s) - --- PASS: TestDefaultConfig/#242 (0.10s) - --- PASS: TestDefaultConfig/#239 (0.06s) - --- PASS: TestDefaultConfig/#238 (0.02s) - --- PASS: TestDefaultConfig/#467 (0.12s) - --- PASS: TestDefaultConfig/#466 (0.12s) - --- PASS: TestDefaultConfig/#245 (0.34s) - --- PASS: TestDefaultConfig/#468 (0.10s) - --- PASS: TestDefaultConfig/#235 (0.05s) - --- PASS: TestDefaultConfig/#232 (0.06s) - --- PASS: TestDefaultConfig/#425 (0.23s) - --- PASS: TestDefaultConfig/#119 (0.07s) - --- PASS: TestDefaultConfig/#118 (0.08s) - --- PASS: TestDefaultConfig/#250 (0.47s) - --- PASS: TestDefaultConfig/#113 (0.01s) - --- PASS: TestDefaultConfig/#240 (0.19s) - --- PASS: TestDefaultConfig/#121 (0.10s) - --- PASS: TestDefaultConfig/#237 (0.12s) - --- PASS: TestDefaultConfig/#111 (0.01s) - --- PASS: TestDefaultConfig/#115 (0.03s) - --- PASS: TestDefaultConfig/#234 (0.11s) - --- PASS: TestDefaultConfig/#60 (0.14s) + --- PASS: TestDefaultConfig/#295 (0.03s) + --- PASS: TestDefaultConfig/#301 (0.05s) + --- PASS: TestDefaultConfig/#285 (0.01s) + --- PASS: TestDefaultConfig/#290 (0.02s) + --- PASS: TestDefaultConfig/#297 (0.03s) + --- PASS: TestDefaultConfig/#307 (0.07s) + --- PASS: TestDefaultConfig/#277 (0.02s) + --- PASS: TestDefaultConfig/#276 (0.02s) + --- PASS: TestDefaultConfig/#274 (0.01s) + --- PASS: TestDefaultConfig/#292 (0.05s) + --- PASS: TestDefaultConfig/#284 (0.03s) + --- PASS: TestDefaultConfig/#293 (0.05s) + --- PASS: TestDefaultConfig/#271 (0.01s) + --- PASS: TestDefaultConfig/#272 (0.01s) + --- PASS: TestDefaultConfig/#286 (0.04s) + --- PASS: TestDefaultConfig/#283 (0.04s) + --- PASS: TestDefaultConfig/#269 (0.01s) + --- PASS: TestDefaultConfig/#275 (0.04s) + --- PASS: TestDefaultConfig/#278 (0.04s) + --- PASS: TestDefaultConfig/#270 (0.02s) + --- PASS: TestDefaultConfig/#268 (0.02s) + --- PASS: TestDefaultConfig/#280 (0.06s) + --- PASS: TestDefaultConfig/#281 (0.06s) + --- PASS: TestDefaultConfig/#279 (0.06s) + --- PASS: TestDefaultConfig/#273 (0.05s) + --- PASS: TestDefaultConfig/#263 (0.02s) + --- PASS: TestDefaultConfig/#267 (0.03s) + --- PASS: TestDefaultConfig/#257 (0.01s) + --- PASS: TestDefaultConfig/#291 (0.08s) + --- PASS: TestDefaultConfig/#262 (0.02s) + --- PASS: TestDefaultConfig/#256 (0.01s) + --- PASS: TestDefaultConfig/#264 (0.03s) + --- PASS: TestDefaultConfig/#259 (0.01s) + --- PASS: TestDefaultConfig/#282 (0.07s) + --- PASS: TestDefaultConfig/#265 (0.03s) + --- PASS: TestDefaultConfig/#258 (0.02s) + --- PASS: TestDefaultConfig/#253 (0.02s) + --- PASS: TestDefaultConfig/#266 (0.05s) + --- PASS: TestDefaultConfig/#252 (0.02s) + --- PASS: TestDefaultConfig/#254 (0.03s) + --- PASS: TestDefaultConfig/#261 (0.06s) + --- PASS: TestDefaultConfig/#255 (0.05s) + --- PASS: TestDefaultConfig/#250 (0.04s) + --- PASS: TestDefaultConfig/#260 (0.06s) + --- PASS: TestDefaultConfig/#246 (0.04s) + --- PASS: TestDefaultConfig/#248 (0.04s) + --- PASS: TestDefaultConfig/#249 (0.04s) + --- PASS: TestDefaultConfig/#431 (0.03s) + --- PASS: TestDefaultConfig/#245 (0.04s) + --- PASS: TestDefaultConfig/#247 (0.05s) + --- PASS: TestDefaultConfig/#114 (0.04s) + --- PASS: TestDefaultConfig/#251 (0.05s) + --- PASS: TestDefaultConfig/#436 (0.03s) + --- PASS: TestDefaultConfig/#244 (0.05s) + --- PASS: TestDefaultConfig/#241 (0.02s) + --- PASS: TestDefaultConfig/#240 (0.02s) + --- PASS: TestDefaultConfig/#432 (0.02s) + --- PASS: TestDefaultConfig/#242 (0.05s) + --- PASS: TestDefaultConfig/#238 (0.03s) + --- PASS: TestDefaultConfig/#435 (0.03s) + --- PASS: TestDefaultConfig/#239 (0.03s) + --- PASS: TestDefaultConfig/#433 (0.03s) + --- PASS: TestDefaultConfig/#419 (0.01s) + --- PASS: TestDefaultConfig/#422 (0.03s) + --- PASS: TestDefaultConfig/#236 (0.03s) + --- PASS: TestDefaultConfig/#434 (0.04s) + --- PASS: TestDefaultConfig/#235 (0.02s) + --- PASS: TestDefaultConfig/#421 (0.02s) + --- PASS: TestDefaultConfig/#234 (0.01s) + --- PASS: TestDefaultConfig/#232 (0.01s) + --- PASS: TestDefaultConfig/#428 (0.01s) + --- PASS: TestDefaultConfig/#233 (0.02s) + --- PASS: TestDefaultConfig/#237 (0.04s) + --- PASS: TestDefaultConfig/#122 (0.09s) + --- PASS: TestDefaultConfig/#231 (0.03s) + --- PASS: TestDefaultConfig/#429 (0.03s) + --- PASS: TestDefaultConfig/#226 (0.02s) + --- PASS: TestDefaultConfig/#444 (0.03s) + --- PASS: TestDefaultConfig/#228 (0.03s) + --- PASS: TestDefaultConfig/#430 (0.03s) + --- PASS: TestDefaultConfig/#448 (0.01s) + --- PASS: TestDefaultConfig/#224 (0.01s) + --- PASS: TestDefaultConfig/#446 (0.01s) + --- PASS: TestDefaultConfig/#447 (0.01s) + --- PASS: TestDefaultConfig/#445 (0.01s) --- PASS: TestDefaultConfig/#229 (0.04s) - --- PASS: TestDefaultConfig/#251 (0.54s) - --- PASS: TestDefaultConfig/#110 (0.08s) - --- PASS: TestDefaultConfig/#120 (0.18s) - --- PASS: TestDefaultConfig/#109 (0.04s) - --- PASS: TestDefaultConfig/#236 (0.21s) - --- PASS: TestDefaultConfig/#117 (0.15s) - --- PASS: TestDefaultConfig/#231 (0.11s) - --- PASS: TestDefaultConfig/#108 (0.03s) - --- PASS: TestDefaultConfig/#228 (0.13s) - --- PASS: TestDefaultConfig/#224 (0.10s) - --- PASS: TestDefaultConfig/#116 (0.14s) - --- PASS: TestDefaultConfig/#220 (0.08s) - --- PASS: TestDefaultConfig/#226 (0.18s) - --- PASS: TestDefaultConfig/#106 (0.09s) - --- PASS: TestDefaultConfig/#107 (0.10s) - --- PASS: TestDefaultConfig/#230 (0.20s) - --- PASS: TestDefaultConfig/#112 (0.22s) - --- PASS: TestDefaultConfig/#225 (0.17s) - --- PASS: TestDefaultConfig/#102 (0.02s) - --- PASS: TestDefaultConfig/#223 (0.12s) + --- PASS: TestDefaultConfig/#223 (0.01s) + --- PASS: TestDefaultConfig/#423 (0.08s) + --- PASS: TestDefaultConfig/#225 (0.07s) + --- PASS: TestDefaultConfig/#218 (0.04s) + --- PASS: TestDefaultConfig/#230 (0.08s) + --- PASS: TestDefaultConfig/#420 (0.11s) + --- PASS: TestDefaultConfig/#222 (0.06s) + --- PASS: TestDefaultConfig/#221 (0.07s) + --- PASS: TestDefaultConfig/#397 (0.01s) + --- PASS: TestDefaultConfig/#219 (0.09s) + --- PASS: TestDefaultConfig/#398 (0.04s) + --- PASS: TestDefaultConfig/#450 (0.12s) + --- PASS: TestDefaultConfig/#212 (0.01s) + --- PASS: TestDefaultConfig/#214 (0.06s) + --- PASS: TestDefaultConfig/#449 (0.13s) + --- PASS: TestDefaultConfig/#210 (0.02s) + --- PASS: TestDefaultConfig/#209 (0.02s) + --- PASS: TestDefaultConfig/#215 (0.07s) + --- PASS: TestDefaultConfig/#396 (0.12s) + --- PASS: TestDefaultConfig/#442 (0.03s) + --- PASS: TestDefaultConfig/#220 (0.14s) + --- PASS: TestDefaultConfig/#443 (0.04s) + --- PASS: TestDefaultConfig/#216 (0.15s) + --- PASS: TestDefaultConfig/#227 (0.20s) + --- PASS: TestDefaultConfig/#399 (0.13s) + --- PASS: TestDefaultConfig/#418 (0.24s) + --- PASS: TestDefaultConfig/#441 (0.08s) + --- PASS: TestDefaultConfig/#395 (0.07s) + --- PASS: TestDefaultConfig/#213 (0.14s) + --- PASS: TestDefaultConfig/#207 (0.08s) + --- PASS: TestDefaultConfig/#211 (0.11s) + --- PASS: TestDefaultConfig/#459 (0.01s) + --- PASS: TestDefaultConfig/#217 (0.20s) + --- PASS: TestDefaultConfig/#465 (0.02s) + --- PASS: TestDefaultConfig/#206 (0.07s) + --- PASS: TestDefaultConfig/#200 (0.03s) + --- PASS: TestDefaultConfig/#464 (0.02s) + --- PASS: TestDefaultConfig/#201 (0.04s) + --- PASS: TestDefaultConfig/#202 (0.08s) + --- PASS: TestDefaultConfig/#440 (0.09s) + --- PASS: TestDefaultConfig/#198 (0.04s) + --- PASS: TestDefaultConfig/#400 (0.26s) + --- PASS: TestDefaultConfig/#199 (0.07s) + --- PASS: TestDefaultConfig/#439 (0.10s) + --- PASS: TestDefaultConfig/#461 (0.04s) + --- PASS: TestDefaultConfig/#197 (0.07s) + --- PASS: TestDefaultConfig/#205 (0.16s) + --- PASS: TestDefaultConfig/#394 (0.16s) + --- PASS: TestDefaultConfig/#157 (0.01s) + --- PASS: TestDefaultConfig/#195 (0.05s) + --- PASS: TestDefaultConfig/#460 (0.06s) + --- PASS: TestDefaultConfig/#463 (0.10s) + --- PASS: TestDefaultConfig/#191 (0.01s) + --- PASS: TestDefaultConfig/#156 (0.02s) + --- PASS: TestDefaultConfig/#204 (0.16s) + --- PASS: TestDefaultConfig/#203 (0.19s) + --- PASS: TestDefaultConfig/#189 (0.06s) + --- PASS: TestDefaultConfig/#208 (0.24s) + --- PASS: TestDefaultConfig/#190 (0.06s) + --- PASS: TestDefaultConfig/#154 (0.05s) + --- PASS: TestDefaultConfig/#196 (0.14s) + --- PASS: TestDefaultConfig/#194 (0.12s) + --- PASS: TestDefaultConfig/#187 (0.07s) + --- PASS: TestDefaultConfig/#186 (0.07s) + --- PASS: TestDefaultConfig/#185 (0.04s) + --- PASS: TestDefaultConfig/#155 (0.09s) + --- PASS: TestDefaultConfig/#192 (0.13s) + --- PASS: TestDefaultConfig/#150 (0.04s) + --- PASS: TestDefaultConfig/#152 (0.05s) + --- PASS: TestDefaultConfig/#182 (0.03s) + --- PASS: TestDefaultConfig/#188 (0.09s) + --- PASS: TestDefaultConfig/#149 (0.03s) + --- PASS: TestDefaultConfig/#184 (0.09s) + --- PASS: TestDefaultConfig/#179 (0.05s) + --- PASS: TestDefaultConfig/#153 (0.14s) + --- PASS: TestDefaultConfig/#180 (0.09s) + --- PASS: TestDefaultConfig/#193 (0.22s) + --- PASS: TestDefaultConfig/#183 (0.13s) + --- PASS: TestDefaultConfig/#178 (0.10s) + --- PASS: TestDefaultConfig/#144 (0.05s) + --- PASS: TestDefaultConfig/#181 (0.11s) + --- PASS: TestDefaultConfig/#174 (0.01s) + --- PASS: TestDefaultConfig/#393 (0.39s) + --- PASS: TestDefaultConfig/#145 (0.06s) + --- PASS: TestDefaultConfig/#143 (0.02s) + --- PASS: TestDefaultConfig/#176 (0.10s) + --- PASS: TestDefaultConfig/#175 (0.05s) + --- PASS: TestDefaultConfig/#142 (0.03s) + --- PASS: TestDefaultConfig/#147 (0.12s) + --- PASS: TestDefaultConfig/#172 (0.03s) + --- PASS: TestDefaultConfig/#462 (0.32s) + --- PASS: TestDefaultConfig/#177 (0.15s) + --- PASS: TestDefaultConfig/#171 (0.05s) + --- PASS: TestDefaultConfig/#170 (0.08s) + --- PASS: TestDefaultConfig/#168 (0.07s) + --- PASS: TestDefaultConfig/#167 (0.05s) + --- PASS: TestDefaultConfig/#169 (0.08s) + --- PASS: TestDefaultConfig/#138 (0.09s) + --- PASS: TestDefaultConfig/#165 (0.01s) + --- PASS: TestDefaultConfig/#148 (0.22s) + --- PASS: TestDefaultConfig/#140 (0.10s) + --- PASS: TestDefaultConfig/#134 (0.02s) + --- PASS: TestDefaultConfig/#137 (0.09s) + --- PASS: TestDefaultConfig/#133 (0.02s) + --- PASS: TestDefaultConfig/#151 (0.25s) + --- PASS: TestDefaultConfig/#139 (0.11s) + --- PASS: TestDefaultConfig/#132 (0.03s) + --- PASS: TestDefaultConfig/#173 (0.14s) + --- PASS: TestDefaultConfig/#131 (0.03s) + --- PASS: TestDefaultConfig/#163 (0.04s) + --- PASS: TestDefaultConfig/#141 (0.16s) + --- PASS: TestDefaultConfig/#129 (0.06s) + --- PASS: TestDefaultConfig/#164 (0.08s) + --- PASS: TestDefaultConfig/#166 (0.12s) + --- PASS: TestDefaultConfig/#126 (0.03s) + --- PASS: TestDefaultConfig/#127 (0.03s) + --- PASS: TestDefaultConfig/#136 (0.13s) + --- PASS: TestDefaultConfig/#407 (0.02s) + --- PASS: TestDefaultConfig/#409 (0.01s) + --- PASS: TestDefaultConfig/#146 (0.28s) + --- PASS: TestDefaultConfig/#130 (0.08s) + --- PASS: TestDefaultConfig/#410 (0.02s) + --- PASS: TestDefaultConfig/#135 (0.15s) + --- PASS: TestDefaultConfig/#128 (0.09s) + --- PASS: TestDefaultConfig/#411 (0.04s) + --- PASS: TestDefaultConfig/#158 (0.08s) + --- PASS: TestDefaultConfig/#408 (0.04s) + --- PASS: TestDefaultConfig/#162 (0.12s) + --- PASS: TestDefaultConfig/#161 (0.12s) + --- PASS: TestDefaultConfig/#124 (0.04s) + --- PASS: TestDefaultConfig/#474 (0.01s) + --- PASS: TestDefaultConfig/#160 (0.12s) + --- PASS: TestDefaultConfig/#391 (0.01s) + --- PASS: TestDefaultConfig/#417 (0.05s) + --- PASS: TestDefaultConfig/#476 (0.01s) + --- PASS: TestDefaultConfig/#159 (0.11s) + --- PASS: TestDefaultConfig/#416 (0.05s) + --- PASS: TestDefaultConfig/#475 (0.01s) + --- PASS: TestDefaultConfig/#481 (0.02s) + --- PASS: TestDefaultConfig/#477 (0.03s) + --- PASS: TestDefaultConfig/#123 (0.07s) + --- PASS: TestDefaultConfig/#478 (0.03s) + --- PASS: TestDefaultConfig/#392 (0.07s) + --- PASS: TestDefaultConfig/#479 (0.03s) + --- PASS: TestDefaultConfig/#406 (0.06s) + --- PASS: TestDefaultConfig/#480 (0.04s) + --- PASS: TestDefaultConfig/#415 (0.09s) + --- PASS: TestDefaultConfig/#403 (0.06s) + --- PASS: TestDefaultConfig/#90 (0.02s) + --- PASS: TestDefaultConfig/#91 (0.03s) + --- PASS: TestDefaultConfig/#404 (0.07s) + --- PASS: TestDefaultConfig/#413 (0.04s) + --- PASS: TestDefaultConfig/#125 (0.11s) + --- PASS: TestDefaultConfig/#86 (0.01s) + --- PASS: TestDefaultConfig/#84 (0.01s) + --- PASS: TestDefaultConfig/#85 (0.02s) + --- PASS: TestDefaultConfig/#405 (0.08s) + --- PASS: TestDefaultConfig/#414 (0.08s) + --- PASS: TestDefaultConfig/#83 (0.01s) + --- PASS: TestDefaultConfig/#117 (0.03s) + --- PASS: TestDefaultConfig/#81 (0.02s) + --- PASS: TestDefaultConfig/#82 (0.03s) + --- PASS: TestDefaultConfig/#87 (0.05s) + --- PASS: TestDefaultConfig/#89 (0.07s) + --- PASS: TestDefaultConfig/#80 (0.04s) + --- PASS: TestDefaultConfig/#112 (0.04s) + --- PASS: TestDefaultConfig/#118 (0.04s) + --- PASS: TestDefaultConfig/#79 (0.02s) + --- PASS: TestDefaultConfig/#109 (0.01s) + --- PASS: TestDefaultConfig/#61 (0.09s) + --- PASS: TestDefaultConfig/#115 (0.05s) + --- PASS: TestDefaultConfig/#77 (0.01s) + --- PASS: TestDefaultConfig/#116 (0.05s) + --- PASS: TestDefaultConfig/#76 (0.01s) + --- PASS: TestDefaultConfig/#74 (0.01s) + --- PASS: TestDefaultConfig/#73 (0.02s) + --- PASS: TestDefaultConfig/#108 (0.02s) + --- PASS: TestDefaultConfig/#113 (0.07s) + --- PASS: TestDefaultConfig/#107 (0.03s) + --- PASS: TestDefaultConfig/#75 (0.03s) + --- PASS: TestDefaultConfig/#105 (0.03s) + --- PASS: TestDefaultConfig/#120 (0.11s) + --- PASS: TestDefaultConfig/#71 (0.02s) + --- PASS: TestDefaultConfig/#72 (0.04s) + --- PASS: TestDefaultConfig/#106 (0.04s) + --- PASS: TestDefaultConfig/#119 (0.11s) + --- PASS: TestDefaultConfig/#88 (0.11s) + --- PASS: TestDefaultConfig/#104 (0.04s) + --- PASS: TestDefaultConfig/#78 (0.07s) + --- PASS: TestDefaultConfig/#70 (0.03s) --- PASS: TestDefaultConfig/#103 (0.03s) - --- PASS: TestDefaultConfig/#100 (0.04s) - --- PASS: TestDefaultConfig/#233 (0.35s) + --- PASS: TestDefaultConfig/#69 (0.02s) + --- PASS: TestDefaultConfig/#121 (0.13s) + --- PASS: TestDefaultConfig/#68 (0.01s) + --- PASS: TestDefaultConfig/#111 (0.08s) + --- PASS: TestDefaultConfig/#102 (0.04s) + --- PASS: TestDefaultConfig/#101 (0.03s) + --- PASS: TestDefaultConfig/#62 (0.02s) + --- PASS: TestDefaultConfig/#96 (0.03s) + --- PASS: TestDefaultConfig/#63 (0.03s) + --- PASS: TestDefaultConfig/#67 (0.04s) + --- PASS: TestDefaultConfig/#65 (0.04s) + --- PASS: TestDefaultConfig/#66 (0.04s) + --- PASS: TestDefaultConfig/#110 (0.11s) + --- PASS: TestDefaultConfig/#99 (0.03s) + --- PASS: TestDefaultConfig/#93 (0.02s) + --- PASS: TestDefaultConfig/#97 (0.04s) + --- PASS: TestDefaultConfig/#100 (0.05s) + --- PASS: TestDefaultConfig/#95 (0.04s) + --- PASS: TestDefaultConfig/#55 (0.01s) + --- PASS: TestDefaultConfig/#59 (0.02s) + --- PASS: TestDefaultConfig/#64 (0.05s) + --- PASS: TestDefaultConfig/#57 (0.02s) + --- PASS: TestDefaultConfig/#42 (0.01s) + --- PASS: TestDefaultConfig/#43 (0.01s) + --- PASS: TestDefaultConfig/#56 (0.02s) + --- PASS: TestDefaultConfig/#53 (0.02s) + --- PASS: TestDefaultConfig/#54 (0.03s) --- PASS: TestDefaultConfig/#98 (0.07s) - --- PASS: TestDefaultConfig/#221 (0.18s) - --- PASS: TestDefaultConfig/#101 (0.10s) - --- PASS: TestDefaultConfig/#227 (0.29s) - --- PASS: TestDefaultConfig/#105 (0.15s) - --- PASS: TestDefaultConfig/#216 (0.09s) - --- PASS: TestDefaultConfig/#214 (0.07s) - --- PASS: TestDefaultConfig/#104 (0.16s) - --- PASS: TestDefaultConfig/#211 (0.02s) - --- PASS: TestDefaultConfig/#219 (0.12s) - --- PASS: TestDefaultConfig/#99 (0.13s) - --- PASS: TestDefaultConfig/#209 (0.05s) - --- PASS: TestDefaultConfig/#215 (0.14s) - --- PASS: TestDefaultConfig/#208 (0.09s) - --- PASS: TestDefaultConfig/#165 (0.08s) - --- PASS: TestDefaultConfig/#96 (0.08s) - --- PASS: TestDefaultConfig/#213 (0.13s) - --- PASS: TestDefaultConfig/#97 (0.09s) - --- PASS: TestDefaultConfig/#162 (0.02s) - --- PASS: TestDefaultConfig/#207 (0.10s) - --- PASS: TestDefaultConfig/#204 (0.02s) - --- PASS: TestDefaultConfig/#94 (0.06s) - --- PASS: TestDefaultConfig/#95 (0.13s) - --- PASS: TestDefaultConfig/#210 (0.19s) - --- PASS: TestDefaultConfig/#218 (0.28s) - --- PASS: TestDefaultConfig/#217 (0.28s) - --- PASS: TestDefaultConfig/#205 (0.14s) - --- PASS: TestDefaultConfig/#212 (0.21s) - --- PASS: TestDefaultConfig/#161 (0.11s) - --- PASS: TestDefaultConfig/#206 (0.19s) - --- PASS: TestDefaultConfig/#201 (0.01s) - --- PASS: TestDefaultConfig/#202 (0.03s) - --- PASS: TestDefaultConfig/#157 (0.02s) - --- PASS: TestDefaultConfig/#158 (0.02s) - --- PASS: TestDefaultConfig/#164 (0.20s) - --- PASS: TestDefaultConfig/#159 (0.07s) - --- PASS: TestDefaultConfig/#160 (0.19s) - --- PASS: TestDefaultConfig/#89 (0.06s) - --- PASS: TestDefaultConfig/#91 (0.12s) - --- PASS: TestDefaultConfig/#199 (0.10s) - --- PASS: TestDefaultConfig/#200 (0.11s) - --- PASS: TestDefaultConfig/#155 (0.10s) - --- PASS: TestDefaultConfig/#197 (0.02s) - --- PASS: TestDefaultConfig/#222 (0.52s) - --- PASS: TestDefaultConfig/#198 (0.11s) - --- PASS: TestDefaultConfig/#87 (0.02s) - --- PASS: TestDefaultConfig/#163 (0.27s) - --- PASS: TestDefaultConfig/#90 (0.16s) - --- PASS: TestDefaultConfig/#88 (0.08s) - --- PASS: TestDefaultConfig/#154 (0.15s) - --- PASS: TestDefaultConfig/#149 (0.09s) - --- PASS: TestDefaultConfig/#203 (0.30s) - --- PASS: TestDefaultConfig/#93 (0.30s) - --- PASS: TestDefaultConfig/#193 (0.02s) - --- PASS: TestDefaultConfig/#148 (0.07s) - --- PASS: TestDefaultConfig/#150 (0.11s) - --- PASS: TestDefaultConfig/#153 (0.12s) - --- PASS: TestDefaultConfig/#194 (0.03s) - --- PASS: TestDefaultConfig/#196 (0.13s) - --- PASS: TestDefaultConfig/#195 (0.11s) - --- PASS: TestDefaultConfig/#147 (0.10s) - --- PASS: TestDefaultConfig/#156 (0.28s) - --- PASS: TestDefaultConfig/#191 (0.07s) - --- PASS: TestDefaultConfig/#92 (0.35s) - --- PASS: TestDefaultConfig/#146 (0.06s) - --- PASS: TestDefaultConfig/#142 (0.08s) - --- PASS: TestDefaultConfig/#151 (0.20s) - --- PASS: TestDefaultConfig/#185 (0.02s) - --- PASS: TestDefaultConfig/#144 (0.11s) - --- PASS: TestDefaultConfig/#152 (0.21s) - --- PASS: TestDefaultConfig/#188 (0.09s) - --- PASS: TestDefaultConfig/#184 (0.02s) - --- PASS: TestDefaultConfig/#180 (0.01s) - --- PASS: TestDefaultConfig/#182 (0.04s) - --- PASS: TestDefaultConfig/#183 (0.04s) - --- PASS: TestDefaultConfig/#145 (0.15s) - --- PASS: TestDefaultConfig/#143 (0.16s) - --- PASS: TestDefaultConfig/#176 (0.08s) - --- PASS: TestDefaultConfig/#192 (0.18s) - --- PASS: TestDefaultConfig/#178 (0.09s) - --- PASS: TestDefaultConfig/#173 (0.05s) - --- PASS: TestDefaultConfig/#172 (0.05s) - --- PASS: TestDefaultConfig/#170 (0.02s) - --- PASS: TestDefaultConfig/#171 (0.03s) - --- PASS: TestDefaultConfig/#174 (0.10s) - --- PASS: TestDefaultConfig/#168 (0.05s) - --- PASS: TestDefaultConfig/#169 (0.10s) - --- PASS: TestDefaultConfig/#175 (0.17s) - --- PASS: TestDefaultConfig/#494 (0.09s) - --- PASS: TestDefaultConfig/#187 (0.26s) - --- PASS: TestDefaultConfig/#179 (0.20s) - --- PASS: TestDefaultConfig/#490 (0.02s) - --- PASS: TestDefaultConfig/#493 (0.07s) - --- PASS: TestDefaultConfig/#189 (0.29s) - --- PASS: TestDefaultConfig/#491 (0.03s) - --- PASS: TestDefaultConfig/#488 (0.11s) - --- PASS: TestDefaultConfig/#420 (0.05s) - --- PASS: TestDefaultConfig/#492 (0.13s) - --- PASS: TestDefaultConfig/#416 (0.10s) - --- PASS: TestDefaultConfig/#177 (0.29s) - --- PASS: TestDefaultConfig/#167 (0.21s) - --- PASS: TestDefaultConfig/#166 (0.21s) - --- PASS: TestDefaultConfig/#190 (0.40s) - --- PASS: TestDefaultConfig/#59 (0.06s) - --- PASS: TestDefaultConfig/#418 (0.12s) - --- PASS: TestDefaultConfig/#58 (0.06s) - --- PASS: TestDefaultConfig/#186 (0.31s) - --- PASS: TestDefaultConfig/#419 (0.12s) - --- PASS: TestDefaultConfig/#489 (0.16s) - --- PASS: TestDefaultConfig/#55 (0.06s) - --- PASS: TestDefaultConfig/#56 (0.10s) - --- PASS: TestDefaultConfig/#30 (0.20s) - --- PASS: TestDefaultConfig/#181 (0.42s) - --- PASS: TestDefaultConfig/#46 (0.05s) - --- PASS: TestDefaultConfig/#57 (0.12s) - --- PASS: TestDefaultConfig/#50 (0.10s) - --- PASS: TestDefaultConfig/#48 (0.10s) - --- PASS: TestDefaultConfig/#44 (0.02s) - --- PASS: TestDefaultConfig/#45 (0.03s) - --- PASS: TestDefaultConfig/#417 (0.23s) - --- PASS: TestDefaultConfig/#52 (0.12s) - --- PASS: TestDefaultConfig/#42 (0.05s) - --- PASS: TestDefaultConfig/#41 (0.05s) - --- PASS: TestDefaultConfig/#43 (0.05s) - --- PASS: TestDefaultConfig/#54 (0.21s) - --- PASS: TestDefaultConfig/#37 (0.08s) - --- PASS: TestDefaultConfig/#40 (0.09s) - --- PASS: TestDefaultConfig/#49 (0.19s) - --- PASS: TestDefaultConfig/#39 (0.10s) - --- PASS: TestDefaultConfig/#51 (0.21s) - --- PASS: TestDefaultConfig/#53 (0.21s) - --- PASS: TestDefaultConfig/#31 (0.02s) - --- PASS: TestDefaultConfig/#35 (0.10s) - --- PASS: TestDefaultConfig/#421 (0.34s) - --- PASS: TestDefaultConfig/#487 (0.02s) - --- PASS: TestDefaultConfig/#485 (0.06s) - --- PASS: TestDefaultConfig/#82 (0.08s) - --- PASS: TestDefaultConfig/#80 (0.08s) - --- PASS: TestDefaultConfig/#33 (0.16s) - --- PASS: TestDefaultConfig/#47 (0.27s) - --- PASS: TestDefaultConfig/#34 (0.16s) - --- PASS: TestDefaultConfig/#85 (0.10s) - --- PASS: TestDefaultConfig/#79 (0.06s) - --- PASS: TestDefaultConfig/#78 (0.03s) - --- PASS: TestDefaultConfig/#486 (0.12s) - --- PASS: TestDefaultConfig/#75 (0.04s) - --- PASS: TestDefaultConfig/#76 (0.04s) - --- PASS: TestDefaultConfig/#74 (0.08s) - --- PASS: TestDefaultConfig/#32 (0.24s) - --- PASS: TestDefaultConfig/#137 (0.04s) - --- PASS: TestDefaultConfig/#73 (0.18s) - --- PASS: TestDefaultConfig/#38 (0.28s) - --- PASS: TestDefaultConfig/#83 (0.19s) - --- PASS: TestDefaultConfig/#140 (0.08s) - --- PASS: TestDefaultConfig/#139 (0.09s) - --- PASS: TestDefaultConfig/#133 (0.01s) - --- PASS: TestDefaultConfig/#81 (0.19s) - --- PASS: TestDefaultConfig/#77 (0.11s) - --- PASS: TestDefaultConfig/#36 (0.29s) - --- PASS: TestDefaultConfig/#135 (0.05s) - --- PASS: TestDefaultConfig/#84 (0.25s) - --- PASS: TestDefaultConfig/#71 (0.07s) - --- PASS: TestDefaultConfig/#483 (0.09s) - --- PASS: TestDefaultConfig/#134 (0.09s) - --- PASS: TestDefaultConfig/#72 (0.09s) - --- PASS: TestDefaultConfig/#132 (0.19s) - --- PASS: TestDefaultConfig/#138 (0.19s) + --- PASS: TestDefaultConfig/#50 (0.01s) + --- PASS: TestDefaultConfig/#39 (0.02s) + --- PASS: TestDefaultConfig/#52 (0.02s) + --- PASS: TestDefaultConfig/#92 (0.01s) + --- PASS: TestDefaultConfig/#58 (0.05s) + --- PASS: TestDefaultConfig/#49 (0.02s) + --- PASS: TestDefaultConfig/#51 (0.03s) + --- PASS: TestDefaultConfig/#35 (0.02s) + --- PASS: TestDefaultConfig/#40 (0.04s) + --- PASS: TestDefaultConfig/#47 (0.02s) + --- PASS: TestDefaultConfig/#31 (0.07s) + --- PASS: TestDefaultConfig/#33 (0.01s) + --- PASS: TestDefaultConfig/#94 (0.09s) + --- PASS: TestDefaultConfig/#46 (0.02s) + --- PASS: TestDefaultConfig/#37 (0.04s) + --- PASS: TestDefaultConfig/#32 (0.03s) + --- PASS: TestDefaultConfig/#48 (0.05s) + --- PASS: TestDefaultConfig/#44 (0.03s) + --- PASS: TestDefaultConfig/#426 (0.04s) + --- PASS: TestDefaultConfig/#36 (0.06s) + --- PASS: TestDefaultConfig/#38 (0.08s) + --- PASS: TestDefaultConfig/#60 (0.11s) + --- PASS: TestDefaultConfig/#41 (0.09s) + --- PASS: TestDefaultConfig/#438 (0.05s) + --- PASS: TestDefaultConfig/#470 (0.04s) + --- PASS: TestDefaultConfig/#29 (0.03s) + --- PASS: TestDefaultConfig/#45 (0.06s) + --- PASS: TestDefaultConfig/#34 (0.07s) + --- PASS: TestDefaultConfig/#458 (0.01s) + --- PASS: TestDefaultConfig/#427 (0.06s) + --- PASS: TestDefaultConfig/#24 (0.01s) + --- PASS: TestDefaultConfig/#26 (0.02s) + --- PASS: TestDefaultConfig/#28 (0.04s) + --- PASS: TestDefaultConfig/#456 (0.01s) --- PASS: TestDefaultConfig/#27 (0.03s) - --- PASS: TestDefaultConfig/#484 (0.11s) - --- PASS: TestDefaultConfig/#29 (0.08s) - --- PASS: TestDefaultConfig/#28 (0.06s) - --- PASS: TestDefaultConfig/#68 (0.12s) - --- PASS: TestDefaultConfig/#26 (0.08s) - --- PASS: TestDefaultConfig/#67 (0.17s) - --- PASS: TestDefaultConfig/#17 (0.06s) - --- PASS: TestDefaultConfig/#15 (0.18s) - --- PASS: TestDefaultConfig/#24 (0.10s) - --- PASS: TestDefaultConfig/#23 (0.10s) - --- PASS: TestDefaultConfig/#16 (0.03s) - --- PASS: TestDefaultConfig/#25 (0.12s) - --- PASS: TestDefaultConfig/#18 (0.08s) - --- PASS: TestDefaultConfig/#07 (0.06s) - --- PASS: TestDefaultConfig/#69 (0.25s) - --- PASS: TestDefaultConfig/#13 (0.09s) - --- PASS: TestDefaultConfig/#19 (0.16s) - --- PASS: TestDefaultConfig/#12 (0.10s) - --- PASS: TestDefaultConfig/#127 (0.06s) - --- PASS: TestDefaultConfig/#70 (0.29s) - --- PASS: TestDefaultConfig/#14 (0.11s) - --- PASS: TestDefaultConfig/#22 (0.19s) - --- PASS: TestDefaultConfig/#130 (0.02s) - --- PASS: TestDefaultConfig/#136 (0.38s) - --- PASS: TestDefaultConfig/#20 (0.23s) - --- PASS: TestDefaultConfig/#10 (0.14s) - --- PASS: TestDefaultConfig/#11 (0.14s) - --- PASS: TestDefaultConfig/#21 (0.28s) - --- PASS: TestDefaultConfig/#65 (0.09s) - --- PASS: TestDefaultConfig/#128 (0.10s) - --- PASS: TestDefaultConfig/#09 (0.21s) - --- PASS: TestDefaultConfig/#05 (0.08s) - --- PASS: TestDefaultConfig/#06 (0.11s) - --- PASS: TestDefaultConfig/#414 (0.02s) - --- PASS: TestDefaultConfig/#413 (0.04s) - --- PASS: TestDefaultConfig/#66 (0.13s) - --- PASS: TestDefaultConfig/#131 (0.12s) - --- PASS: TestDefaultConfig/#129 (0.22s) - --- PASS: TestDefaultConfig/#124 (0.10s) - --- PASS: TestDefaultConfig/#126 (0.10s) - --- PASS: TestDefaultConfig/#08 (0.31s) - --- PASS: TestDefaultConfig/#02 (0.18s) - --- PASS: TestDefaultConfig/#482 (0.02s) - --- PASS: TestDefaultConfig/#415 (0.15s) - --- PASS: TestDefaultConfig/#04 (0.20s) - --- PASS: TestDefaultConfig/#411 (0.12s) - --- PASS: TestDefaultConfig/#442 (0.03s) - --- PASS: TestDefaultConfig/#446 (0.06s) - --- PASS: TestDefaultConfig/#448 (0.10s) - --- PASS: TestDefaultConfig/#64 (0.33s) - --- PASS: TestDefaultConfig/#443 (0.08s) - --- PASS: TestDefaultConfig/#125 (0.21s) - --- PASS: TestDefaultConfig/#62 (0.21s) - --- PASS: TestDefaultConfig/#438 (0.09s) - --- PASS: TestDefaultConfig/#445 (0.10s) - --- PASS: TestDefaultConfig/#412 (0.21s) - --- PASS: TestDefaultConfig/#01 (0.29s) - --- PASS: TestDefaultConfig/#03 (0.34s) - --- PASS: TestDefaultConfig/#61 (0.02s) - --- PASS: TestDefaultConfig/#123 (0.03s) - --- PASS: TestDefaultConfig/#86 (0.04s) - --- PASS: TestDefaultConfig/#410 (0.05s) - --- PASS: TestDefaultConfig/#447 (0.16s) - --- PASS: TestDefaultConfig/#436 (0.06s) - --- PASS: TestDefaultConfig/#444 (0.17s) - --- PASS: TestDefaultConfig/#439 (0.08s) - --- PASS: TestDefaultConfig/#440 (0.12s) - --- PASS: TestDefaultConfig/#441 (0.17s) - --- PASS: TestDefaultConfig/#63 (0.29s) - --- PASS: TestDefaultConfig/#141 (0.08s) - --- PASS: TestDefaultConfig/#437 (0.08s) + --- PASS: TestDefaultConfig/#457 (0.03s) + --- PASS: TestDefaultConfig/#455 (0.02s) + --- PASS: TestDefaultConfig/#20 (0.02s) + --- PASS: TestDefaultConfig/#473 (0.08s) + --- PASS: TestDefaultConfig/#23 (0.03s) + --- PASS: TestDefaultConfig/#14 (0.01s) + --- PASS: TestDefaultConfig/#17 (0.01s) + --- PASS: TestDefaultConfig/#471 (0.05s) + --- PASS: TestDefaultConfig/#13 (0.01s) + --- PASS: TestDefaultConfig/#21 (0.03s) + --- PASS: TestDefaultConfig/#12 (0.01s) + --- PASS: TestDefaultConfig/#15 (0.02s) + --- PASS: TestDefaultConfig/#472 (0.06s) + --- PASS: TestDefaultConfig/#11 (0.01s) + --- PASS: TestDefaultConfig/#469 (0.01s) + --- PASS: TestDefaultConfig/#30 (0.10s) + --- PASS: TestDefaultConfig/#468 (0.03s) + --- PASS: TestDefaultConfig/#25 (0.06s) + --- PASS: TestDefaultConfig/#10 (0.03s) + --- PASS: TestDefaultConfig/#09 (0.03s) + --- PASS: TestDefaultConfig/#08 (0.06s) + --- PASS: TestDefaultConfig/#19 (0.06s) + --- PASS: TestDefaultConfig/#02 (0.01s) + --- PASS: TestDefaultConfig/#05 (0.01s) + --- PASS: TestDefaultConfig/#452 (0.01s) + --- PASS: TestDefaultConfig/#453 (0.04s) + --- PASS: TestDefaultConfig/#18 (0.06s) + --- PASS: TestDefaultConfig/#425 (0.04s) + --- PASS: TestDefaultConfig/#22 (0.08s) + --- PASS: TestDefaultConfig/#03 (0.02s) + --- PASS: TestDefaultConfig/#16 (0.13s) + --- PASS: TestDefaultConfig/#402 (0.05s) + --- PASS: TestDefaultConfig/#06 (0.03s) + --- PASS: TestDefaultConfig/#04 (0.05s) + --- PASS: TestDefaultConfig/#01 (0.01s) + --- PASS: TestDefaultConfig/#07 (0.03s) + --- PASS: TestDefaultConfig/#454 (0.05s) + --- PASS: TestDefaultConfig/#467 (0.05s) === RUN TestTxnEndpoint_Bad_JSON === PAUSE TestTxnEndpoint_Bad_JSON === RUN TestTxnEndpoint_Bad_Size_Item @@ -21947,20 +21994,20 @@ util_test.go:230: DM-skipped --- SKIP: TestForwardSignals (0.00s) === RUN TestMakeWatchHandler -2021-10-31T23:13:04.681Z [DEBUG] TestMakeWatchHandler: watch handler output: watch_handler="bash -c 'echo $CONSUL_INDEX >> handler_index_out && cat >> handler_out'" output= ---- PASS: TestMakeWatchHandler (0.04s) +2022-12-04T05:49:14.173Z [DEBUG] TestMakeWatchHandler: watch handler output: watch_handler="bash -c 'echo $CONSUL_INDEX >> handler_index_out && cat >> handler_out'" output= +--- PASS: TestMakeWatchHandler (0.00s) === RUN TestMakeHTTPWatchHandler -2021-10-31T23:13:04.686Z [TRACE] TestMakeHTTPWatchHandler: http watch handler output: watch=http://127.0.0.1:38551 output="Ok, i see" +2022-12-04T05:49:14.174Z [TRACE] TestMakeHTTPWatchHandler: http watch handler output: watch=http://127.0.0.1:34957 output="Ok, i see" --- PASS: TestMakeHTTPWatchHandler (0.00s) === RUN TestMakeWatchPlan === RUN TestMakeWatchPlan/handler_type_script,_with_deprecated_handler_field -2021-10-31T23:13:04.686Z [WARN] The 'handler' field in watches has been deprecated and replaced with the 'args' field. See https://www.consul.io/docs/agent/watches.html +2022-12-04T05:49:14.174Z [WARN] The 'handler' field in watches has been deprecated and replaced with the 'args' field. See https://www.consul.io/docs/agent/watches.html === RUN TestMakeWatchPlan/handler_type_script,_with_single_arg === RUN TestMakeWatchPlan/handler_type_script,_with_multiple_args_from_slice_of_interface === RUN TestMakeWatchPlan/handler_type_script,_with_multiple_args_from_slice_of_strings === RUN TestMakeWatchPlan/handler_type_script,_with_not_string_args === RUN TestMakeWatchPlan/conflicting_handler -2021-10-31T23:13:04.687Z [WARN] The 'handler' field in watches has been deprecated and replaced with the 'args' field. See https://www.consul.io/docs/agent/watches.html +2022-12-04T05:49:14.175Z [WARN] The 'handler' field in watches has been deprecated and replaced with the 'args' field. See https://www.consul.io/docs/agent/watches.html === RUN TestMakeWatchPlan/no_handler_type --- PASS: TestMakeWatchPlan (0.00s) --- PASS: TestMakeWatchPlan/handler_type_script,_with_deprecated_handler_field (0.00s) @@ -21970,29 +22017,28 @@ --- PASS: TestMakeWatchPlan/handler_type_script,_with_not_string_args (0.00s) --- PASS: TestMakeWatchPlan/conflicting_handler (0.00s) --- PASS: TestMakeWatchPlan/no_handler_type (0.00s) +=== CONT TestSessionCreate_Delete === CONT TestACL_Legacy_Disabled_Response +=== CONT TestSessionCreate +=== CONT TestRemoteExecGetSpec_ACLDeny === CONT TestSetFilePermissions +=== CONT TestServiceManager_PersistService_ConfigFiles +=== CONT TestHandleRemoteExecFailed +=== CONT TestRemoteExecGetSpec +=== CONT TestOperator_AutopilotCASConfiguration +=== CONT TestPreparedQuery_Integration --- PASS: TestSetFilePermissions (0.00s) === CONT TestStringHash +=== CONT TestPreparedQuery_parseLimit --- PASS: TestStringHash (0.00s) === CONT TestUserEventToken -=== CONT TestTxnEndpoint_Bad_Size_Net -=== RUN TestTxnEndpoint_Bad_Size_Net/exceeds_default_limits -=== CONT TestServiceManager_PersistService_ConfigFiles -=== CONT TestPreparedQuery_Get -=== RUN TestPreparedQuery_Get/#00 -=== CONT TestPreparedQuery_Explain -=== RUN TestPreparedQuery_Explain/#00 -=== CONT TestPreparedQuery_ExecuteCached -=== CONT TestOperator_KeyringRemove -=== CONT TestKVSEndpoint_Recurse -=== CONT TestIntentionsCheck_noSource -=== CONT TestIntentionsCheck_basic -=== CONT TestKVSEndpoint_PUT_GET_DELETE -=== CONT TestParseConsistency_Invalid -=== CONT TestIntentionsMatch_noName -=== CONT TestIntentionsMatch_byInvalid -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +--- PASS: TestPreparedQuery_parseLimit (0.00s) +=== CONT TestOperator_AutopilotGetConfiguration +=== CONT TestOperator_AutopilotSetConfiguration +=== CONT TestRemoteExecGetSpec_ACLToken +=== CONT TestRemoteExecGetSpec_ACLAgentToken +=== CONT TestPreparedQuery_Delete +=== CONT TestSessionCreate_NodeChecks [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestACL_Legacy_Disabled_Response/0 === RUN TestACL_Legacy_Disabled_Response/1 @@ -22001,781 +22047,6694 @@ === RUN TestACL_Legacy_Disabled_Response/4 === RUN TestACL_Legacy_Disabled_Response/5 [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:04.905Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:04.936Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:04.956 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:69464632-1085-b205-9f3a-db3705889edb Address:127.0.0.1:16225}]" -13:04.958 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-69464632-1085-b205-9f3a-db3705889edb.dc1 127.0.0.1 -13:04.959 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-69464632-1085-b205-9f3a-db3705889edb 127.0.0.1 -2021-10-31T23:13:04.959Z [INFO] agent.router: Initializing LAN area manager -13:04.959 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16220 network=udp -13:04.959 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16225 [Follower]" leader= -13:04.967 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-69464632-1085-b205-9f3a-db3705889edb.dc1 area=wan -13:04.973 [INFO] TestAgent.server: Adding LAN server: server="Node-69464632-1085-b205-9f3a-db3705889edb (Addr: tcp/127.0.0.1:16225) (DC: dc1)" -13:04.974 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16220 network=tcp -13:04.982 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16221 network=tcp -13:04.982 [INFO] TestAgent: started state syncer -13:05.005 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.005 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16225 [Candidate]" term=2 -13:05.014 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.014 [DEBUG] TestAgent.server.raft: vote granted: from=69464632-1085-b205-9f3a-db3705889edb term=2 tally=1 -13:05.014 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.014 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16225 [Leader]" -13:05.019 [INFO] TestAgent.server: cluster leadership acquired -13:05.019 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16225 -13:05.020 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.021 [INFO] TestAgent.server: New leader elected: payload=Node-69464632-1085-b205-9f3a-db3705889edb -13:05.022 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.022 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.022 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.022 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.022 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.022 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-69464632-1085-b205-9f3a-db3705889edb -13:05.022 [INFO] TestAgent.server: member joined, marking health alive: member=Node-69464632-1085-b205-9f3a-db3705889edb -13:05.022 [INFO] TestAgent.server: federation state anti-entropy synced -13:05.031 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:05.032 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16226 network=tcp -13:05.032 [INFO] TestAgent: Synced node info -13:05.032 [DEBUG] TestAgent: Node info in sync -13:05.047 [INFO] TestAgent: Requesting shutdown -13:05.047 [INFO] TestAgent.server: shutting down server -13:05.047 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:05.047 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:05.047 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:05.047 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:05.047 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.047 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:05.047 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:05.047 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:05.047 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:05.047 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:05.047 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:05.047Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:05.047Z [INFO] agent.router.manager: shutting down -13:05.048 [INFO] TestAgent: consul server down -13:05.048 [INFO] TestAgent: shutdown complete -13:05.048 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16220 network=tcp -13:05.048 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16220 network=udp -13:05.048 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16221 network=tcp -13:05.550 [INFO] TestAgent: Waiting for endpoints to shut down -13:05.550 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_Recurse (0.85s) -=== CONT TestIntentionsMatch_noBy -2021-10-31T23:13:04.823Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:04.919Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:04.921 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4dafc7aa-4cb9-5575-fb69-213cdd177de6 Address:127.0.0.1:16246}]" -13:04.921 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16246 [Follower]" leader= -13:04.921 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4dafc7aa-4cb9-5575-fb69-213cdd177de6.dc1 127.0.0.1 -13:04.930 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4dafc7aa-4cb9-5575-fb69-213cdd177de6 127.0.0.1 -2021-10-31T23:13:04.930Z [INFO] agent.router: Initializing LAN area manager -13:04.931 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16241 network=udp -13:04.931 [INFO] TestAgent.server: Adding LAN server: server="Node-4dafc7aa-4cb9-5575-fb69-213cdd177de6 (Addr: tcp/127.0.0.1:16246) (DC: dc1)" -13:04.931 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4dafc7aa-4cb9-5575-fb69-213cdd177de6.dc1 area=wan -13:04.931 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16241 network=tcp -13:04.931 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16242 network=tcp -13:04.932 [INFO] TestAgent: started state syncer -13:04.932 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16247 network=tcp -13:05.032 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.032 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16246 [Candidate]" term=2 -13:05.042 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.042 [DEBUG] TestAgent.server.raft: vote granted: from=4dafc7aa-4cb9-5575-fb69-213cdd177de6 term=2 tally=1 -13:05.042 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.042 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16246 [Leader]" -13:05.042 [INFO] TestAgent.server: cluster leadership acquired -13:05.042 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16246 -13:05.043 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.044 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.044 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.044 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.044 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.044 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.044 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4dafc7aa-4cb9-5575-fb69-213cdd177de6 -13:05.044 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4dafc7aa-4cb9-5575-fb69-213cdd177de6 -13:05.045 [INFO] TestAgent.server: New leader elected: payload=Node-4dafc7aa-4cb9-5575-fb69-213cdd177de6 -13:05.045 [INFO] TestAgent.server: federation state anti-entropy synced -13:05.074 [INFO] TestAgent: Requesting shutdown -13:05.074 [INFO] TestAgent.server: shutting down server -13:05.074 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:05.074 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:05.074 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:05.074 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:05.074 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.074 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:05.074 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:05.074 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:05.074 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:05.074 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:05.074 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:05.074 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:05.074Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:05.074Z [INFO] agent.router.manager: shutting down -13:05.074 [INFO] TestAgent: consul server down -13:05.074 [INFO] TestAgent: shutdown complete -13:05.074 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16241 network=tcp -13:05.074 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16241 network=udp -13:05.074 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16242 network=tcp -13:05.582 [INFO] TestAgent: Waiting for endpoints to shut down -13:05.582 [INFO] TestAgent: Endpoints down ---- PASS: TestParseConsistency_Invalid (0.88s) -=== CONT TestIntentionsMatch_basic -2021-10-31T23:13:04.741Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:04.741Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:04.818Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:04.818Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:04.819 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e46e26f1-434b-3eb3-9f9e-1c381cfea072 Address:127.0.0.1:16155}]" -13:04.820 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e46e26f1-434b-3eb3-9f9e-1c381cfea072.dc1 127.0.0.1 -13:04.821 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e46e26f1-434b-3eb3-9f9e-1c381cfea072 127.0.0.1 -2021-10-31T23:13:04.821Z [INFO] agent.router: Initializing LAN area manager -13:04.821 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16150 network=udp -13:04.821 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16155 [Follower]" leader= -13:04.821 [INFO] TestAgent.server: Adding LAN server: server="Node-e46e26f1-434b-3eb3-9f9e-1c381cfea072 (Addr: tcp/127.0.0.1:16155) (DC: dc1)" -13:04.821 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e46e26f1-434b-3eb3-9f9e-1c381cfea072.dc1 area=wan -13:04.822 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16150 network=tcp -13:04.822 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16151 network=tcp -13:04.822 [INFO] TestAgent: started state syncer -13:04.822 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16156 network=tcp -13:04.930 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:04.930 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16155 [Candidate]" term=2 -13:04.930 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:04.930 [DEBUG] TestAgent.server.raft: vote granted: from=e46e26f1-434b-3eb3-9f9e-1c381cfea072 term=2 tally=1 -13:04.930 [INFO] TestAgent.server.raft: election won: tally=1 -13:04.930 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16155 [Leader]" -13:04.931 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=2 mode=2 found=true leader=127.0.0.1:16155 -13:04.931 [INFO] TestAgent.server: cluster leadership acquired -13:04.931 [INFO] TestAgent.server: initializing acls -13:04.931 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:04.931 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:04.931 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:04.932 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:04.932 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:04.932 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:04.932 [INFO] TestAgent.server: New leader elected: payload=Node-e46e26f1-434b-3eb3-9f9e-1c381cfea072 -13:04.932 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e46e26f1-434b-3eb3-9f9e-1c381cfea072 -13:04.932 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e46e26f1-434b-3eb3-9f9e-1c381cfea072.dc1 -13:04.932 [INFO] TestAgent.server: Updating LAN server: server="Node-e46e26f1-434b-3eb3-9f9e-1c381cfea072 (Addr: tcp/127.0.0.1:16155) (DC: dc1)" -13:04.932 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e46e26f1-434b-3eb3-9f9e-1c381cfea072.dc1 area=wan -13:04.932 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:04.954 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:04.954 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:04.954 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:04.954 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:04.954 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:04.955 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e46e26f1-434b-3eb3-9f9e-1c381cfea072 -13:04.955 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e46e26f1-434b-3eb3-9f9e-1c381cfea072 -13:04.964 [INFO] TestAgent.server: federation state anti-entropy synced -13:05.031 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e46e26f1-434b-3eb3-9f9e-1c381cfea072 -13:05.062 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e46e26f1-434b-3eb3-9f9e-1c381cfea072 -13:05.063 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=foo accessorID= -13:05.063 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=bar accessorID= -13:05.063 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=zip accessorID= -13:05.063 [INFO] TestAgent: Requesting shutdown -13:05.063 [INFO] TestAgent.server: shutting down server -13:05.063 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:05.063 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:05.063 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:05.063 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:05.063 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:05.063 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:05.063 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.063 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:05.063 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:05.063 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:05.063 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:05.063 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:05.063 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:05.063 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:05.063 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:05.063 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:05.064Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:05.064Z [INFO] agent.router.manager: shutting down -13:05.064 [INFO] TestAgent: consul server down -13:05.064 [INFO] TestAgent: shutdown complete -13:05.064 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16150 network=tcp -13:05.064 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16150 network=udp -13:05.064 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16151 network=tcp -13:05.585 [INFO] TestAgent: Waiting for endpoints to shut down -13:05.585 [INFO] TestAgent: Endpoints down ---- PASS: TestUserEventToken (0.90s) -=== CONT TestIntentionsList_values -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:04.814Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:05.025Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:05.035 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d1e3cde5-7f84-3e8d-e5f5-d178119726dd Address:127.0.0.1:16218}]" -13:05.036 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d1e3cde5-7f84-3e8d-e5f5-d178119726dd.dc1 127.0.0.1 -13:05.036 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d1e3cde5-7f84-3e8d-e5f5-d178119726dd 127.0.0.1 -2021-10-31T23:13:05.036Z [INFO] agent.router: Initializing LAN area manager -13:05.038 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16218 [Follower]" leader= -13:05.045 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d1e3cde5-7f84-3e8d-e5f5-d178119726dd.dc1 area=wan -13:05.045 [INFO] TestAgent.server: Adding LAN server: server="Node-d1e3cde5-7f84-3e8d-e5f5-d178119726dd (Addr: tcp/127.0.0.1:16218) (DC: dc1)" -13:05.073 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.073 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16218 [Candidate]" term=2 -13:05.074 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.074 [DEBUG] TestAgent.server.raft: vote granted: from=d1e3cde5-7f84-3e8d-e5f5-d178119726dd term=2 tally=1 -13:05.074 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.074 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16218 [Leader]" -13:05.074 [INFO] TestAgent.server: cluster leadership acquired -13:05.074 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16218 -13:05.074 [INFO] TestAgent.server: New leader elected: payload=Node-d1e3cde5-7f84-3e8d-e5f5-d178119726dd -13:05.075 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.076 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.076 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.076 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.076 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.076 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.076 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d1e3cde5-7f84-3e8d-e5f5-d178119726dd -13:05.076 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d1e3cde5-7f84-3e8d-e5f5-d178119726dd -13:05.077 [INFO] TestAgent.server: federation state anti-entropy synced -13:05.106 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16213 network=udp -13:05.106 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16213 network=tcp -13:05.107 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16214 network=tcp -13:05.107 [INFO] TestAgent: started state syncer -13:05.107 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:05.107 [INFO] TestAgent: Requesting shutdown -13:05.107 [INFO] TestAgent.server: shutting down server -13:05.107 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:05.107 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:05.107 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:05.107 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:05.107 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.107 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:05.107 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16219 network=tcp -13:05.107 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -13:05.107 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:05.107 [INFO] TestAgent: Synced node info -13:05.107 [DEBUG] TestAgent: Node info in sync -13:05.107 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:05.107 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:05.108 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" -13:05.108 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:05.110 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:05.122Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:05.122Z [INFO] agent.router.manager: shutting down -13:05.130 [INFO] TestAgent: consul server down -13:05.130 [INFO] TestAgent: shutdown complete -13:05.130 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16213 network=tcp -13:05.130 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16213 network=udp -13:05.130 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16214 network=tcp -13:05.633 [INFO] TestAgent: Waiting for endpoints to shut down -13:05.634 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Explain/#01 -2021-10-31T23:13:04.717Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:04.916Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:04.918 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3f07ba3f-668a-a624-ba87-97323b2a3fb5 Address:127.0.0.1:16148}]" -13:04.919 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3f07ba3f-668a-a624-ba87-97323b2a3fb5.dc1 127.0.0.1 -13:04.919 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3f07ba3f-668a-a624-ba87-97323b2a3fb5 127.0.0.1 -2021-10-31T23:13:04.919Z [INFO] agent.router: Initializing LAN area manager -13:04.919 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16143 network=udp -13:04.919 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16148 [Follower]" leader= -13:04.920 [INFO] TestAgent.server: Adding LAN server: server="Node-3f07ba3f-668a-a624-ba87-97323b2a3fb5 (Addr: tcp/127.0.0.1:16148) (DC: dc1)" -13:04.920 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3f07ba3f-668a-a624-ba87-97323b2a3fb5.dc1 area=wan -13:04.920 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16143 network=tcp -13:04.920 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16144 network=tcp -13:04.921 [INFO] TestAgent: started state syncer -13:04.921 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16149 network=tcp -13:05.026 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.026 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16148 [Candidate]" term=2 -13:05.034 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.034 [DEBUG] TestAgent.server.raft: vote granted: from=3f07ba3f-668a-a624-ba87-97323b2a3fb5 term=2 tally=1 -13:05.034 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.034 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16148 [Leader]" -13:05.034 [INFO] TestAgent.server: cluster leadership acquired -13:05.035 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16148 -13:05.035 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.037 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.037 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.037 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.037 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.037 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.037 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3f07ba3f-668a-a624-ba87-97323b2a3fb5 -13:05.037 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3f07ba3f-668a-a624-ba87-97323b2a3fb5 -13:05.038 [INFO] TestAgent.server: federation state anti-entropy synced -13:05.041 [INFO] TestAgent.server: New leader elected: payload=Node-3f07ba3f-668a-a624-ba87-97323b2a3fb5 -13:05.198 [INFO] TestAgent: Requesting shutdown -13:05.198 [INFO] TestAgent.server: shutting down server -13:05.198 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:05.198 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:05.198 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:05.198 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:05.198 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.198 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:05.198 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:05.198 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:05.198 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:05.198 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:05.198 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:05.199Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:05.199Z [INFO] agent.router.manager: shutting down -13:05.202 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:05.202 [WARN] TestAgent: Syncing node info failed.: error="No cluster leader" -13:05.202 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:05.212 [INFO] TestAgent: consul server down -13:05.212 [INFO] TestAgent: shutdown complete -13:05.212 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16143 network=tcp -13:05.212 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16143 network=udp -13:05.212 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16144 network=tcp -13:05.713 [INFO] TestAgent: Waiting for endpoints to shut down -13:05.713 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Legacy_Disabled_Response (1.03s) +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:14.215Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.251Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.253 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0919537a-8edc-62c4-59ba-483e4123eacb Address:127.0.0.1:23323}]" +49:14.253 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0919537a-8edc-62c4-59ba-483e4123eacb.dc1 127.0.0.1 +49:14.254 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23323 [Follower]" leader= +49:14.254 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0919537a-8edc-62c4-59ba-483e4123eacb 127.0.0.1 +2022-12-04T05:49:14.254Z [INFO] agent.router: Initializing LAN area manager +49:14.254 [INFO] TestAgent.server: Adding LAN server: server="Node-0919537a-8edc-62c4-59ba-483e4123eacb (Addr: tcp/127.0.0.1:23323) (DC: dc1)" +49:14.254 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0919537a-8edc-62c4-59ba-483e4123eacb.dc1 area=wan +49:14.254 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23311 network=udp +49:14.254 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23311 network=tcp +49:14.255 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23312 network=tcp +49:14.255 [INFO] TestAgent: started state syncer +49:14.255 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23324 network=tcp +49:14.291 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.291 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23323 [Candidate]" term=2 +49:14.291 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.291 [DEBUG] TestAgent.server.raft: vote granted: from=0919537a-8edc-62c4-59ba-483e4123eacb term=2 tally=1 +49:14.291 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.291 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23323 [Leader]" +49:14.291 [INFO] TestAgent.server: cluster leadership acquired +49:14.291 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23323 +49:14.291 [INFO] TestAgent.server: New leader elected: payload=Node-0919537a-8edc-62c4-59ba-483e4123eacb +49:14.292 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.294 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.294 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.294 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.294 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.294 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.294 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0919537a-8edc-62c4-59ba-483e4123eacb +49:14.294 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0919537a-8edc-62c4-59ba-483e4123eacb +49:14.295 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.363 [INFO] TestAgent: Requesting shutdown +49:14.363 [INFO] TestAgent.server: shutting down server +49:14.363 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.363 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.363 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.363 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.363 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.363 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.363 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.363 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:14.363 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.363 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:14.363 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.363 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:14.363Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.363Z [INFO] agent.router.manager: shutting down +49:14.363 [INFO] TestAgent: consul server down +49:14.363 [INFO] TestAgent: shutdown complete +49:14.363 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23311 network=tcp +49:14.363 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23311 network=udp +49:14.363 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23312 network=tcp +49:14.863 [INFO] TestAgent: Waiting for endpoints to shut down +49:14.863 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionCreate_Delete (0.69s) +=== CONT TestPreparedQuery_Update +2022-12-04T05:49:14.222Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:14.222Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.256Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:14.256Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.257 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8 Address:127.0.0.1:23344}]" +49:14.258 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8.dc1 127.0.0.1 +49:14.259 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8 127.0.0.1 +2022-12-04T05:49:14.259Z [INFO] agent.router: Initializing LAN area manager +49:14.259 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23332 network=udp +49:14.259 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23344 [Follower]" leader= +49:14.259 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8.dc1 area=wan +49:14.259 [INFO] TestAgent.server: Adding LAN server: server="Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8 (Addr: tcp/127.0.0.1:23344) (DC: dc1)" +49:14.259 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23332 network=tcp +49:14.260 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23333 network=tcp +49:14.260 [INFO] TestAgent: started state syncer +49:14.260 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23345 network=tcp +49:14.309 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.309 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23344 [Candidate]" term=2 +49:14.309 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.309 [DEBUG] TestAgent.server.raft: vote granted: from=484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8 term=2 tally=1 +49:14.309 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.309 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23344 [Leader]" +49:14.309 [INFO] TestAgent.server: cluster leadership acquired +49:14.309 [INFO] TestAgent.server: New leader elected: payload=Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8 +49:14.310 [INFO] TestAgent.server: initializing acls +49:14.310 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:14.310 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:14.310 [INFO] TestAgent.server: initializing acls +49:14.310 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:14.310 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:14.310 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:14.311 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:14.311 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:14.311 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:14.311 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +49:14.311 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:14.311 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8 +49:14.312 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8.dc1 +49:14.312 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8.dc1 area=wan +49:14.312 [INFO] TestAgent.server: Updating LAN server: server="Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8 (Addr: tcp/127.0.0.1:23344) (DC: dc1)" +49:14.312 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8 +49:14.312 [INFO] TestAgent.server: Updating LAN server: server="Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8 (Addr: tcp/127.0.0.1:23344) (DC: dc1)" +49:14.312 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8.dc1 +49:14.312 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8.dc1 area=wan +49:14.313 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.316 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.316 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.316 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.316 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.316 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.316 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8 +49:14.316 [INFO] TestAgent.server: member joined, marking health alive: member=Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8 +49:14.317 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8 +49:14.317 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8 +49:14.328 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.364 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:14.364 [INFO] TestAgent: Synced node info +49:14.404 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8 +49:14.404 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-484b2cb7-c44a-0d7b-5b1d-fa2a71d626d8 +49:14.405 [INFO] TestAgent: Requesting shutdown +49:14.405 [INFO] TestAgent.server: shutting down server +49:14.405 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:14.405 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:14.405 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.405 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.405 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.405 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.405 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.405 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.405 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:14.405 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.405 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:14.405 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:14.405 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.405 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.405 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:14.406Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.406Z [INFO] agent.router.manager: shutting down +49:14.406 [INFO] TestAgent: consul server down +49:14.406 [INFO] TestAgent: shutdown complete +49:14.406 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23332 network=tcp +49:14.406 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23332 network=udp +49:14.406 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23333 network=tcp +49:14.907 [INFO] TestAgent: Waiting for endpoints to shut down +49:14.907 [INFO] TestAgent: Endpoints down +--- PASS: TestRemoteExecGetSpec_ACLToken (0.73s) +=== CONT TestPreparedQuery_Get +=== RUN TestPreparedQuery_Get/#00 +2022-12-04T05:49:14.209Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.221Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.237 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2bc07fbc-bb6a-e0e5-cd7d-649b4eabd3ba Address:127.0.0.1:23302}]" +49:14.237 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23302 [Follower]" leader= +49:14.239 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2bc07fbc-bb6a-e0e5-cd7d-649b4eabd3ba.dc1 127.0.0.1 +49:14.240 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2bc07fbc-bb6a-e0e5-cd7d-649b4eabd3ba 127.0.0.1 +2022-12-04T05:49:14.240Z [INFO] agent.router: Initializing LAN area manager +49:14.241 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23283 network=udp +49:14.241 [INFO] TestAgent.server: Adding LAN server: server="Node-2bc07fbc-bb6a-e0e5-cd7d-649b4eabd3ba (Addr: tcp/127.0.0.1:23302) (DC: dc1)" +49:14.241 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2bc07fbc-bb6a-e0e5-cd7d-649b4eabd3ba.dc1 area=wan +49:14.241 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23283 network=tcp +49:14.242 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23284 network=tcp +49:14.242 [INFO] TestAgent: started state syncer +49:14.242 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23303 network=tcp +49:14.292 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.292 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23302 [Candidate]" term=2 +49:14.292 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.292 [DEBUG] TestAgent.server.raft: vote granted: from=2bc07fbc-bb6a-e0e5-cd7d-649b4eabd3ba term=2 tally=1 +49:14.292 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.292 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23302 [Leader]" +49:14.292 [INFO] TestAgent.server: cluster leadership acquired +49:14.292 [INFO] TestAgent.server: New leader elected: payload=Node-2bc07fbc-bb6a-e0e5-cd7d-649b4eabd3ba +49:14.293 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23302 +49:14.293 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.295 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.295 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.295 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.295 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.295 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.295 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2bc07fbc-bb6a-e0e5-cd7d-649b4eabd3ba +49:14.295 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2bc07fbc-bb6a-e0e5-cd7d-649b4eabd3ba +49:14.295 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.394 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:14.394 [INFO] TestAgent: Synced node info +49:14.394 [DEBUG] TestAgent: Node info in sync +49:14.423 [INFO] TestAgent: Requesting shutdown +49:14.423 [INFO] TestAgent.server: shutting down server +49:14.423 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.423 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.423 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.423 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.423 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.423 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.423 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.423 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.423 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.423 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:14.423 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:14.424Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.424Z [INFO] agent.router.manager: shutting down +49:14.424 [INFO] TestAgent: consul server down +49:14.424 [INFO] TestAgent: shutdown complete +49:14.424 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23283 network=tcp +49:14.424 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23283 network=udp +49:14.424 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23284 network=tcp +49:14.924 [INFO] TestAgent: Waiting for endpoints to shut down +49:14.924 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Legacy_Disabled_Response (0.75s) --- PASS: TestACL_Legacy_Disabled_Response/0 (0.00s) --- PASS: TestACL_Legacy_Disabled_Response/1 (0.00s) --- PASS: TestACL_Legacy_Disabled_Response/2 (0.00s) --- PASS: TestACL_Legacy_Disabled_Response/3 (0.00s) --- PASS: TestACL_Legacy_Disabled_Response/4 (0.00s) --- PASS: TestACL_Legacy_Disabled_Response/5 (0.00s) +=== CONT TestPreparedQuery_Explain +=== RUN TestPreparedQuery_Explain/#00 +2022-12-04T05:49:14.257Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.314Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.317 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:79fdea2a-44f7-3a4b-0cba-cf87ef81ee06 Address:127.0.0.1:23386}]" +49:14.317 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-79fdea2a-44f7-3a4b-0cba-cf87ef81ee06.dc1 127.0.0.1 +49:14.318 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-79fdea2a-44f7-3a4b-0cba-cf87ef81ee06 127.0.0.1 +2022-12-04T05:49:14.318Z [INFO] agent.router: Initializing LAN area manager +49:14.329 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23386 [Follower]" leader= +49:14.329 [INFO] TestAgent.server: Adding LAN server: server="Node-79fdea2a-44f7-3a4b-0cba-cf87ef81ee06 (Addr: tcp/127.0.0.1:23386) (DC: dc1)" +49:14.329 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-79fdea2a-44f7-3a4b-0cba-cf87ef81ee06.dc1 area=wan +49:14.330 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23381 network=udp +49:14.330 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23381 network=tcp +49:14.331 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23382 network=tcp +49:14.331 [INFO] TestAgent: started state syncer +49:14.331 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23387 network=tcp +49:14.376 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.376 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23386 [Candidate]" term=2 +49:14.376 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.376 [DEBUG] TestAgent.server.raft: vote granted: from=79fdea2a-44f7-3a4b-0cba-cf87ef81ee06 term=2 tally=1 +49:14.376 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.376 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23386 [Leader]" +49:14.376 [INFO] TestAgent.server: cluster leadership acquired +49:14.376 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23386 +49:14.376 [INFO] TestAgent.server: New leader elected: payload=Node-79fdea2a-44f7-3a4b-0cba-cf87ef81ee06 +49:14.377 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.378 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.378 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.378 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.378 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.378 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.378 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-79fdea2a-44f7-3a4b-0cba-cf87ef81ee06 +49:14.378 [INFO] TestAgent.server: member joined, marking health alive: member=Node-79fdea2a-44f7-3a4b-0cba-cf87ef81ee06 +49:14.379 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.414 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:14.414 [INFO] TestAgent: Synced node info +49:14.503 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:14.503 [INFO] TestAgent: Requesting shutdown +49:14.503 [INFO] TestAgent.server: shutting down server +49:14.503 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.503 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.503 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.503 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.503 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.503 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.503 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.503 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.503 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:14.503 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.503 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:14.503Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.503Z [INFO] agent.router.manager: shutting down +49:14.503 [INFO] TestAgent: consul server down +49:14.503 [INFO] TestAgent: shutdown complete +49:14.503 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23381 network=tcp +49:14.503 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23381 network=udp +49:14.503 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23382 network=tcp +49:15.004 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.004 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_Delete (0.83s) +=== CONT TestPreparedQuery_ExecuteCached +2022-12-04T05:49:14.221Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:14.221Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.270Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:14.270Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.272 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7056e7e9-4139-b569-76aa-3de9968b6663 Address:127.0.0.1:23351}]" +49:14.273 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23351 [Follower]" leader= +49:14.313 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7056e7e9-4139-b569-76aa-3de9968b6663.dc1 127.0.0.1 +49:14.314 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7056e7e9-4139-b569-76aa-3de9968b6663 127.0.0.1 +2022-12-04T05:49:14.314Z [INFO] agent.router: Initializing LAN area manager +49:14.314 [INFO] TestAgent.server: Adding LAN server: server="Node-7056e7e9-4139-b569-76aa-3de9968b6663 (Addr: tcp/127.0.0.1:23351) (DC: dc1)" +49:14.314 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7056e7e9-4139-b569-76aa-3de9968b6663.dc1 area=wan +49:14.314 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23360 network=tcp +49:14.314 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23360 network=udp +49:14.325 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23361 network=tcp +49:14.325 [INFO] TestAgent: started state syncer +49:14.325 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23352 network=tcp +49:14.341 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.341 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23351 [Candidate]" term=2 +49:14.341 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.341 [DEBUG] TestAgent.server.raft: vote granted: from=7056e7e9-4139-b569-76aa-3de9968b6663 term=2 tally=1 +49:14.341 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.341 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23351 [Leader]" +49:14.341 [INFO] TestAgent.server: cluster leadership acquired +49:14.341 [INFO] TestAgent.server: initializing acls +49:14.342 [INFO] TestAgent.server: New leader elected: payload=Node-7056e7e9-4139-b569-76aa-3de9968b6663 +49:14.342 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:14.342 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:14.342 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:14.342 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:14.342 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:14.342 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:14.342 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7056e7e9-4139-b569-76aa-3de9968b6663 +49:14.342 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7056e7e9-4139-b569-76aa-3de9968b6663.dc1 +49:14.342 [INFO] TestAgent.server: Updating LAN server: server="Node-7056e7e9-4139-b569-76aa-3de9968b6663 (Addr: tcp/127.0.0.1:23351) (DC: dc1)" +49:14.343 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7056e7e9-4139-b569-76aa-3de9968b6663.dc1 area=wan +49:14.343 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.344 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.344 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.344 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.344 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.344 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.344 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7056e7e9-4139-b569-76aa-3de9968b6663 +49:14.344 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7056e7e9-4139-b569-76aa-3de9968b6663 +49:14.345 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.345 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7056e7e9-4139-b569-76aa-3de9968b6663 +49:14.510 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7056e7e9-4139-b569-76aa-3de9968b6663 +49:14.510 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7056e7e9-4139-b569-76aa-3de9968b6663 +49:14.511 [ERROR] TestAgent: failed to get remote exec job: error="Permission denied" +49:14.511 [INFO] TestAgent: Requesting shutdown +49:14.512 [INFO] TestAgent.server: shutting down server +49:14.512 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:14.512 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:14.512 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.512 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.512 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.512 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.512 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.512 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.512 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.512 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:14.512 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:14.512 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:14.512 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.512 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.512 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:14.512 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:14.512Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.512Z [INFO] agent.router.manager: shutting down +49:14.512 [INFO] TestAgent: consul server down +49:14.512 [INFO] TestAgent: shutdown complete +49:14.512 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23360 network=tcp +49:14.512 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23360 network=udp +49:14.512 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23361 network=tcp +49:15.013 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.013 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:14.216Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.255Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.256 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:40db082f-454c-1dff-e9ff-a4a0f74b1eba Address:127.0.0.1:23379}]" +49:14.257 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-40db082f-454c-1dff-e9ff-a4a0f74b1eba.dc1 127.0.0.1 +49:14.257 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-40db082f-454c-1dff-e9ff-a4a0f74b1eba 127.0.0.1 +2022-12-04T05:49:14.257Z [INFO] agent.router: Initializing LAN area manager +49:14.257 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23374 network=udp +49:14.258 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23379 [Follower]" leader= +49:14.258 [INFO] TestAgent.server: Adding LAN server: server="Node-40db082f-454c-1dff-e9ff-a4a0f74b1eba (Addr: tcp/127.0.0.1:23379) (DC: dc1)" +49:14.258 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-40db082f-454c-1dff-e9ff-a4a0f74b1eba.dc1 area=wan +49:14.258 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23374 network=tcp +49:14.259 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23375 network=tcp +49:14.259 [INFO] TestAgent: started state syncer +49:14.259 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23380 network=tcp +49:14.296 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.296 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23379 [Candidate]" term=2 +49:14.296 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.296 [DEBUG] TestAgent.server.raft: vote granted: from=40db082f-454c-1dff-e9ff-a4a0f74b1eba term=2 tally=1 +49:14.296 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.296 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23379 [Leader]" +49:14.296 [INFO] TestAgent.server: cluster leadership acquired +49:14.297 [INFO] TestAgent.server: New leader elected: payload=Node-40db082f-454c-1dff-e9ff-a4a0f74b1eba +49:14.297 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23379 +49:14.297 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.301 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.301 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.301 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.302 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.302 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.302 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-40db082f-454c-1dff-e9ff-a4a0f74b1eba +49:14.302 [INFO] TestAgent.server: member joined, marking health alive: member=Node-40db082f-454c-1dff-e9ff-a4a0f74b1eba +49:14.303 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.502 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:14.503 [INFO] TestAgent: Synced node info +49:14.503 [DEBUG] TestAgent: Node info in sync +49:14.512 [INFO] TestAgent: Requesting shutdown +49:14.512 [INFO] TestAgent.server: shutting down server +49:14.512 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.512 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.512 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.512 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.512 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.512 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.512 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.512 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.512 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.512 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:14.512 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:14.512Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.512Z [INFO] agent.router.manager: shutting down +49:14.512 [INFO] TestAgent: consul server down +49:14.512 [INFO] TestAgent: shutdown complete +49:14.512 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23374 network=tcp +49:14.512 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23374 network=udp +49:14.512 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23375 network=tcp +49:15.013 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.013 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_AutopilotGetConfiguration (0.84s) +=== CONT TestPreparedQuery_Execute +=== RUN TestPreparedQuery_Execute/#00 +--- PASS: TestRemoteExecGetSpec_ACLDeny (0.84s) +=== CONT TestPreparedQuery_List +=== RUN TestPreparedQuery_List/#00 +2022-12-04T05:49:14.212Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.244Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.258 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:415fc71f-7c54-b051-58f1-5a1a9730ac43 Address:127.0.0.1:23337}]" +49:14.258 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-415fc71f-7c54-b051-58f1-5a1a9730ac43.dc1 127.0.0.1 +49:14.259 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-415fc71f-7c54-b051-58f1-5a1a9730ac43 127.0.0.1 +2022-12-04T05:49:14.259Z [INFO] agent.router: Initializing LAN area manager +49:14.259 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23337 [Follower]" leader= +49:14.259 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23346 network=udp +49:14.259 [INFO] TestAgent.server: Adding LAN server: server="Node-415fc71f-7c54-b051-58f1-5a1a9730ac43 (Addr: tcp/127.0.0.1:23337) (DC: dc1)" +49:14.259 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-415fc71f-7c54-b051-58f1-5a1a9730ac43.dc1 area=wan +49:14.259 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23346 network=tcp +49:14.260 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23347 network=tcp +49:14.260 [INFO] TestAgent: started state syncer +49:14.260 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23338 network=tcp +49:14.295 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.295 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23337 [Candidate]" term=2 +49:14.296 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.296 [DEBUG] TestAgent.server.raft: vote granted: from=415fc71f-7c54-b051-58f1-5a1a9730ac43 term=2 tally=1 +49:14.296 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.296 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23337 [Leader]" +49:14.296 [INFO] TestAgent.server: cluster leadership acquired +49:14.297 [INFO] TestAgent.server: New leader elected: payload=Node-415fc71f-7c54-b051-58f1-5a1a9730ac43 +49:14.297 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23337 +49:14.297 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.302 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.302 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.302 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.302 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.302 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.302 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-415fc71f-7c54-b051-58f1-5a1a9730ac43 +49:14.302 [INFO] TestAgent.server: member joined, marking health alive: member=Node-415fc71f-7c54-b051-58f1-5a1a9730ac43 +49:14.303 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.535 [INFO] TestAgent: Requesting shutdown +49:14.535 [INFO] TestAgent.server: shutting down server +49:14.535 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.535 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.535 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.535 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.535 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.535 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.535 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.535 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:14.535 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.535 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.535 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:14.535 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:14.536Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.536Z [INFO] agent.router.manager: shutting down +49:14.536 [INFO] TestAgent: consul server down +49:14.536 [INFO] TestAgent: shutdown complete +49:14.536 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23346 network=tcp +49:14.536 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23346 network=udp +49:14.536 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23347 network=tcp +49:15.036 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.036 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionCreate (0.86s) +=== CONT TestFireReceiveEvent +[WARN] freeport: 10 out of 59 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:14.314Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.338Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.340 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6ed17b00-c4a3-7ee6-0e28-55298f73dfeb Address:127.0.0.1:23365}]" +49:14.340 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23365 [Follower]" leader= +49:14.340 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6ed17b00-c4a3-7ee6-0e28-55298f73dfeb.dc1 127.0.0.1 +49:14.340 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6ed17b00-c4a3-7ee6-0e28-55298f73dfeb 127.0.0.1 +2022-12-04T05:49:14.340Z [INFO] agent.router: Initializing LAN area manager +49:14.341 [INFO] TestAgent.server: Adding LAN server: server="Node-6ed17b00-c4a3-7ee6-0e28-55298f73dfeb (Addr: tcp/127.0.0.1:23365) (DC: dc1)" +49:14.341 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6ed17b00-c4a3-7ee6-0e28-55298f73dfeb.dc1 area=wan +49:14.341 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23353 network=udp +49:14.341 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23353 network=tcp +49:14.341 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23354 network=tcp +49:14.341 [INFO] TestAgent: started state syncer +49:14.341 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23366 network=tcp +49:14.379 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.379 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23365 [Candidate]" term=2 +49:14.379 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.379 [DEBUG] TestAgent.server.raft: vote granted: from=6ed17b00-c4a3-7ee6-0e28-55298f73dfeb term=2 tally=1 +49:14.379 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.379 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23365 [Leader]" +49:14.379 [INFO] TestAgent.server: cluster leadership acquired +49:14.379 [INFO] TestAgent.server: New leader elected: payload=Node-6ed17b00-c4a3-7ee6-0e28-55298f73dfeb +49:14.379 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23365 +49:14.380 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.381 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.381 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.381 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.381 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.381 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.381 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6ed17b00-c4a3-7ee6-0e28-55298f73dfeb +49:14.381 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6ed17b00-c4a3-7ee6-0e28-55298f73dfeb +49:14.381 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.425 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:14.425 [INFO] TestAgent: Synced node info +49:14.425 [DEBUG] TestAgent: Node info in sync +49:14.548 [INFO] TestAgent: Requesting shutdown +49:14.549 [INFO] TestAgent.server: shutting down server +49:14.549 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.549 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.549 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.549 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.549 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.549 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.549 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.549 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.549 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.549 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:14.549 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:14.549Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.549Z [INFO] agent.router.manager: shutting down +49:14.549 [INFO] TestAgent: consul server down +49:14.549 [INFO] TestAgent: shutdown complete +49:14.549 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23353 network=tcp +49:14.549 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23353 network=udp +49:14.549 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23354 network=tcp +49:15.052 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.052 [INFO] TestAgent: Endpoints down +--- PASS: TestRemoteExecGetSpec (0.88s) +=== CONT TestPreparedQuery_Create +2022-12-04T05:49:14.207Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.222Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.241 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:793ab15b-f2b2-20ad-c44b-75ced7d6cabe Address:127.0.0.1:23330}]" +49:14.242 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-793ab15b-f2b2-20ad-c44b-75ced7d6cabe.dc1 127.0.0.1 +49:14.242 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-793ab15b-f2b2-20ad-c44b-75ced7d6cabe 127.0.0.1 +2022-12-04T05:49:14.242Z [INFO] agent.router: Initializing LAN area manager +49:14.243 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23330 [Follower]" leader= +49:14.252 [INFO] TestAgent.server: Adding LAN server: server="Node-793ab15b-f2b2-20ad-c44b-75ced7d6cabe (Addr: tcp/127.0.0.1:23330) (DC: dc1)" +49:14.253 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-793ab15b-f2b2-20ad-c44b-75ced7d6cabe.dc1 area=wan +49:14.253 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23325 network=tcp +49:14.253 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23325 network=udp +49:14.254 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23326 network=tcp +49:14.254 [INFO] TestAgent: started state syncer +49:14.254 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23331 network=tcp +49:14.312 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.312 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23330 [Candidate]" term=2 +49:14.312 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.312 [DEBUG] TestAgent.server.raft: vote granted: from=793ab15b-f2b2-20ad-c44b-75ced7d6cabe term=2 tally=1 +49:14.312 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.312 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23330 [Leader]" +49:14.312 [INFO] TestAgent.server: cluster leadership acquired +49:14.312 [INFO] TestAgent.server: New leader elected: payload=Node-793ab15b-f2b2-20ad-c44b-75ced7d6cabe +49:14.312 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23330 +49:14.314 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.326 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.326 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.326 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.326 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.326 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.326 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-793ab15b-f2b2-20ad-c44b-75ced7d6cabe +49:14.326 [INFO] TestAgent.server: member joined, marking health alive: member=Node-793ab15b-f2b2-20ad-c44b-75ced7d6cabe +49:14.327 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.558 [INFO] TestAgent: Requesting shutdown +49:14.558 [INFO] TestAgent.server: shutting down server +49:14.558 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.558 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.558 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.558 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.558 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.558 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.558 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.558 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:14.558 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:14.558 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.558 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.558 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:14.558Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.558Z [INFO] agent.router.manager: shutting down +49:14.558 [INFO] TestAgent: consul server down +49:14.558 [INFO] TestAgent: shutdown complete +49:14.558 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23325 network=tcp +49:14.559 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23325 network=udp +49:14.559 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23326 network=tcp +49:15.059 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.059 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_Integration (0.88s) +=== CONT TestIngestUserEvent +2022-12-04T05:49:14.220Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.258Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.261 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c387dea1-1e26-5e93-16a3-fef21fca7dc3 Address:127.0.0.1:23316}]" +49:14.261 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23316 [Follower]" leader= +49:14.262 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c387dea1-1e26-5e93-16a3-fef21fca7dc3.dc1 127.0.0.1 +49:14.262 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c387dea1-1e26-5e93-16a3-fef21fca7dc3 127.0.0.1 +2022-12-04T05:49:14.262Z [INFO] agent.router: Initializing LAN area manager +49:14.263 [INFO] TestAgent.server: Adding LAN server: server="Node-c387dea1-1e26-5e93-16a3-fef21fca7dc3 (Addr: tcp/127.0.0.1:23316) (DC: dc1)" +49:14.263 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c387dea1-1e26-5e93-16a3-fef21fca7dc3.dc1 area=wan +49:14.263 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23297 network=tcp +49:14.263 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23297 network=udp +49:14.265 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23298 network=tcp +49:14.265 [INFO] TestAgent: started state syncer +49:14.266 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23317 network=tcp +49:14.328 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.328 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23316 [Candidate]" term=2 +49:14.328 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.328 [DEBUG] TestAgent.server.raft: vote granted: from=c387dea1-1e26-5e93-16a3-fef21fca7dc3 term=2 tally=1 +49:14.328 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.328 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23316 [Leader]" +49:14.328 [INFO] TestAgent.server: cluster leadership acquired +49:14.328 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23316 +49:14.329 [INFO] TestAgent.server: New leader elected: payload=Node-c387dea1-1e26-5e93-16a3-fef21fca7dc3 +49:14.329 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.330 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.330 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.330 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.330 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.330 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.331 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c387dea1-1e26-5e93-16a3-fef21fca7dc3 +49:14.331 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c387dea1-1e26-5e93-16a3-fef21fca7dc3 +49:14.331 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.512 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:14.512 [INFO] TestAgent: Synced node info +49:14.544 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:14.544 [DEBUG] TestAgent: Node info in sync +49:14.544 [DEBUG] TestAgent: Node info in sync +49:14.578 [INFO] TestAgent: Requesting shutdown +49:14.578 [INFO] TestAgent.server: shutting down server +49:14.578 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.578 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.578 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.578 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.578 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.578 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.578 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.578 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:14.578 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.578 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.578 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:14.579Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.579Z [INFO] agent.router.manager: shutting down +49:14.579 [INFO] TestAgent: consul server down +49:14.579 [INFO] TestAgent: shutdown complete +49:14.579 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23297 network=tcp +49:14.579 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23297 network=udp +49:14.579 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23298 network=tcp +49:15.080 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.080 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_AutopilotSetConfiguration (0.90s) +=== CONT TestOperator_ServerHealth_Unhealthy +2022-12-04T05:49:14.200Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.253Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.255 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f9c2989c-0685-99f7-dd33-056b9a6f8cb6 Address:127.0.0.1:23295}]" +49:14.255 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f9c2989c-0685-99f7-dd33-056b9a6f8cb6.dc1 127.0.0.1 +49:14.255 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23295 [Follower]" leader= +49:14.256 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f9c2989c-0685-99f7-dd33-056b9a6f8cb6 127.0.0.1 +2022-12-04T05:49:14.256Z [INFO] agent.router: Initializing LAN area manager +49:14.256 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f9c2989c-0685-99f7-dd33-056b9a6f8cb6.dc1 area=wan +49:14.256 [INFO] TestAgent.server: Adding LAN server: server="Node-f9c2989c-0685-99f7-dd33-056b9a6f8cb6 (Addr: tcp/127.0.0.1:23295) (DC: dc1)" +49:14.256 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23304 network=udp +49:14.256 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23304 network=tcp +49:14.257 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23305 network=tcp +49:14.257 [INFO] TestAgent: started state syncer +49:14.257 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23296 network=tcp +49:14.328 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.328 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23295 [Candidate]" term=2 +49:14.328 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.328 [DEBUG] TestAgent.server.raft: vote granted: from=f9c2989c-0685-99f7-dd33-056b9a6f8cb6 term=2 tally=1 +49:14.328 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.328 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23295 [Leader]" +49:14.329 [INFO] TestAgent.server: cluster leadership acquired +49:14.329 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23295 +49:14.330 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.330 [INFO] TestAgent.server: New leader elected: payload=Node-f9c2989c-0685-99f7-dd33-056b9a6f8cb6 +49:14.331 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.331 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.331 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.331 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.331 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.331 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f9c2989c-0685-99f7-dd33-056b9a6f8cb6 +49:14.331 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f9c2989c-0685-99f7-dd33-056b9a6f8cb6 +49:14.332 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.591 [INFO] TestAgent: Requesting shutdown +49:14.591 [INFO] TestAgent.server: shutting down server +49:14.591 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.591 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.591 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.591 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.591 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.591 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.591 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.591 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:14.591 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.591 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.591 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:14.591 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:14.592Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.592Z [INFO] agent.router.manager: shutting down +49:14.592 [INFO] TestAgent: consul server down +49:14.592 [INFO] TestAgent: shutdown complete +49:14.592 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23304 network=tcp +49:14.592 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23304 network=udp +49:14.592 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23305 network=tcp +49:15.103 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.103 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_AutopilotCASConfiguration (0.93s) +=== CONT TestOperator_ServerHealth +2022-12-04T05:49:14.253Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.314Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.326 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6c1568b1-c464-3b41-cf83-61a005e63c0f Address:127.0.0.1:23393}]" +49:14.327 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23393 [Follower]" leader= +49:14.327 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6c1568b1-c464-3b41-cf83-61a005e63c0f.dc1 127.0.0.1 +49:14.328 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6c1568b1-c464-3b41-cf83-61a005e63c0f 127.0.0.1 +2022-12-04T05:49:14.328Z [INFO] agent.router: Initializing LAN area manager +49:14.328 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23388 network=udp +49:14.328 [INFO] TestAgent.server: Adding LAN server: server="Node-6c1568b1-c464-3b41-cf83-61a005e63c0f (Addr: tcp/127.0.0.1:23393) (DC: dc1)" +49:14.328 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6c1568b1-c464-3b41-cf83-61a005e63c0f.dc1 area=wan +49:14.328 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23388 network=tcp +49:14.329 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23389 network=tcp +49:14.329 [INFO] TestAgent: started state syncer +49:14.329 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23394 network=tcp +49:14.383 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.383 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23393 [Candidate]" term=2 +49:14.383 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.383 [DEBUG] TestAgent.server.raft: vote granted: from=6c1568b1-c464-3b41-cf83-61a005e63c0f term=2 tally=1 +49:14.383 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.383 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23393 [Leader]" +49:14.383 [INFO] TestAgent.server: cluster leadership acquired +49:14.383 [INFO] TestAgent.server: New leader elected: payload=Node-6c1568b1-c464-3b41-cf83-61a005e63c0f +49:14.383 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23393 +49:14.384 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.385 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.385 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.385 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.385 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.385 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.385 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6c1568b1-c464-3b41-cf83-61a005e63c0f +49:14.385 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6c1568b1-c464-3b41-cf83-61a005e63c0f +49:14.386 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.604 [DEBUG] TestAgent: received remote exec event: id=90ff3bb5-8b19-ec8b-9877-e0d4c5628fb5 +49:14.604 [INFO] TestAgent: remote exec script: script="echo failing;exit 2" +49:14.608 [INFO] TestAgent: Requesting shutdown +49:14.608 [INFO] TestAgent.server: shutting down server +49:14.608 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.608 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.608 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.608 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.608 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.608 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.608 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.608 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:14.608 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:14.608 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.608 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.608 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:14.608Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.608Z [INFO] agent.router.manager: shutting down +49:14.608 [INFO] TestAgent: consul server down +49:14.608 [INFO] TestAgent: shutdown complete +49:14.608 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23388 network=tcp +49:14.608 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23388 network=udp +49:14.608 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23389 network=tcp +49:15.112 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.112 [INFO] TestAgent: Endpoints down +--- PASS: TestHandleRemoteExecFailed (0.94s) +=== CONT TestShouldProcessUserEvent +2022-12-04T05:49:14.250Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:14.250Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.273Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:14.273Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.325 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:de0796df-43b4-89f5-ab54-722511a9d35a Address:127.0.0.1:23309}]" +49:14.325 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-de0796df-43b4-89f5-ab54-722511a9d35a.dc1 127.0.0.1 +49:14.326 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-de0796df-43b4-89f5-ab54-722511a9d35a 127.0.0.1 +2022-12-04T05:49:14.326Z [INFO] agent.router: Initializing LAN area manager +49:14.326 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23318 network=udp +49:14.325 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23309 [Follower]" leader= +49:14.326 [INFO] TestAgent.server: Adding LAN server: server="Node-de0796df-43b4-89f5-ab54-722511a9d35a (Addr: tcp/127.0.0.1:23309) (DC: dc1)" +49:14.326 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-de0796df-43b4-89f5-ab54-722511a9d35a.dc1 area=wan +49:14.326 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23318 network=tcp +49:14.327 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23319 network=tcp +49:14.327 [INFO] TestAgent: started state syncer +49:14.327 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23310 network=tcp +49:14.372 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.372 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23309 [Candidate]" term=2 +49:14.372 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.372 [DEBUG] TestAgent.server.raft: vote granted: from=de0796df-43b4-89f5-ab54-722511a9d35a term=2 tally=1 +49:14.372 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.372 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23309 [Leader]" +49:14.372 [INFO] TestAgent.server: cluster leadership acquired +49:14.372 [INFO] TestAgent.server: initializing acls +49:14.373 [INFO] TestAgent.server: New leader elected: payload=Node-de0796df-43b4-89f5-ab54-722511a9d35a +49:14.373 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:14.373 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:14.373 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:14.373 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:14.373 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:14.373 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:14.373 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-de0796df-43b4-89f5-ab54-722511a9d35a +49:14.373 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-de0796df-43b4-89f5-ab54-722511a9d35a.dc1 +49:14.374 [INFO] TestAgent.server: Updating LAN server: server="Node-de0796df-43b4-89f5-ab54-722511a9d35a (Addr: tcp/127.0.0.1:23309) (DC: dc1)" +49:14.374 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-de0796df-43b4-89f5-ab54-722511a9d35a.dc1 area=wan +49:14.374 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.375 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.375 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.375 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.375 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.375 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.375 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-de0796df-43b4-89f5-ab54-722511a9d35a +49:14.375 [INFO] TestAgent.server: member joined, marking health alive: member=Node-de0796df-43b4-89f5-ab54-722511a9d35a +49:14.376 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.376 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-de0796df-43b4-89f5-ab54-722511a9d35a +49:14.504 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:14.504 [INFO] TestAgent: Synced node info +49:14.689 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-de0796df-43b4-89f5-ab54-722511a9d35a +49:14.689 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-de0796df-43b4-89f5-ab54-722511a9d35a +49:14.690 [INFO] TestAgent: Requesting shutdown +49:14.690 [INFO] TestAgent.server: shutting down server +49:14.690 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.690 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.690 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:14.690 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:14.690 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.690 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.690 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.690 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.690 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.690 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.690 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:14.690 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:14.690 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:14.690 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.690 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:14.690Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.690Z [INFO] agent.router.manager: shutting down +49:14.691 [INFO] TestAgent: consul server down +49:14.691 [INFO] TestAgent: shutdown complete +49:14.691 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23318 network=tcp +49:14.691 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23318 network=udp +49:14.691 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23319 network=tcp +49:15.191 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.191 [INFO] TestAgent: Endpoints down +--- PASS: TestRemoteExecGetSpec_ACLAgentToken (1.02s) +=== CONT TestValidateUserEventParams +--- PASS: TestValidateUserEventParams (0.00s) +=== CONT TestUIGatewayServiceNodes_Ingress +2022-12-04T05:49:14.327Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:14.327Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.339Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:14.339Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.340 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:22847976-0a9a-3c07-db2e-21b822ac0359 Address:127.0.0.1:23372}]" +49:14.340 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23372 [Follower]" leader= +49:14.341 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-22847976-0a9a-3c07-db2e-21b822ac0359.dc1 127.0.0.1 +49:14.341 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-22847976-0a9a-3c07-db2e-21b822ac0359 127.0.0.1 +2022-12-04T05:49:14.341Z [INFO] agent.router: Initializing LAN area manager +49:14.341 [INFO] TestAgent.server: Adding LAN server: server="Node-22847976-0a9a-3c07-db2e-21b822ac0359 (Addr: tcp/127.0.0.1:23372) (DC: dc1)" +49:14.342 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-22847976-0a9a-3c07-db2e-21b822ac0359.dc1 area=wan +49:14.342 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23367 network=udp +49:14.342 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23367 network=tcp +49:14.342 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23368 network=tcp +49:14.342 [INFO] TestAgent: started state syncer +49:14.342 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23373 network=tcp +49:14.392 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:14.393 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.393 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23372 [Candidate]" term=2 +49:14.393 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.393 [DEBUG] TestAgent.server.raft: vote granted: from=22847976-0a9a-3c07-db2e-21b822ac0359 term=2 tally=1 +49:14.393 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.393 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23372 [Leader]" +49:14.393 [INFO] TestAgent.server: cluster leadership acquired +49:14.393 [INFO] TestAgent.server: New leader elected: payload=Node-22847976-0a9a-3c07-db2e-21b822ac0359 +49:14.393 [INFO] TestAgent.server: initializing acls +49:14.393 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:14.393 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:14.393 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:14.394 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:14.394 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:14.394 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:14.394 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-22847976-0a9a-3c07-db2e-21b822ac0359 +49:14.394 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-22847976-0a9a-3c07-db2e-21b822ac0359.dc1 +49:14.394 [INFO] TestAgent.server: Updating LAN server: server="Node-22847976-0a9a-3c07-db2e-21b822ac0359 (Addr: tcp/127.0.0.1:23372) (DC: dc1)" +49:14.394 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-22847976-0a9a-3c07-db2e-21b822ac0359.dc1 area=wan +49:14.394 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.395 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.395 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.395 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.395 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.395 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.395 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-22847976-0a9a-3c07-db2e-21b822ac0359 +49:14.395 [INFO] TestAgent.server: member joined, marking health alive: member=Node-22847976-0a9a-3c07-db2e-21b822ac0359 +49:14.396 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-22847976-0a9a-3c07-db2e-21b822ac0359 +49:14.396 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.567 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:14.567 [INFO] TestAgent: Synced node info +49:14.567 [DEBUG] TestAgent: Node info in sync +49:14.694 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-22847976-0a9a-3c07-db2e-21b822ac0359 +49:14.695 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=foo accessorID= +49:14.695 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=bar accessorID= +49:14.695 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=zip accessorID= +49:14.695 [INFO] TestAgent: Requesting shutdown +49:14.695 [INFO] TestAgent.server: shutting down server +49:14.695 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.695 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.695 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.695 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.695 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:14.695 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:14.695 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.695 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:14.695 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.695 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.695 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.695 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.695 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:14.695 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:14.695 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:14.695Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.695Z [INFO] agent.router.manager: shutting down +49:14.695 [INFO] TestAgent: consul server down +49:14.695 [INFO] TestAgent: shutdown complete +49:14.695 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23367 network=tcp +49:14.695 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23367 network=udp +49:14.695 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23368 network=tcp +49:15.195 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.195 [INFO] TestAgent: Endpoints down +--- PASS: TestUserEventToken (1.02s) +=== CONT TestUIGatewayServiceNodes_Terminating +2022-12-04T05:49:14.260Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.328Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.329 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5efbd134-d3bb-4a87-3e83-d2fe866e8187 Address:127.0.0.1:23400}]" +49:14.330 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5efbd134-d3bb-4a87-3e83-d2fe866e8187.dc1 127.0.0.1 +49:14.330 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5efbd134-d3bb-4a87-3e83-d2fe866e8187 127.0.0.1 +2022-12-04T05:49:14.330Z [INFO] agent.router: Initializing LAN area manager +49:14.330 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23400 [Follower]" leader= +49:14.330 [INFO] TestAgent.server: Adding LAN server: server="Node-5efbd134-d3bb-4a87-3e83-d2fe866e8187 (Addr: tcp/127.0.0.1:23400) (DC: dc1)" +49:14.331 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5efbd134-d3bb-4a87-3e83-d2fe866e8187.dc1 area=wan +49:14.331 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23395 network=tcp +49:14.331 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23395 network=udp +49:14.331 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23396 network=tcp +49:14.331 [INFO] TestAgent: started state syncer +49:14.331 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23401 network=tcp +49:14.367 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.367 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23400 [Candidate]" term=2 +49:14.367 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.367 [DEBUG] TestAgent.server.raft: vote granted: from=5efbd134-d3bb-4a87-3e83-d2fe866e8187 term=2 tally=1 +49:14.367 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.367 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23400 [Leader]" +49:14.367 [INFO] TestAgent.server: cluster leadership acquired +49:14.368 [INFO] TestAgent.server: New leader elected: payload=Node-5efbd134-d3bb-4a87-3e83-d2fe866e8187 +49:14.368 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23400 +49:14.368 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.369 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.369 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.369 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.369 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.369 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.369 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5efbd134-d3bb-4a87-3e83-d2fe866e8187 +49:14.369 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5efbd134-d3bb-4a87-3e83-d2fe866e8187 +49:14.370 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.662 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:14.663 [INFO] TestAgent: Synced node info +49:14.708 [INFO] TestAgent: Requesting shutdown +49:14.708 [INFO] TestAgent.server: shutting down server +49:14.708 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.708 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.708 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.708 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.708 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.708 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.708 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.708 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.708 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.708 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:14.708 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:14.708Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.708Z [INFO] agent.router.manager: shutting down +49:14.708 [INFO] TestAgent: consul server down +49:14.708 [INFO] TestAgent: shutdown complete +49:14.708 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23395 network=tcp +49:14.708 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23395 network=udp +49:14.708 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23396 network=tcp +49:15.208 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.208 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionCreate_NodeChecks (1.03s) +=== CONT TestKVSEndpoint_ListKeys +[WARN] freeport: 10 out of 66 pending ports are still in use; something probably didn't wait around for the port to be closed! +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:14.874Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.890Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.894 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7307b6f3-99c1-c91a-f1a6-fdbce2d866ca Address:127.0.0.1:23407}]" +49:14.894 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23407 [Follower]" leader= +49:14.894 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7307b6f3-99c1-c91a-f1a6-fdbce2d866ca.dc1 127.0.0.1 +49:14.895 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7307b6f3-99c1-c91a-f1a6-fdbce2d866ca 127.0.0.1 +2022-12-04T05:49:14.895Z [INFO] agent.router: Initializing LAN area manager +49:14.895 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23416 network=udp +49:14.895 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7307b6f3-99c1-c91a-f1a6-fdbce2d866ca.dc1 area=wan +49:14.895 [INFO] TestAgent.server: Adding LAN server: server="Node-7307b6f3-99c1-c91a-f1a6-fdbce2d866ca (Addr: tcp/127.0.0.1:23407) (DC: dc1)" +49:14.895 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23416 network=tcp +49:14.896 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23417 network=tcp +49:14.896 [INFO] TestAgent: started state syncer +49:14.896 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23408 network=tcp +49:14.949 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.949 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23407 [Candidate]" term=2 +49:14.949 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.949 [DEBUG] TestAgent.server.raft: vote granted: from=7307b6f3-99c1-c91a-f1a6-fdbce2d866ca term=2 tally=1 +49:14.949 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.949 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23407 [Leader]" +49:14.949 [INFO] TestAgent.server: cluster leadership acquired +49:14.949 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23407 +49:14.950 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.950 [INFO] TestAgent.server: New leader elected: payload=Node-7307b6f3-99c1-c91a-f1a6-fdbce2d866ca +49:14.952 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.952 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.952 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.952 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.952 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.952 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7307b6f3-99c1-c91a-f1a6-fdbce2d866ca +49:14.952 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7307b6f3-99c1-c91a-f1a6-fdbce2d866ca +49:14.952 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.979 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:14.980 [INFO] TestAgent: Synced node info +49:14.980 [DEBUG] TestAgent: Node info in sync +49:15.112 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:15.112 [INFO] TestAgent: Requesting shutdown +49:15.112 [INFO] TestAgent.server: shutting down server +49:15.112 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.112 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.112 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.112 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.112 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.113 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.113 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.113 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.113 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.113 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:15.113 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:15.113Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.113Z [INFO] agent.router.manager: shutting down +49:15.113 [INFO] TestAgent: consul server down +49:15.113 [INFO] TestAgent: shutdown complete +49:15.113 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23416 network=tcp +49:15.113 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23416 network=udp +49:15.113 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23417 network=tcp +49:15.613 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.613 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_Update (0.75s) +=== CONT TestUiServices +2022-12-04T05:49:15.026Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.038Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.040 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:92ea540a-099d-4508-a357-e3333a4d00eb Address:127.0.0.1:23435}]" +49:15.040 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23435 [Follower]" leader= +49:15.040 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-92ea540a-099d-4508-a357-e3333a4d00eb.dc1 127.0.0.1 +49:15.041 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-92ea540a-099d-4508-a357-e3333a4d00eb 127.0.0.1 +2022-12-04T05:49:15.041Z [INFO] agent.router: Initializing LAN area manager +49:15.041 [INFO] TestAgent.server: Adding LAN server: server="Node-92ea540a-099d-4508-a357-e3333a4d00eb (Addr: tcp/127.0.0.1:23435) (DC: dc1)" +49:15.041 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-92ea540a-099d-4508-a357-e3333a4d00eb.dc1 area=wan +49:15.041 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23444 network=udp +49:15.041 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23444 network=tcp +49:15.042 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23445 network=tcp +49:15.042 [INFO] TestAgent: started state syncer +49:15.042 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23436 network=tcp +49:15.097 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.097 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23435 [Candidate]" term=2 +49:15.097 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.097 [DEBUG] TestAgent.server.raft: vote granted: from=92ea540a-099d-4508-a357-e3333a4d00eb term=2 tally=1 +49:15.097 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.097 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23435 [Leader]" +49:15.098 [INFO] TestAgent.server: cluster leadership acquired +49:15.098 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23435 +49:15.099 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.100 [INFO] TestAgent.server: New leader elected: payload=Node-92ea540a-099d-4508-a357-e3333a4d00eb +49:15.101 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.101 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.101 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.101 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.101 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.101 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-92ea540a-099d-4508-a357-e3333a4d00eb +49:15.101 [INFO] TestAgent.server: member joined, marking health alive: member=Node-92ea540a-099d-4508-a357-e3333a4d00eb +49:15.102 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.220 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:15.220 [INFO] TestAgent: Requesting shutdown +49:15.220 [INFO] TestAgent.server: shutting down server +49:15.220 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.220 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.220 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.220 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.220 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.220 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.220 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:15.220 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.220 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.220 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:15.220 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.220 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:15.220Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.220Z [INFO] agent.router.manager: shutting down +49:15.221 [INFO] TestAgent: consul server down +49:15.221 [INFO] TestAgent: shutdown complete +49:15.221 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23444 network=tcp +49:15.221 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23444 network=udp +49:15.221 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23445 network=tcp +49:15.721 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.721 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Execute/#01 +2022-12-04T05:49:15.099Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.126Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.128 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1b1885a8-4d18-6c7f-d2d5-518d8c4febed Address:127.0.0.1:23463}]" +49:15.128 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23463 [Follower]" leader= +49:15.132 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1b1885a8-4d18-6c7f-d2d5-518d8c4febed.dc1 127.0.0.1 +49:15.133 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1b1885a8-4d18-6c7f-d2d5-518d8c4febed 127.0.0.1 +2022-12-04T05:49:15.133Z [INFO] agent.router: Initializing LAN area manager +49:15.133 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23458 network=udp +49:15.134 [INFO] TestAgent.server: Adding LAN server: server="Node-1b1885a8-4d18-6c7f-d2d5-518d8c4febed (Addr: tcp/127.0.0.1:23463) (DC: dc1)" +49:15.134 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1b1885a8-4d18-6c7f-d2d5-518d8c4febed.dc1 area=wan +49:15.134 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23458 network=tcp +49:15.134 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23459 network=tcp +49:15.134 [INFO] TestAgent: started state syncer +49:15.135 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23464 network=tcp +49:15.185 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.185 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23463 [Candidate]" term=2 +49:15.185 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.185 [DEBUG] TestAgent.server.raft: vote granted: from=1b1885a8-4d18-6c7f-d2d5-518d8c4febed term=2 tally=1 +49:15.185 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.185 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23463 [Leader]" +49:15.185 [INFO] TestAgent.server: cluster leadership acquired +49:15.185 [INFO] TestAgent.server: New leader elected: payload=Node-1b1885a8-4d18-6c7f-d2d5-518d8c4febed +49:15.185 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23463 +49:15.186 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.187 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.187 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.187 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.187 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.187 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.187 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1b1885a8-4d18-6c7f-d2d5-518d8c4febed +49:15.187 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1b1885a8-4d18-6c7f-d2d5-518d8c4febed +49:15.188 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.231 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:15.231 [INFO] TestAgent: Requesting shutdown +49:15.231 [INFO] TestAgent.server: shutting down server +49:15.231 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.232 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.232 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.232 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.232 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.232 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.232 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:15.232 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.232 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:15.232 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.232 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.232 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:15.232Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.232Z [INFO] agent.router.manager: shutting down +49:15.232 [INFO] TestAgent: consul server down +49:15.232 [INFO] TestAgent: shutdown complete +49:15.232 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23458 network=tcp +49:15.232 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23458 network=udp +49:15.232 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23459 network=tcp +49:15.732 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.732 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_Create (0.68s) +=== CONT TestKVSEndpoint_CAS +2022-12-04T05:49:14.916Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.932Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.933 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:96fa9afe-884e-f08f-9cff-86750a8d1593 Address:127.0.0.1:23428}]" +49:14.934 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23428 [Follower]" leader= +49:14.934 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-96fa9afe-884e-f08f-9cff-86750a8d1593.dc1 127.0.0.1 +49:14.935 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-96fa9afe-884e-f08f-9cff-86750a8d1593 127.0.0.1 +2022-12-04T05:49:14.935Z [INFO] agent.router: Initializing LAN area manager +49:14.935 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23409 network=udp +49:14.935 [INFO] TestAgent.server: Adding LAN server: server="Node-96fa9afe-884e-f08f-9cff-86750a8d1593 (Addr: tcp/127.0.0.1:23428) (DC: dc1)" +49:14.935 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-96fa9afe-884e-f08f-9cff-86750a8d1593.dc1 area=wan +49:14.935 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23409 network=tcp +49:14.936 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23410 network=tcp +49:14.936 [INFO] TestAgent: started state syncer +49:14.936 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23429 network=tcp +49:14.986 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.986 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23428 [Candidate]" term=2 +49:14.987 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.987 [DEBUG] TestAgent.server.raft: vote granted: from=96fa9afe-884e-f08f-9cff-86750a8d1593 term=2 tally=1 +49:14.987 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.987 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23428 [Leader]" +49:14.987 [INFO] TestAgent.server: cluster leadership acquired +49:14.987 [INFO] TestAgent.server: New leader elected: payload=Node-96fa9afe-884e-f08f-9cff-86750a8d1593 +49:14.987 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23428 +49:14.988 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.989 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.989 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.989 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.989 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.989 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.989 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-96fa9afe-884e-f08f-9cff-86750a8d1593 +49:14.989 [INFO] TestAgent.server: member joined, marking health alive: member=Node-96fa9afe-884e-f08f-9cff-86750a8d1593 +49:14.990 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.117 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:15.117 [INFO] TestAgent: Synced node info +49:15.236 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:15.237 [INFO] TestAgent: Requesting shutdown +49:15.237 [INFO] TestAgent.server: shutting down server +49:15.237 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.237 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.237 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.237 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.237 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.237 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.237 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.237 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.237 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.237 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:15.237 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:15.237Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.237Z [INFO] agent.router.manager: shutting down +49:15.237 [INFO] TestAgent: consul server down +49:15.237 [INFO] TestAgent: shutdown complete +49:15.237 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23409 network=tcp +49:15.237 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23409 network=udp +49:15.238 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23410 network=tcp +49:15.738 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.738 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Get/#01 +2022-12-04T05:49:14.941Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.951Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.952 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:90217e38-f2e2-1975-f5e9-d7eb2a096959 Address:127.0.0.1:23421}]" +49:14.952 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23421 [Follower]" leader= +49:14.953 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-90217e38-f2e2-1975-f5e9-d7eb2a096959.dc1 127.0.0.1 +49:14.953 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-90217e38-f2e2-1975-f5e9-d7eb2a096959 127.0.0.1 +2022-12-04T05:49:14.953Z [INFO] agent.router: Initializing LAN area manager +49:14.953 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-90217e38-f2e2-1975-f5e9-d7eb2a096959.dc1 area=wan +49:14.954 [INFO] TestAgent.server: Adding LAN server: server="Node-90217e38-f2e2-1975-f5e9-d7eb2a096959 (Addr: tcp/127.0.0.1:23421) (DC: dc1)" +49:14.954 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23430 network=udp +49:14.954 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23430 network=tcp +49:14.955 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23431 network=tcp +49:14.955 [INFO] TestAgent: started state syncer +49:14.955 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23422 network=tcp +49:15.023 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.023 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23421 [Candidate]" term=2 +49:15.023 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.023 [DEBUG] TestAgent.server.raft: vote granted: from=90217e38-f2e2-1975-f5e9-d7eb2a096959 term=2 tally=1 +49:15.023 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.023 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23421 [Leader]" +49:15.023 [INFO] TestAgent.server: cluster leadership acquired +49:15.023 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23421 +49:15.024 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.024 [INFO] TestAgent.server: New leader elected: payload=Node-90217e38-f2e2-1975-f5e9-d7eb2a096959 +49:15.025 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.025 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.025 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.025 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.025 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.025 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-90217e38-f2e2-1975-f5e9-d7eb2a096959 +49:15.025 [INFO] TestAgent.server: member joined, marking health alive: member=Node-90217e38-f2e2-1975-f5e9-d7eb2a096959 +49:15.026 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.286 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:15.286 [INFO] TestAgent: Requesting shutdown +49:15.286 [INFO] TestAgent.server: shutting down server +49:15.286 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.286 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.286 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.286 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.286 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.286 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.286 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.286 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:15.286 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.286 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.286 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:15.286 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:15.286Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.286Z [INFO] agent.router.manager: shutting down +49:15.286 [INFO] TestAgent: consul server down +49:15.286 [INFO] TestAgent: shutdown complete +49:15.286 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23430 network=tcp +49:15.286 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23430 network=udp +49:15.286 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23431 network=tcp +49:15.786 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.786 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Explain/#01 +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:15.016Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.028Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.029 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6b023eb3-60bf-c463-7914-8fc1877483a4 Address:127.0.0.1:23442}]" +49:15.030 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6b023eb3-60bf-c463-7914-8fc1877483a4.dc1 127.0.0.1 +49:15.030 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6b023eb3-60bf-c463-7914-8fc1877483a4 127.0.0.1 +2022-12-04T05:49:15.030Z [INFO] agent.router: Initializing LAN area manager +49:15.031 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23423 network=udp +49:15.031 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23442 [Follower]" leader= +49:15.031 [INFO] TestAgent.server: Adding LAN server: server="Node-6b023eb3-60bf-c463-7914-8fc1877483a4 (Addr: tcp/127.0.0.1:23442) (DC: dc1)" +49:15.031 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6b023eb3-60bf-c463-7914-8fc1877483a4.dc1 area=wan +49:15.031 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23423 network=tcp +49:15.032 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23424 network=tcp +49:15.032 [INFO] TestAgent: started state syncer +49:15.032 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23443 network=tcp +49:15.071 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.071 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23442 [Candidate]" term=2 +49:15.071 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.071 [DEBUG] TestAgent.server.raft: vote granted: from=6b023eb3-60bf-c463-7914-8fc1877483a4 term=2 tally=1 +49:15.071 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.071 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23442 [Leader]" +49:15.072 [INFO] TestAgent.server: cluster leadership acquired +49:15.072 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23442 +49:15.072 [INFO] TestAgent.server: New leader elected: payload=Node-6b023eb3-60bf-c463-7914-8fc1877483a4 +49:15.073 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.075 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.075 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.075 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.075 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.075 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.075 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6b023eb3-60bf-c463-7914-8fc1877483a4 +49:15.075 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6b023eb3-60bf-c463-7914-8fc1877483a4 +49:15.076 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.311 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:15.312 [INFO] TestAgent: Requesting shutdown +49:15.312 [INFO] TestAgent.server: shutting down server +49:15.312 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.312 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.312 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.312 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.312 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.312 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.312 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.312 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:15.312 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.312 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.312 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:15.312 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:15.312Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.312Z [INFO] agent.router.manager: shutting down +49:15.312 [INFO] TestAgent: consul server down +49:15.312 [INFO] TestAgent: shutdown complete +49:15.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23423 network=tcp +49:15.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23423 network=udp +49:15.312 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23424 network=tcp +49:15.813 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.813 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_ExecuteCached (0.81s) +=== CONT TestUiNodeInfo +2022-12-04T05:49:15.026Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.038Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.040 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:233ecb7a-ed75-327c-be83-f698e52f23b0 Address:127.0.0.1:23449}]" +49:15.040 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23449 [Follower]" leader= +49:15.040 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-233ecb7a-ed75-327c-be83-f698e52f23b0.dc1 127.0.0.1 +49:15.041 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-233ecb7a-ed75-327c-be83-f698e52f23b0 127.0.0.1 +2022-12-04T05:49:15.041Z [INFO] agent.router: Initializing LAN area manager +49:15.041 [INFO] TestAgent.server: Adding LAN server: server="Node-233ecb7a-ed75-327c-be83-f698e52f23b0 (Addr: tcp/127.0.0.1:23449) (DC: dc1)" +49:15.041 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23437 network=udp +49:15.041 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-233ecb7a-ed75-327c-be83-f698e52f23b0.dc1 area=wan +49:15.041 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23437 network=tcp +49:15.042 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23438 network=tcp +49:15.042 [INFO] TestAgent: started state syncer +49:15.042 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23450 network=tcp +49:15.080 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.080 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23449 [Candidate]" term=2 +49:15.080 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.080 [DEBUG] TestAgent.server.raft: vote granted: from=233ecb7a-ed75-327c-be83-f698e52f23b0 term=2 tally=1 +49:15.080 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.080 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23449 [Leader]" +49:15.080 [INFO] TestAgent.server: cluster leadership acquired +49:15.081 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23449 +49:15.082 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.082 [INFO] TestAgent.server: New leader elected: payload=Node-233ecb7a-ed75-327c-be83-f698e52f23b0 +49:15.102 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.102 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.102 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.102 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.102 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.102 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-233ecb7a-ed75-327c-be83-f698e52f23b0 +49:15.102 [INFO] TestAgent.server: member joined, marking health alive: member=Node-233ecb7a-ed75-327c-be83-f698e52f23b0 +49:15.103 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.338 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:15.339 [INFO] TestAgent: Requesting shutdown +49:15.339 [INFO] TestAgent.server: shutting down server +49:15.339 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.339 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.339 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.339 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.339 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.339 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.339 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.339 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.339 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:15.339 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.339 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:15.339 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:15.339Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.339Z [INFO] agent.router.manager: shutting down +49:15.339 [INFO] TestAgent: consul server down +49:15.339 [INFO] TestAgent: shutdown complete +49:15.339 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23437 network=tcp +49:15.339 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23437 network=udp +49:15.339 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23438 network=tcp +49:15.840 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.840 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_List/#01 +2022-12-04T05:49:15.206Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.230Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.234 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bf1d6e0f-cd21-d575-5d7b-0d8c7a136c52 Address:127.0.0.1:23498}]" +49:15.234 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23498 [Follower]" leader= +49:15.235 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bf1d6e0f-cd21-d575-5d7b-0d8c7a136c52.dc1 127.0.0.1 +49:15.240 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bf1d6e0f-cd21-d575-5d7b-0d8c7a136c52 127.0.0.1 +2022-12-04T05:49:15.240Z [INFO] agent.router: Initializing LAN area manager +49:15.241 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23493 network=udp +49:15.241 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bf1d6e0f-cd21-d575-5d7b-0d8c7a136c52.dc1 area=wan +49:15.241 [INFO] TestAgent.server: Adding LAN server: server="Node-bf1d6e0f-cd21-d575-5d7b-0d8c7a136c52 (Addr: tcp/127.0.0.1:23498) (DC: dc1)" +49:15.241 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23493 network=tcp +49:15.242 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23494 network=tcp +49:15.242 [INFO] TestAgent: started state syncer +49:15.242 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23499 network=tcp +49:15.280 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.280 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23498 [Candidate]" term=2 +49:15.280 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.280 [DEBUG] TestAgent.server.raft: vote granted: from=bf1d6e0f-cd21-d575-5d7b-0d8c7a136c52 term=2 tally=1 +49:15.280 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.280 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23498 [Leader]" +49:15.280 [INFO] TestAgent.server: cluster leadership acquired +49:15.280 [INFO] TestAgent.server: New leader elected: payload=Node-bf1d6e0f-cd21-d575-5d7b-0d8c7a136c52 +49:15.280 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23498 +49:15.281 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.282 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.282 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.282 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.282 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.282 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.282 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bf1d6e0f-cd21-d575-5d7b-0d8c7a136c52 +49:15.282 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bf1d6e0f-cd21-d575-5d7b-0d8c7a136c52 +49:15.283 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.422 [INFO] TestAgent: Requesting shutdown +49:15.422 [INFO] TestAgent.server: shutting down server +49:15.422 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.422 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.422 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.422 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.422 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.422 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.422 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.422 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:15.422 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.422 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:15.422 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.422 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:15.422Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.422Z [INFO] agent.router.manager: shutting down +49:15.422 [INFO] TestAgent: consul server down +49:15.422 [INFO] TestAgent: shutdown complete +49:15.422 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23493 network=tcp +49:15.422 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23493 network=udp +49:15.422 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23494 network=tcp +49:15.924 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.924 [INFO] TestAgent: Endpoints down +--- PASS: TestUIGatewayServiceNodes_Ingress (0.73s) +=== CONT TestKVSEndpoint_DELETE_CAS +2022-12-04T05:49:15.209Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.252Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.254 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7cc6c951-98a1-456e-5a25-bb8082273cfe Address:127.0.0.1:23505}]" +49:15.254 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23505 [Follower]" leader= +49:15.254 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7cc6c951-98a1-456e-5a25-bb8082273cfe.dc1 127.0.0.1 +49:15.254 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7cc6c951-98a1-456e-5a25-bb8082273cfe 127.0.0.1 +2022-12-04T05:49:15.254Z [INFO] agent.router: Initializing LAN area manager +49:15.255 [INFO] TestAgent.server: Adding LAN server: server="Node-7cc6c951-98a1-456e-5a25-bb8082273cfe (Addr: tcp/127.0.0.1:23505) (DC: dc1)" +49:15.255 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7cc6c951-98a1-456e-5a25-bb8082273cfe.dc1 area=wan +49:15.255 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23500 network=udp +49:15.255 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23500 network=tcp +49:15.255 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23501 network=tcp +49:15.255 [INFO] TestAgent: started state syncer +49:15.255 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23506 network=tcp +49:15.304 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.304 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23505 [Candidate]" term=2 +49:15.304 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.304 [DEBUG] TestAgent.server.raft: vote granted: from=7cc6c951-98a1-456e-5a25-bb8082273cfe term=2 tally=1 +49:15.304 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.304 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23505 [Leader]" +49:15.304 [INFO] TestAgent.server: cluster leadership acquired +49:15.304 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23505 +49:15.304 [INFO] TestAgent.server: New leader elected: payload=Node-7cc6c951-98a1-456e-5a25-bb8082273cfe +49:15.305 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.306 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.306 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.306 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.306 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.306 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.307 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7cc6c951-98a1-456e-5a25-bb8082273cfe +49:15.307 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7cc6c951-98a1-456e-5a25-bb8082273cfe +49:15.307 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.437 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:15.438 [INFO] TestAgent: Synced node info +49:15.438 [DEBUG] TestAgent: Node info in sync +49:15.451 [INFO] TestAgent: Requesting shutdown +49:15.451 [INFO] TestAgent.server: shutting down server +49:15.451 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.451 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.451 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.451 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.451 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.451 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.451 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.451 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.451 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.451 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:15.451 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:15.451Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.451Z [INFO] agent.router.manager: shutting down +49:15.451 [INFO] TestAgent: consul server down +49:15.451 [INFO] TestAgent: shutdown complete +49:15.451 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23500 network=tcp +49:15.451 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23500 network=udp +49:15.451 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23501 network=tcp +49:15.952 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.952 [INFO] TestAgent: Endpoints down +--- PASS: TestUIGatewayServiceNodes_Terminating (0.76s) +=== CONT TestUiNodes_Filter +2022-12-04T05:49:15.137Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.150Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.151 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2e75077f-4e33-3b54-9778-48920e824b35 Address:127.0.0.1:23491}]" +49:15.152 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2e75077f-4e33-3b54-9778-48920e824b35.dc1 127.0.0.1 +49:15.152 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2e75077f-4e33-3b54-9778-48920e824b35 127.0.0.1 +2022-12-04T05:49:15.153Z [INFO] agent.router: Initializing LAN area manager +49:15.153 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23491 [Follower]" leader= +49:15.153 [INFO] TestAgent.server: Adding LAN server: server="Node-2e75077f-4e33-3b54-9778-48920e824b35 (Addr: tcp/127.0.0.1:23491) (DC: dc1)" +49:15.153 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2e75077f-4e33-3b54-9778-48920e824b35.dc1 area=wan +49:15.153 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23479 network=tcp +49:15.153 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23479 network=udp +49:15.154 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23480 network=tcp +49:15.154 [INFO] TestAgent: started state syncer +49:15.154 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23492 network=tcp +49:15.216 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.216 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23491 [Candidate]" term=2 +49:15.217 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.217 [DEBUG] TestAgent.server.raft: vote granted: from=2e75077f-4e33-3b54-9778-48920e824b35 term=2 tally=1 +49:15.217 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.217 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23491 [Leader]" +49:15.217 [INFO] TestAgent.server: cluster leadership acquired +49:15.218 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23491 +49:15.219 [INFO] TestAgent.server: New leader elected: payload=Node-2e75077f-4e33-3b54-9778-48920e824b35 +49:15.220 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.222 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.222 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.222 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.222 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.222 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.222 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2e75077f-4e33-3b54-9778-48920e824b35 +49:15.222 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2e75077f-4e33-3b54-9778-48920e824b35 +49:15.223 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.269 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:15.270 [INFO] TestAgent: Synced node info +49:15.270 [DEBUG] TestAgent: Node info in sync +49:15.484 [INFO] TestAgent: Requesting shutdown +49:15.484 [INFO] TestAgent.server: shutting down server +49:15.484 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.484 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.484 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.484 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.484 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.484 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.484 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.484 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:15.484 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.484 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.484 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:15.484Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.485Z [INFO] agent.router.manager: shutting down +49:15.485 [INFO] TestAgent: consul server down +49:15.485 [INFO] TestAgent: shutdown complete +49:15.485 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23479 network=tcp +49:15.485 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23479 network=udp +49:15.485 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23480 network=tcp +49:15.987 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.987 [INFO] TestAgent: Endpoints down +--- PASS: TestShouldProcessUserEvent (0.87s) +=== CONT TestKVSEndpoint_Recurse +=== RUN TestUiServices/No_Filter +=== PAUSE TestUiServices/No_Filter +=== RUN TestUiServices/Filtered +2022-12-04T05:49:15.054Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.102Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.104 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:59d2d903-9f99-3f38-fa39-7bf0aa788295 Address:127.0.0.1:23456}]" +49:15.104 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23456 [Follower]" leader= +49:15.104 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-59d2d903-9f99-3f38-fa39-7bf0aa788295.dc1 127.0.0.1 +49:15.105 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-59d2d903-9f99-3f38-fa39-7bf0aa788295 127.0.0.1 +2022-12-04T05:49:15.105Z [INFO] agent.router: Initializing LAN area manager +49:15.105 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23451 network=udp +49:15.105 [INFO] TestAgent.server: Adding LAN server: server="Node-59d2d903-9f99-3f38-fa39-7bf0aa788295 (Addr: tcp/127.0.0.1:23456) (DC: dc1)" +49:15.105 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23451 network=tcp +49:15.105 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-59d2d903-9f99-3f38-fa39-7bf0aa788295.dc1 area=wan +49:15.106 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23452 network=tcp +49:15.106 [INFO] TestAgent: started state syncer +49:15.106 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23457 network=tcp +49:15.157 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.157 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23456 [Candidate]" term=2 +49:15.157 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.157 [DEBUG] TestAgent.server.raft: vote granted: from=59d2d903-9f99-3f38-fa39-7bf0aa788295 term=2 tally=1 +49:15.157 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.157 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23456 [Leader]" +49:15.157 [INFO] TestAgent.server: cluster leadership acquired +49:15.157 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23456 +49:15.157 [INFO] TestAgent.server: New leader elected: payload=Node-59d2d903-9f99-3f38-fa39-7bf0aa788295 +49:15.158 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.159 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.159 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.159 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.159 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.159 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.159 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-59d2d903-9f99-3f38-fa39-7bf0aa788295 +49:15.159 [INFO] TestAgent.server: member joined, marking health alive: member=Node-59d2d903-9f99-3f38-fa39-7bf0aa788295 +49:15.160 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.301 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:15.301 [INFO] TestAgent: Synced node info +49:15.518 [DEBUG] TestAgent.server: User event: event=deploy +49:15.518 [DEBUG] TestAgent.server: User event: event=deploy +49:15.518 [DEBUG] TestAgent: new event: event_name=deploy event_id=aa474e39-27ca-ec77-476a-7b2a52428770 +49:15.543 [INFO] TestAgent: Requesting shutdown +49:15.543 [INFO] TestAgent.server: shutting down server +49:15.543 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.543 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.543 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.543 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.543 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.543 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.543 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.543 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.543 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:15.543 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.543 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:15.543Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.543Z [INFO] agent.router.manager: shutting down +49:15.543 [INFO] TestAgent: consul server down +49:15.543 [INFO] TestAgent: shutdown complete +49:15.543 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23451 network=tcp +49:15.543 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23451 network=udp +49:15.544 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23452 network=tcp +49:16.044 [INFO] TestAgent: Waiting for endpoints to shut down +49:16.044 [INFO] TestAgent: Endpoints down +--- PASS: TestFireReceiveEvent (1.01s) +=== CONT TestKVSEndpoint_PUT_GET_DELETE +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:15.103Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.128Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.130 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:48c91ae2-7f90-df32-17e0-4500249a1485 Address:127.0.0.1:23470}]" +49:15.130 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-48c91ae2-7f90-df32-17e0-4500249a1485.dc1 127.0.0.1 +49:15.131 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-48c91ae2-7f90-df32-17e0-4500249a1485 127.0.0.1 +2022-12-04T05:49:15.131Z [INFO] agent.router: Initializing LAN area manager +49:15.131 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23465 network=udp +49:15.131 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23470 [Follower]" leader= +49:15.131 [INFO] TestAgent.server: Adding LAN server: server="Node-48c91ae2-7f90-df32-17e0-4500249a1485 (Addr: tcp/127.0.0.1:23470) (DC: dc1)" +49:15.131 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-48c91ae2-7f90-df32-17e0-4500249a1485.dc1 area=wan +49:15.132 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23465 network=tcp +49:15.132 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23466 network=tcp +49:15.132 [INFO] TestAgent: started state syncer +49:15.132 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23471 network=tcp +49:15.191 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.191 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23470 [Candidate]" term=2 +49:15.191 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.191 [DEBUG] TestAgent.server.raft: vote granted: from=48c91ae2-7f90-df32-17e0-4500249a1485 term=2 tally=1 +49:15.191 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.191 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23470 [Leader]" +49:15.191 [INFO] TestAgent.server: cluster leadership acquired +49:15.191 [INFO] TestAgent.server: New leader elected: payload=Node-48c91ae2-7f90-df32-17e0-4500249a1485 +49:15.191 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23470 +49:15.192 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.193 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.193 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.193 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.193 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.193 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.193 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-48c91ae2-7f90-df32-17e0-4500249a1485 +49:15.193 [INFO] TestAgent.server: member joined, marking health alive: member=Node-48c91ae2-7f90-df32-17e0-4500249a1485 +49:15.194 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.347 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:15.347 [INFO] TestAgent: Synced node info +49:15.347 [DEBUG] TestAgent: Node info in sync +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.554 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.555 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.556 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [DEBUG] TestAgent: new event: event_name=test event_id= +49:15.557 [INFO] TestAgent: Requesting shutdown +49:15.557 [INFO] TestAgent.server: shutting down server +49:15.557 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.557 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.557 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.557 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.557 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.557 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.557 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.557 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.557 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.557 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:15.557 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:15.557Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.557Z [INFO] agent.router.manager: shutting down +49:15.557 [INFO] TestAgent: consul server down +49:15.557 [INFO] TestAgent: shutdown complete +49:15.557 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23465 network=tcp +49:15.557 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23465 network=udp +49:15.557 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23466 network=tcp +49:16.072 [INFO] TestAgent: Waiting for endpoints to shut down +49:16.072 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:15.240Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.263Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.264 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2d979b6c-e32d-c279-2732-64c39e989b3e Address:127.0.0.1:23512}]" +49:15.264 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23512 [Follower]" leader= +49:15.265 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2d979b6c-e32d-c279-2732-64c39e989b3e.dc1 127.0.0.1 +49:15.266 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2d979b6c-e32d-c279-2732-64c39e989b3e 127.0.0.1 +2022-12-04T05:49:15.266Z [INFO] agent.router: Initializing LAN area manager +49:15.266 [INFO] TestAgent.server: Adding LAN server: server="Node-2d979b6c-e32d-c279-2732-64c39e989b3e (Addr: tcp/127.0.0.1:23512) (DC: dc1)" +49:15.266 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23507 network=udp +49:15.266 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2d979b6c-e32d-c279-2732-64c39e989b3e.dc1 area=wan +49:15.266 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23507 network=tcp +49:15.266 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23508 network=tcp +49:15.266 [INFO] TestAgent: started state syncer +49:15.266 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23513 network=tcp +49:15.322 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.322 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23512 [Candidate]" term=2 +49:15.322 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.322 [DEBUG] TestAgent.server.raft: vote granted: from=2d979b6c-e32d-c279-2732-64c39e989b3e term=2 tally=1 +49:15.322 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.322 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23512 [Leader]" +49:15.322 [INFO] TestAgent.server: cluster leadership acquired +49:15.323 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23512 +49:15.323 [INFO] TestAgent.server: New leader elected: payload=Node-2d979b6c-e32d-c279-2732-64c39e989b3e +49:15.323 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.325 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.325 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.325 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.325 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.325 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.325 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2d979b6c-e32d-c279-2732-64c39e989b3e +49:15.325 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2d979b6c-e32d-c279-2732-64c39e989b3e +49:15.327 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.385 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:15.385 [INFO] TestAgent: Synced node info +49:15.566 [INFO] TestAgent: Requesting shutdown +49:15.567 [INFO] TestAgent.server: shutting down server +49:15.567 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.567 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.567 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.567 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.567 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.567 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.567 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.567 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.567 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.567 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:15.567 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:15.567Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.567Z [INFO] agent.router.manager: shutting down +49:15.567 [INFO] TestAgent: consul server down +49:15.567 [INFO] TestAgent: shutdown complete +49:15.567 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23507 network=tcp +49:15.567 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23507 network=udp +49:15.567 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23508 network=tcp +49:16.072 [INFO] TestAgent: Waiting for endpoints to shut down +49:16.072 [INFO] TestAgent: Endpoints down +--- PASS: TestIngestUserEvent (1.01s) +--- PASS: TestKVSEndpoint_ListKeys (0.86s) +=== CONT TestUiNodes +=== CONT TestAgentKeyring_ACL +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:15.730Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.740Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.741 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8c20033c-9b7a-5d0f-7966-ee68460d0e7f Address:127.0.0.1:23526}]" +49:15.742 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23526 [Follower]" leader= +49:15.742 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8c20033c-9b7a-5d0f-7966-ee68460d0e7f.dc1 127.0.0.1 +49:15.742 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8c20033c-9b7a-5d0f-7966-ee68460d0e7f 127.0.0.1 +2022-12-04T05:49:15.742Z [INFO] agent.router: Initializing LAN area manager +49:15.743 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23521 network=udp +49:15.743 [INFO] TestAgent.server: Adding LAN server: server="Node-8c20033c-9b7a-5d0f-7966-ee68460d0e7f (Addr: tcp/127.0.0.1:23526) (DC: dc1)" +49:15.743 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8c20033c-9b7a-5d0f-7966-ee68460d0e7f.dc1 area=wan +49:15.743 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23521 network=tcp +49:15.743 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23522 network=tcp +49:15.743 [INFO] TestAgent: started state syncer +49:15.743 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23527 network=tcp +49:15.807 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.807 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23526 [Candidate]" term=2 +49:15.808 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.808 [DEBUG] TestAgent.server.raft: vote granted: from=8c20033c-9b7a-5d0f-7966-ee68460d0e7f term=2 tally=1 +49:15.808 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.808 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23526 [Leader]" +49:15.808 [INFO] TestAgent.server: cluster leadership acquired +49:15.808 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23526 +49:15.808 [INFO] TestAgent.server: New leader elected: payload=Node-8c20033c-9b7a-5d0f-7966-ee68460d0e7f +49:15.809 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.810 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.810 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.810 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.810 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.810 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.810 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8c20033c-9b7a-5d0f-7966-ee68460d0e7f +49:15.810 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8c20033c-9b7a-5d0f-7966-ee68460d0e7f +49:15.811 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.876 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:15.876 [INFO] TestAgent: Synced node info +49:15.895 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:15.895 [INFO] TestAgent: Requesting shutdown +49:15.895 [INFO] TestAgent.server: shutting down server +49:15.895 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.895 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.895 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.895 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.895 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.895 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.895 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.895 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:15.895 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.895 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.895 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:15.895Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.895Z [INFO] agent.router.manager: shutting down +49:15.895 [INFO] TestAgent: consul server down +49:15.895 [INFO] TestAgent: shutdown complete +49:15.895 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23521 network=tcp +49:15.896 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23521 network=udp +49:15.896 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23522 network=tcp +49:16.396 [INFO] TestAgent: Waiting for endpoints to shut down +49:16.396 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Execute/#02 +2022-12-04T05:49:15.743Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.752Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.753 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:02744dbd-52cc-4292-4633-bdf974d04e85 Address:127.0.0.1:23533}]" +49:15.754 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23533 [Follower]" leader= +49:15.754 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-02744dbd-52cc-4292-4633-bdf974d04e85.dc1 127.0.0.1 +49:15.754 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-02744dbd-52cc-4292-4633-bdf974d04e85 127.0.0.1 +2022-12-04T05:49:15.754Z [INFO] agent.router: Initializing LAN area manager +49:15.755 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23528 network=udp +49:15.755 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-02744dbd-52cc-4292-4633-bdf974d04e85.dc1 area=wan +49:15.755 [INFO] TestAgent.server: Adding LAN server: server="Node-02744dbd-52cc-4292-4633-bdf974d04e85 (Addr: tcp/127.0.0.1:23533) (DC: dc1)" +49:15.755 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23528 network=tcp +49:15.755 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23529 network=tcp +49:15.755 [INFO] TestAgent: started state syncer +49:15.755 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23534 network=tcp +49:15.823 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.823 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23533 [Candidate]" term=2 +49:15.823 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.823 [DEBUG] TestAgent.server.raft: vote granted: from=02744dbd-52cc-4292-4633-bdf974d04e85 term=2 tally=1 +49:15.823 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.823 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23533 [Leader]" +49:15.824 [INFO] TestAgent.server: cluster leadership acquired +49:15.824 [INFO] TestAgent.server: New leader elected: payload=Node-02744dbd-52cc-4292-4633-bdf974d04e85 +49:15.824 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23533 +49:15.824 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.826 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.826 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.826 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.826 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.826 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.826 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-02744dbd-52cc-4292-4633-bdf974d04e85 +49:15.826 [INFO] TestAgent.server: member joined, marking health alive: member=Node-02744dbd-52cc-4292-4633-bdf974d04e85 +49:15.827 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.977 [INFO] TestAgent: Requesting shutdown +49:15.977 [INFO] TestAgent.server: shutting down server +49:15.977 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.977 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.977 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.977 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.977 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.977 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.977 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.977 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:15.977 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.977 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.977 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:15.977 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:15.977Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.977Z [INFO] agent.router.manager: shutting down +49:15.977 [INFO] TestAgent: consul server down +49:15.977 [INFO] TestAgent: shutdown complete +49:15.977 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23528 network=tcp +49:15.977 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23528 network=udp +49:15.977 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23529 network=tcp +49:16.478 [INFO] TestAgent: Waiting for endpoints to shut down +49:16.478 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_CAS (0.75s) +=== CONT TestUiIndex +2022-12-04T05:49:15.850Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.859Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.860 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6b286bfe-d6ee-76b9-3929-78b29bf39ce4 Address:127.0.0.1:23561}]" +49:15.860 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23561 [Follower]" leader= +49:15.861 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6b286bfe-d6ee-76b9-3929-78b29bf39ce4.dc1 127.0.0.1 +49:15.861 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6b286bfe-d6ee-76b9-3929-78b29bf39ce4 127.0.0.1 +2022-12-04T05:49:15.862Z [INFO] agent.router: Initializing LAN area manager +49:15.862 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6b286bfe-d6ee-76b9-3929-78b29bf39ce4.dc1 area=wan +49:15.862 [INFO] TestAgent.server: Adding LAN server: server="Node-6b286bfe-d6ee-76b9-3929-78b29bf39ce4 (Addr: tcp/127.0.0.1:23561) (DC: dc1)" +49:15.862 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23556 network=udp +49:15.862 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23556 network=tcp +49:15.862 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23557 network=tcp +49:15.862 [INFO] TestAgent: started state syncer +49:15.862 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23562 network=tcp +49:15.912 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.912 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23561 [Candidate]" term=2 +49:15.912 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.912 [DEBUG] TestAgent.server.raft: vote granted: from=6b286bfe-d6ee-76b9-3929-78b29bf39ce4 term=2 tally=1 +49:15.912 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.912 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23561 [Leader]" +49:15.912 [INFO] TestAgent.server: cluster leadership acquired +49:15.912 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23561 +49:15.913 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.915 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.915 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.915 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.915 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.915 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.915 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6b286bfe-d6ee-76b9-3929-78b29bf39ce4 +49:15.915 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6b286bfe-d6ee-76b9-3929-78b29bf39ce4 +49:15.915 [INFO] TestAgent.server: New leader elected: payload=Node-6b286bfe-d6ee-76b9-3929-78b29bf39ce4 +49:15.916 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.988 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:15.988 [INFO] TestAgent: Requesting shutdown +49:15.988 [INFO] TestAgent.server: shutting down server +49:15.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.988 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.988 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.988 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.988 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.988 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.988 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.988 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:15.988 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:15.988 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.988 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:15.989Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.989Z [INFO] agent.router.manager: shutting down +49:15.989 [INFO] TestAgent: consul server down +49:15.989 [INFO] TestAgent: shutdown complete +49:15.989 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23556 network=tcp +49:15.989 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23556 network=udp +49:15.989 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23557 network=tcp +49:16.489 [INFO] TestAgent: Waiting for endpoints to shut down +49:16.489 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_List (1.48s) + --- PASS: TestPreparedQuery_List/#00 (0.83s) + --- PASS: TestPreparedQuery_List/#01 (0.65s) +=== CONT TestAgent_InitKeyring +--- PASS: TestAgent_InitKeyring (0.00s) +=== CONT TestTxnEndpoint_UpdateCheck +2022-12-04T05:49:15.824Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.834Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.835 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fbd909ee-39d5-2abd-5555-aca1cd54b40a Address:127.0.0.1:23554}]" +49:15.835 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23554 [Follower]" leader= +49:15.836 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fbd909ee-39d5-2abd-5555-aca1cd54b40a.dc1 127.0.0.1 +49:15.836 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fbd909ee-39d5-2abd-5555-aca1cd54b40a 127.0.0.1 +2022-12-04T05:49:15.836Z [INFO] agent.router: Initializing LAN area manager +49:15.836 [INFO] TestAgent.server: Adding LAN server: server="Node-fbd909ee-39d5-2abd-5555-aca1cd54b40a (Addr: tcp/127.0.0.1:23554) (DC: dc1)" +49:15.836 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fbd909ee-39d5-2abd-5555-aca1cd54b40a.dc1 area=wan +49:15.837 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23549 network=udp +49:15.837 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23549 network=tcp +49:15.837 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23550 network=tcp +49:15.837 [INFO] TestAgent: started state syncer +49:15.837 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23555 network=tcp +49:15.882 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.882 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23554 [Candidate]" term=2 +49:15.882 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.882 [DEBUG] TestAgent.server.raft: vote granted: from=fbd909ee-39d5-2abd-5555-aca1cd54b40a term=2 tally=1 +49:15.882 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.882 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23554 [Leader]" +49:15.882 [INFO] TestAgent.server: cluster leadership acquired +49:15.882 [INFO] TestAgent.server: New leader elected: payload=Node-fbd909ee-39d5-2abd-5555-aca1cd54b40a +49:15.882 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23554 +49:15.883 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.884 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.884 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.884 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.884 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.884 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.884 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fbd909ee-39d5-2abd-5555-aca1cd54b40a +49:15.884 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fbd909ee-39d5-2abd-5555-aca1cd54b40a +49:15.885 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.997 [INFO] TestAgent: Requesting shutdown +49:15.997 [INFO] TestAgent.server: shutting down server +49:15.997 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:15.997 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:15.997 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:15.997 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:15.997 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:15.997 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:15.997 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:15.997 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:15.997 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:15.997 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:15.997 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:15.997 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:15.997Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:15.997Z [INFO] agent.router.manager: shutting down +49:15.997 [INFO] TestAgent: consul server down +49:15.997 [INFO] TestAgent: shutdown complete +49:15.997 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23549 network=tcp +49:15.997 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23549 network=udp +49:15.997 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23550 network=tcp +49:16.498 [INFO] TestAgent: Waiting for endpoints to shut down +49:16.498 [INFO] TestAgent: Endpoints down +--- PASS: TestUiNodeInfo (0.69s) +=== CONT TestTxnEndpoint_KV_Actions +=== RUN TestTxnEndpoint_KV_Actions/#00 +=== CONT TestAgent_InmemKeyrings +=== RUN TestAgent_InmemKeyrings/no_keys +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:15.749Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.759Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.760 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1d57424c-888e-42e6-9d57-c7300268e159 Address:127.0.0.1:23540}]" +49:15.760 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23540 [Follower]" leader= +49:15.761 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1d57424c-888e-42e6-9d57-c7300268e159.dc1 127.0.0.1 +49:15.761 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1d57424c-888e-42e6-9d57-c7300268e159 127.0.0.1 +2022-12-04T05:49:15.761Z [INFO] agent.router: Initializing LAN area manager +49:15.761 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23535 network=udp +49:15.761 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1d57424c-888e-42e6-9d57-c7300268e159.dc1 area=wan +49:15.761 [INFO] TestAgent.server: Adding LAN server: server="Node-1d57424c-888e-42e6-9d57-c7300268e159 (Addr: tcp/127.0.0.1:23540) (DC: dc1)" +49:15.761 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23535 network=tcp +49:15.762 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23536 network=tcp +49:15.762 [INFO] TestAgent: started state syncer +49:15.762 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23541 network=tcp +49:15.811 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.811 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23540 [Candidate]" term=2 +49:15.811 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.811 [DEBUG] TestAgent.server.raft: vote granted: from=1d57424c-888e-42e6-9d57-c7300268e159 term=2 tally=1 +49:15.811 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.811 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23540 [Leader]" +49:15.811 [INFO] TestAgent.server: cluster leadership acquired +49:15.811 [INFO] TestAgent.server: New leader elected: payload=Node-1d57424c-888e-42e6-9d57-c7300268e159 +49:15.811 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23540 +49:15.812 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.813 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.813 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.813 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.813 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.813 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.813 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1d57424c-888e-42e6-9d57-c7300268e159 +49:15.813 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1d57424c-888e-42e6-9d57-c7300268e159 +49:15.814 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.156 [INFO] TestAgent: Requesting shutdown +49:16.156 [INFO] TestAgent.server: shutting down server +49:16.156 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.156 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.156 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.156 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.156 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.156 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.156 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.156 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:16.156 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.156 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:16.156 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.156 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:16.157Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.157Z [INFO] agent.router.manager: shutting down +49:16.157 [INFO] TestAgent: consul server down +49:16.157 [INFO] TestAgent: shutdown complete +49:16.157 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23535 network=tcp +49:16.157 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23535 network=udp +49:16.157 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23536 network=tcp +49:16.660 [INFO] TestAgent: Waiting for endpoints to shut down +49:16.660 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_Get (1.75s) + --- PASS: TestPreparedQuery_Get/#00 (0.83s) + --- PASS: TestPreparedQuery_Get/#01 (0.92s) +=== CONT TestTxnEndpoint_Bad_Size_Ops +2022-12-04T05:49:15.804Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.817Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.819 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:627e2302-7f59-0a62-4643-90405121f883 Address:127.0.0.1:23547}]" +49:15.819 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23547 [Follower]" leader= +49:15.819 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-627e2302-7f59-0a62-4643-90405121f883.dc1 127.0.0.1 +49:15.820 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-627e2302-7f59-0a62-4643-90405121f883 127.0.0.1 +2022-12-04T05:49:15.820Z [INFO] agent.router: Initializing LAN area manager +49:15.820 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-627e2302-7f59-0a62-4643-90405121f883.dc1 area=wan +49:15.820 [INFO] TestAgent.server: Adding LAN server: server="Node-627e2302-7f59-0a62-4643-90405121f883 (Addr: tcp/127.0.0.1:23547) (DC: dc1)" +49:15.820 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23542 network=tcp +49:15.820 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23542 network=udp +49:15.821 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23543 network=tcp +49:15.821 [INFO] TestAgent: started state syncer +49:15.821 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23548 network=tcp +49:15.887 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.887 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23547 [Candidate]" term=2 +49:15.887 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.887 [DEBUG] TestAgent.server.raft: vote granted: from=627e2302-7f59-0a62-4643-90405121f883 term=2 tally=1 +49:15.887 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.887 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23547 [Leader]" +49:15.888 [INFO] TestAgent.server: cluster leadership acquired +49:15.888 [INFO] TestAgent.server: New leader elected: payload=Node-627e2302-7f59-0a62-4643-90405121f883 +49:15.888 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23547 +49:15.888 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.889 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.889 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.889 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.889 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.889 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.889 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-627e2302-7f59-0a62-4643-90405121f883 +49:15.889 [INFO] TestAgent.server: member joined, marking health alive: member=Node-627e2302-7f59-0a62-4643-90405121f883 +49:15.890 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.208 [INFO] TestAgent: Requesting shutdown +49:16.208 [INFO] TestAgent.server: shutting down server +49:16.208 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.208 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.208 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.208 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.208 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.208 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.208 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.208 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:16.208 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:16.208 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.208 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.208 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:16.208Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.209Z [INFO] agent.router.manager: shutting down +49:16.209 [INFO] TestAgent: consul server down +49:16.209 [INFO] TestAgent: shutdown complete +49:16.209 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23542 network=tcp +49:16.209 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23542 network=udp +49:16.214 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23543 network=tcp +49:16.714 [INFO] TestAgent: Waiting for endpoints to shut down +49:16.714 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Explain/#02 +2022-12-04T05:49:16.012Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.037Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.038 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5dec06c7-1e3b-f248-0268-36672a35c76a Address:127.0.0.1:23582}]" +49:16.039 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5dec06c7-1e3b-f248-0268-36672a35c76a.dc1 127.0.0.1 +49:16.039 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5dec06c7-1e3b-f248-0268-36672a35c76a 127.0.0.1 +2022-12-04T05:49:16.039Z [INFO] agent.router: Initializing LAN area manager +49:16.040 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23577 network=udp +49:16.040 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23582 [Follower]" leader= +49:16.040 [INFO] TestAgent.server: Adding LAN server: server="Node-5dec06c7-1e3b-f248-0268-36672a35c76a (Addr: tcp/127.0.0.1:23582) (DC: dc1)" +49:16.040 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5dec06c7-1e3b-f248-0268-36672a35c76a.dc1 area=wan +49:16.040 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23577 network=tcp +49:16.041 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23578 network=tcp +49:16.041 [INFO] TestAgent: started state syncer +49:16.041 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23583 network=tcp +49:16.076 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.076 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23582 [Candidate]" term=2 +49:16.076 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.076 [DEBUG] TestAgent.server.raft: vote granted: from=5dec06c7-1e3b-f248-0268-36672a35c76a term=2 tally=1 +49:16.076 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.076 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23582 [Leader]" +49:16.076 [INFO] TestAgent.server: cluster leadership acquired +49:16.077 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23582 +49:16.078 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.079 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.079 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.079 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.079 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.079 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.079 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5dec06c7-1e3b-f248-0268-36672a35c76a +49:16.079 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5dec06c7-1e3b-f248-0268-36672a35c76a +49:16.080 [INFO] TestAgent.server: New leader elected: payload=Node-5dec06c7-1e3b-f248-0268-36672a35c76a +49:16.081 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.224 [INFO] TestAgent: Requesting shutdown +49:16.224 [INFO] TestAgent.server: shutting down server +49:16.224 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.224 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.224 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.224 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.224 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.224 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.224 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.224 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:16.225 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.225 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:16.225 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.225 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:16.225Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.225Z [INFO] agent.router.manager: shutting down +49:16.225 [INFO] TestAgent: consul server down +49:16.225 [INFO] TestAgent: shutdown complete +49:16.225 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23577 network=tcp +49:16.225 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23577 network=udp +49:16.225 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23578 network=tcp +49:16.725 [INFO] TestAgent: Waiting for endpoints to shut down +49:16.725 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_Recurse (0.74s) +=== CONT TestTxnEndpoint_Bad_Size_Net +=== RUN TestTxnEndpoint_Bad_Size_Net/exceeds_default_limits +2022-12-04T05:49:16.136Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:16.136Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.153Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:16.153Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.155 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d Address:127.0.0.1:23610}]" +49:16.156 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d.dc1 127.0.0.1 +49:16.156 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23610 [Follower]" leader= +49:16.157 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d 127.0.0.1 +2022-12-04T05:49:16.157Z [INFO] agent.router: Initializing LAN area manager +49:16.157 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23598 network=udp +49:16.157 [INFO] TestAgent.server: Adding LAN server: server="Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d (Addr: tcp/127.0.0.1:23610) (DC: dc1)" +49:16.157 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d.dc1 area=wan +49:16.157 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23598 network=tcp +49:16.158 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23599 network=tcp +49:16.158 [INFO] TestAgent: started state syncer +49:16.158 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23611 network=tcp +49:16.208 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:16.208 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.208 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23610 [Candidate]" term=2 +49:16.208 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.208 [DEBUG] TestAgent.server.raft: vote granted: from=0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d term=2 tally=1 +49:16.208 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.208 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23610 [Leader]" +49:16.208 [INFO] TestAgent.server: cluster leadership acquired +49:16.208 [INFO] TestAgent.server: initializing acls +49:16.209 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:16.209 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:16.209 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:16.209 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:16.209 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:16.209 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:16.209 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d +49:16.210 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d.dc1 +49:16.210 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d.dc1 area=wan +49:16.210 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.212 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.212 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.212 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.212 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.212 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.212 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d +49:16.212 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d +49:16.212 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.214 [INFO] TestAgent.server: New leader elected: payload=Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d +49:16.214 [INFO] TestAgent.server: Updating LAN server: server="Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d (Addr: tcp/127.0.0.1:23610) (DC: dc1)" +49:16.214 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d +49:16.233 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d +49:16.233 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +49:16.234 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d.dc1 +49:16.234 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +49:16.234 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d +49:16.234 [INFO] TestAgent.server.serf.wan: serf: Received install-key query +49:16.235 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d.dc1 +49:16.235 [INFO] TestAgent.server.serf.lan: serf: Received install-key query +49:16.235 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d +49:16.235 [INFO] TestAgent.server.serf.wan: serf: Received use-key query +49:16.236 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d.dc1 +49:16.236 [INFO] TestAgent.server.serf.lan: serf: Received use-key query +49:16.236 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d +49:16.236 [INFO] TestAgent.server.serf.wan: serf: Received remove-key query +49:16.236 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d.dc1 +49:16.236 [INFO] TestAgent.server.serf.lan: serf: Received remove-key query +49:16.237 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-0b12f6f3-d5c8-0ad6-66b8-9ba110b5747d +49:16.237 [INFO] TestAgent: Requesting shutdown +49:16.237 [INFO] TestAgent.server: shutting down server +49:16.237 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:16.237 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:16.237 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.237 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.237 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.237 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.237 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.237 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.237 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.237 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:16.237 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:16.237 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:16.237 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.237 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.237 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:16.237 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:16.237Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.237Z [INFO] agent.router.manager: shutting down +49:16.237 [INFO] TestAgent: consul server down +49:16.237 [INFO] TestAgent: shutdown complete +49:16.237 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23598 network=tcp +49:16.237 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23598 network=udp +49:16.237 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23599 network=tcp +49:16.737 [INFO] TestAgent: Waiting for endpoints to shut down +49:16.737 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentKeyring_ACL (0.66s) +=== CONT TestAgent_LoadKeyrings +=== RUN TestAgent_LoadKeyrings/no_keys +2022-12-04T05:49:16.088Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.135Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.140 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fff4c0d8-1514-4082-d20e-75480db03ea4 Address:127.0.0.1:23589}]" +49:16.141 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fff4c0d8-1514-4082-d20e-75480db03ea4.dc1 127.0.0.1 +49:16.142 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fff4c0d8-1514-4082-d20e-75480db03ea4 127.0.0.1 +2022-12-04T05:49:16.142Z [INFO] agent.router: Initializing LAN area manager +49:16.142 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23589 [Follower]" leader= +49:16.142 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23584 network=udp +49:16.142 [INFO] TestAgent.server: Adding LAN server: server="Node-fff4c0d8-1514-4082-d20e-75480db03ea4 (Addr: tcp/127.0.0.1:23589) (DC: dc1)" +49:16.142 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fff4c0d8-1514-4082-d20e-75480db03ea4.dc1 area=wan +49:16.142 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23584 network=tcp +49:16.143 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23585 network=tcp +49:16.143 [INFO] TestAgent: started state syncer +49:16.143 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23590 network=tcp +49:16.210 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.210 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23589 [Candidate]" term=2 +49:16.210 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.210 [DEBUG] TestAgent.server.raft: vote granted: from=fff4c0d8-1514-4082-d20e-75480db03ea4 term=2 tally=1 +49:16.210 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.210 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23589 [Leader]" +49:16.211 [INFO] TestAgent.server: cluster leadership acquired +49:16.211 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23589 +49:16.211 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.212 [INFO] TestAgent.server: New leader elected: payload=Node-fff4c0d8-1514-4082-d20e-75480db03ea4 +49:16.213 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.213 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.213 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.213 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.213 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.213 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fff4c0d8-1514-4082-d20e-75480db03ea4 +49:16.213 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fff4c0d8-1514-4082-d20e-75480db03ea4 +49:16.213 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.258 [INFO] TestAgent: Requesting shutdown +49:16.258 [INFO] TestAgent.server: shutting down server +49:16.258 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.258 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.258 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.259 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.259 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.259 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.259 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.259 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:16.259 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.259 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.259 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:16.259 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:16.259Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.259Z [INFO] agent.router.manager: shutting down +49:16.259 [INFO] TestAgent: consul server down +49:16.259 [INFO] TestAgent: shutdown complete +49:16.259 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23584 network=tcp +49:16.259 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23584 network=udp +49:16.259 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23585 network=tcp +49:16.760 [INFO] TestAgent: Waiting for endpoints to shut down +49:16.760 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_PUT_GET_DELETE (0.72s) +=== CONT TestTxnEndpoint_Bad_Size_Item +=== RUN TestTxnEndpoint_Bad_Size_Item/exceeds_default_limits +2022-12-04T05:49:15.946Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.980Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.982 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dc0941b6-7766-f992-43d0-e0a29586c54e Address:127.0.0.1:23568}]" +49:15.982 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23568 [Follower]" leader= +49:15.982 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dc0941b6-7766-f992-43d0-e0a29586c54e.dc1 127.0.0.1 +49:15.983 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dc0941b6-7766-f992-43d0-e0a29586c54e 127.0.0.1 +2022-12-04T05:49:15.983Z [INFO] agent.router: Initializing LAN area manager +49:15.983 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23563 network=udp +49:15.983 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dc0941b6-7766-f992-43d0-e0a29586c54e.dc1 area=wan +49:15.983 [INFO] TestAgent.server: Adding LAN server: server="Node-dc0941b6-7766-f992-43d0-e0a29586c54e (Addr: tcp/127.0.0.1:23568) (DC: dc1)" +49:15.983 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23563 network=tcp +49:15.988 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23564 network=tcp +49:15.988 [INFO] TestAgent: started state syncer +49:15.988 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23569 network=tcp +49:16.020 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.020 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23568 [Candidate]" term=2 +49:16.020 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.020 [DEBUG] TestAgent.server.raft: vote granted: from=dc0941b6-7766-f992-43d0-e0a29586c54e term=2 tally=1 +49:16.020 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.020 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23568 [Leader]" +49:16.020 [INFO] TestAgent.server: cluster leadership acquired +49:16.020 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23568 +49:16.021 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.022 [INFO] TestAgent.server: New leader elected: payload=Node-dc0941b6-7766-f992-43d0-e0a29586c54e +49:16.022 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.022 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.022 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.022 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.022 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.023 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dc0941b6-7766-f992-43d0-e0a29586c54e +49:16.023 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dc0941b6-7766-f992-43d0-e0a29586c54e +49:16.023 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.251 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:16.251 [INFO] TestAgent: Synced node info +49:16.251 [DEBUG] TestAgent: Node info in sync +49:16.290 [INFO] TestAgent: Requesting shutdown +49:16.290 [INFO] TestAgent.server: shutting down server +49:16.290 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.290 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.290 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.290 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.290 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.290 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.290 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.290 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.290 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.290 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:16.291 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:16.291Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.291Z [INFO] agent.router.manager: shutting down +49:16.291 [INFO] TestAgent: consul server down +49:16.291 [INFO] TestAgent: shutdown complete +49:16.291 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23563 network=tcp +49:16.291 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23563 network=udp +49:16.291 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23564 network=tcp +49:16.792 [INFO] TestAgent: Waiting for endpoints to shut down +49:16.792 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_DELETE_CAS (0.87s) +=== CONT TestIntentionsCreate_good +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:16.142Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.164Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.174 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:36de474a-009a-18eb-877a-3a8a7e6dcfcf Address:127.0.0.1:23596}]" +49:16.174 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23596 [Follower]" leader= +49:16.174 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-36de474a-009a-18eb-877a-3a8a7e6dcfcf.dc1 127.0.0.1 +49:16.175 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-36de474a-009a-18eb-877a-3a8a7e6dcfcf 127.0.0.1 +2022-12-04T05:49:16.175Z [INFO] agent.router: Initializing LAN area manager +49:16.175 [INFO] TestAgent.server: Adding LAN server: server="Node-36de474a-009a-18eb-877a-3a8a7e6dcfcf (Addr: tcp/127.0.0.1:23596) (DC: dc1)" +49:16.175 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-36de474a-009a-18eb-877a-3a8a7e6dcfcf.dc1 area=wan +49:16.175 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23591 network=udp +49:16.175 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23591 network=tcp +49:16.176 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23592 network=tcp +49:16.176 [INFO] TestAgent: started state syncer +49:16.176 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23597 network=tcp +49:16.242 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.242 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23596 [Candidate]" term=2 +49:16.242 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.242 [DEBUG] TestAgent.server.raft: vote granted: from=36de474a-009a-18eb-877a-3a8a7e6dcfcf term=2 tally=1 +49:16.242 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.242 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23596 [Leader]" +49:16.242 [INFO] TestAgent.server: cluster leadership acquired +49:16.242 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23596 +49:16.243 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.244 [INFO] TestAgent.server: New leader elected: payload=Node-36de474a-009a-18eb-877a-3a8a7e6dcfcf +49:16.245 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.245 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.245 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.245 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.245 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.245 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-36de474a-009a-18eb-877a-3a8a7e6dcfcf +49:16.245 [INFO] TestAgent.server: member joined, marking health alive: member=Node-36de474a-009a-18eb-877a-3a8a7e6dcfcf +49:16.246 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.266 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:16.266 [INFO] TestAgent: Synced node info +49:16.266 [DEBUG] TestAgent: Node info in sync +49:16.418 [INFO] TestAgent: Requesting shutdown +49:16.418 [INFO] TestAgent.server: shutting down server +49:16.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.418 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.418 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.418 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.418 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.418 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.418 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.418 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.419 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:16.419 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:16.419Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.419Z [INFO] agent.router.manager: shutting down +49:16.419 [INFO] TestAgent: consul server down +49:16.419 [INFO] TestAgent: shutdown complete +49:16.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23591 network=tcp +49:16.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23591 network=udp +49:16.419 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23592 network=tcp +49:16.919 [INFO] TestAgent: Waiting for endpoints to shut down +49:16.919 [INFO] TestAgent: Endpoints down +--- PASS: TestUiNodes (0.85s) +=== CONT TestTxnEndpoint_Bad_JSON +2022-12-04T05:49:15.974Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.006Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.012 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7de414b1-3f79-866a-79b7-ab286b3be2c9 Address:127.0.0.1:23575}]" +49:16.013 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7de414b1-3f79-866a-79b7-ab286b3be2c9.dc1 127.0.0.1 +49:16.013 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23575 [Follower]" leader= +49:16.020 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7de414b1-3f79-866a-79b7-ab286b3be2c9 127.0.0.1 +2022-12-04T05:49:16.021Z [INFO] agent.router: Initializing LAN area manager +49:16.021 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23570 network=udp +49:16.021 [INFO] TestAgent.server: Adding LAN server: server="Node-7de414b1-3f79-866a-79b7-ab286b3be2c9 (Addr: tcp/127.0.0.1:23575) (DC: dc1)" +49:16.021 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7de414b1-3f79-866a-79b7-ab286b3be2c9.dc1 area=wan +49:16.021 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23570 network=tcp +49:16.022 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23571 network=tcp +49:16.022 [INFO] TestAgent: started state syncer +49:16.022 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23576 network=tcp +49:16.080 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.084 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23575 [Candidate]" term=2 +49:16.085 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.085 [DEBUG] TestAgent.server.raft: vote granted: from=7de414b1-3f79-866a-79b7-ab286b3be2c9 term=2 tally=1 +49:16.085 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.085 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23575 [Leader]" +49:16.085 [INFO] TestAgent.server: cluster leadership acquired +49:16.086 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23575 +49:16.086 [INFO] TestAgent.server: New leader elected: payload=Node-7de414b1-3f79-866a-79b7-ab286b3be2c9 +49:16.089 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.124 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.124 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.124 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.124 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.124 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.124 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7de414b1-3f79-866a-79b7-ab286b3be2c9 +49:16.124 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7de414b1-3f79-866a-79b7-ab286b3be2c9 +49:16.127 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.417 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:16.418 [INFO] TestAgent: Synced node info +49:16.418 [DEBUG] TestAgent: Node info in sync +49:16.425 [INFO] TestAgent: Requesting shutdown +49:16.425 [INFO] TestAgent.server: shutting down server +49:16.425 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.425 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.425 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.425 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.425 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.426 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.426 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.426 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.426 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:16.426 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.426 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:16.426Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.426Z [INFO] agent.router.manager: shutting down +49:16.426 [INFO] TestAgent: consul server down +49:16.426 [INFO] TestAgent: shutdown complete +49:16.426 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23570 network=tcp +49:16.426 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23570 network=udp +49:16.426 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23571 network=tcp +49:16.927 [INFO] TestAgent: Waiting for endpoints to shut down +49:16.927 [INFO] TestAgent: Endpoints down +--- PASS: TestUiNodes_Filter (0.97s) +=== CONT TestStatusPeersSecondary +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:16.501Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.532Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.535 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f13f875e-f99b-4412-f278-93378ef61bc1 Address:127.0.0.1:23617}]" +49:16.535 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23617 [Follower]" leader= +49:16.536 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f13f875e-f99b-4412-f278-93378ef61bc1.dc1 127.0.0.1 +49:16.542 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f13f875e-f99b-4412-f278-93378ef61bc1 127.0.0.1 +2022-12-04T05:49:16.542Z [INFO] agent.router: Initializing LAN area manager +49:16.542 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23626 network=udp +49:16.543 [INFO] TestAgent.server: Adding LAN server: server="Node-f13f875e-f99b-4412-f278-93378ef61bc1 (Addr: tcp/127.0.0.1:23617) (DC: dc1)" +49:16.543 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f13f875e-f99b-4412-f278-93378ef61bc1.dc1 area=wan +49:16.543 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23626 network=tcp +49:16.543 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23627 network=tcp +49:16.543 [INFO] TestAgent: started state syncer +49:16.543 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23618 network=tcp +49:16.584 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.584 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23617 [Candidate]" term=2 +49:16.584 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.584 [DEBUG] TestAgent.server.raft: vote granted: from=f13f875e-f99b-4412-f278-93378ef61bc1 term=2 tally=1 +49:16.584 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.584 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23617 [Leader]" +49:16.584 [INFO] TestAgent.server: cluster leadership acquired +49:16.584 [INFO] TestAgent.server: New leader elected: payload=Node-f13f875e-f99b-4412-f278-93378ef61bc1 +49:16.584 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23617 +49:16.585 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.587 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.587 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.587 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.587 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.587 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.587 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f13f875e-f99b-4412-f278-93378ef61bc1 +49:16.587 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f13f875e-f99b-4412-f278-93378ef61bc1 +49:16.587 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.608 [INFO] TestAgent: Requesting shutdown +49:16.608 [INFO] TestAgent.server: shutting down server +49:16.608 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.608 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.608 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.608 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.608 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.608 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.608 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:16.608 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:16.608 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.608 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.608 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.608 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:16.608Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.608Z [INFO] agent.router.manager: shutting down +49:16.608 [INFO] TestAgent: consul server down +49:16.608 [INFO] TestAgent: shutdown complete +49:16.609 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23626 network=tcp +49:16.609 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23626 network=udp +49:16.609 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23627 network=tcp +49:17.109 [INFO] TestAgent: Waiting for endpoints to shut down +49:17.109 [INFO] TestAgent: Endpoints down +--- PASS: TestTxnEndpoint_UpdateCheck (0.62s) +=== CONT TestIntentionsSpecificDelete_good +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:16.547Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.561Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.562 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:690bb0c3-f914-a533-5053-5c3772dc59c7 Address:127.0.0.1:23631}]" +49:16.562 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23631 [Follower]" leader= +49:16.563 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-690bb0c3-f914-a533-5053-5c3772dc59c7.dc1 127.0.0.1 +49:16.563 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-690bb0c3-f914-a533-5053-5c3772dc59c7 127.0.0.1 +2022-12-04T05:49:16.564Z [INFO] agent.router: Initializing LAN area manager +49:16.564 [INFO] TestAgent.server: Adding LAN server: server="Node-690bb0c3-f914-a533-5053-5c3772dc59c7 (Addr: tcp/127.0.0.1:23631) (DC: dc1)" +49:16.564 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-690bb0c3-f914-a533-5053-5c3772dc59c7.dc1 area=wan +49:16.564 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23640 network=udp +49:16.564 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23640 network=tcp +49:16.564 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23641 network=tcp +49:16.564 [INFO] TestAgent: started state syncer +49:16.564 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23632 network=tcp +49:16.614 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.614 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23631 [Candidate]" term=2 +49:16.615 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.615 [DEBUG] TestAgent.server.raft: vote granted: from=690bb0c3-f914-a533-5053-5c3772dc59c7 term=2 tally=1 +49:16.615 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.615 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23631 [Leader]" +49:16.615 [INFO] TestAgent.server: cluster leadership acquired +49:16.615 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23631 +49:16.615 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.616 [INFO] TestAgent.server: New leader elected: payload=Node-690bb0c3-f914-a533-5053-5c3772dc59c7 +49:16.617 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.617 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.617 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.617 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.617 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.617 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-690bb0c3-f914-a533-5053-5c3772dc59c7 +49:16.617 [INFO] TestAgent.server: member joined, marking health alive: member=Node-690bb0c3-f914-a533-5053-5c3772dc59c7 +49:16.618 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.812 [INFO] TestAgent: Requesting shutdown +49:16.812 [INFO] TestAgent.server: shutting down server +49:16.812 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.812 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.812 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.812 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.812 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.812 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:16.812 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.812 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.812 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.812 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.812 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:16.812 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:16.812Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.812Z [INFO] agent.router.manager: shutting down +49:16.812 [INFO] TestAgent: consul server down +49:16.812 [INFO] TestAgent: shutdown complete +49:16.812 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23640 network=tcp +49:16.812 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23640 network=udp +49:16.812 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23641 network=tcp +49:17.314 [INFO] TestAgent: Waiting for endpoints to shut down +49:17.314 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_InmemKeyrings/server_with_keys +2022-12-04T05:49:16.421Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.446Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.448 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:76d2d531-da0d-ef64-01db-9778c55afbad Address:127.0.0.1:23603}]" +49:16.448 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23603 [Follower]" leader= +49:16.449 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-76d2d531-da0d-ef64-01db-9778c55afbad.dc1 127.0.0.1 +49:16.450 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-76d2d531-da0d-ef64-01db-9778c55afbad 127.0.0.1 +2022-12-04T05:49:16.450Z [INFO] agent.router: Initializing LAN area manager +49:16.450 [INFO] TestAgent.server: Adding LAN server: server="Node-76d2d531-da0d-ef64-01db-9778c55afbad (Addr: tcp/127.0.0.1:23603) (DC: dc1)" +49:16.450 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-76d2d531-da0d-ef64-01db-9778c55afbad.dc1 area=wan +49:16.450 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23612 network=udp +49:16.450 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23612 network=tcp +49:16.451 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23613 network=tcp +49:16.451 [INFO] TestAgent: started state syncer +49:16.451 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23604 network=tcp +49:16.494 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.494 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23603 [Candidate]" term=2 +49:16.495 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.495 [DEBUG] TestAgent.server.raft: vote granted: from=76d2d531-da0d-ef64-01db-9778c55afbad term=2 tally=1 +49:16.495 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.495 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23603 [Leader]" +49:16.495 [INFO] TestAgent.server: cluster leadership acquired +49:16.495 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23603 +49:16.496 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.496 [INFO] TestAgent.server: New leader elected: payload=Node-76d2d531-da0d-ef64-01db-9778c55afbad +49:16.497 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.497 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.497 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.497 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.497 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.497 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-76d2d531-da0d-ef64-01db-9778c55afbad +49:16.497 [INFO] TestAgent.server: member joined, marking health alive: member=Node-76d2d531-da0d-ef64-01db-9778c55afbad +49:16.498 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.755 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:16.756 [INFO] TestAgent: Synced node info +49:16.756 [DEBUG] TestAgent: Node info in sync +49:16.828 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:16.828 [INFO] TestAgent: Requesting shutdown +49:16.828 [INFO] TestAgent.server: shutting down server +49:16.828 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.828 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.828 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.828 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.828 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.828 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.828 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.828 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.828 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.828 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:16.828 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:16.829Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.829Z [INFO] agent.router.manager: shutting down +49:16.829 [INFO] TestAgent: consul server down +49:16.829 [INFO] TestAgent: shutdown complete +49:16.829 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23612 network=tcp +49:16.829 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23612 network=udp +49:16.829 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23613 network=tcp +49:17.329 [INFO] TestAgent: Waiting for endpoints to shut down +49:17.329 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Execute/#03 +2022-12-04T05:49:16.494Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.507Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.525 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a7580069-94fe-26e0-f997-85efd3585cf7 Address:127.0.0.1:23624}]" +49:16.526 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a7580069-94fe-26e0-f997-85efd3585cf7.dc1 127.0.0.1 +49:16.526 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a7580069-94fe-26e0-f997-85efd3585cf7 127.0.0.1 +2022-12-04T05:49:16.526Z [INFO] agent.router: Initializing LAN area manager +49:16.527 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23605 network=udp +49:16.527 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23624 [Follower]" leader= +49:16.528 [INFO] TestAgent.server: Adding LAN server: server="Node-a7580069-94fe-26e0-f997-85efd3585cf7 (Addr: tcp/127.0.0.1:23624) (DC: dc1)" +49:16.528 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a7580069-94fe-26e0-f997-85efd3585cf7.dc1 area=wan +49:16.529 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23605 network=tcp +49:16.530 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23606 network=tcp +49:16.530 [INFO] TestAgent: started state syncer +49:16.530 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23625 network=tcp +49:16.586 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.586 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23624 [Candidate]" term=2 +49:16.586 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.586 [DEBUG] TestAgent.server.raft: vote granted: from=a7580069-94fe-26e0-f997-85efd3585cf7 term=2 tally=1 +49:16.586 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.586 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23624 [Leader]" +49:16.586 [INFO] TestAgent.server: cluster leadership acquired +49:16.586 [INFO] TestAgent.server: New leader elected: payload=Node-a7580069-94fe-26e0-f997-85efd3585cf7 +49:16.587 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23624 +49:16.587 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.589 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.589 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.589 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.589 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.589 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.589 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a7580069-94fe-26e0-f997-85efd3585cf7 +49:16.589 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a7580069-94fe-26e0-f997-85efd3585cf7 +49:16.589 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.851 [INFO] TestAgent: Requesting shutdown +49:16.851 [INFO] TestAgent.server: shutting down server +49:16.851 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.851 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.851 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.851 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.851 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.851 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:16.851 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:16.851 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.851 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.851 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.851 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.851 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:16.851Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.851Z [INFO] agent.router.manager: shutting down +49:16.851 [INFO] TestAgent: consul server down +49:16.851 [INFO] TestAgent: shutdown complete +49:16.851 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23605 network=tcp +49:16.851 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23605 network=udp +49:16.851 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23606 network=tcp +49:17.351 [INFO] TestAgent: Waiting for endpoints to shut down +49:17.351 [INFO] TestAgent: Endpoints down +--- PASS: TestUiIndex (0.87s) +=== CONT TestIntentionsSpecificUpdate_good +2022-12-04T05:49:16.749Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.761Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.763 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:143979ca-aa53-b50d-19d5-746ddb87413c Address:127.0.0.1:23659}]" +49:16.763 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-143979ca-aa53-b50d-19d5-746ddb87413c.dc1 127.0.0.1 +49:16.764 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-143979ca-aa53-b50d-19d5-746ddb87413c 127.0.0.1 +2022-12-04T05:49:16.764Z [INFO] agent.router: Initializing LAN area manager +49:16.764 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23659 [Follower]" leader= +49:16.764 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23675 network=tcp +49:16.764 [INFO] TestAgent.server: Adding LAN server: server="Node-143979ca-aa53-b50d-19d5-746ddb87413c (Addr: tcp/127.0.0.1:23659) (DC: dc1)" +49:16.764 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-143979ca-aa53-b50d-19d5-746ddb87413c.dc1 area=wan +49:16.764 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23675 network=udp +49:16.765 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23676 network=tcp +49:16.765 [INFO] TestAgent: started state syncer +49:16.765 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23660 network=tcp +49:16.816 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.816 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23659 [Candidate]" term=2 +49:16.816 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.816 [DEBUG] TestAgent.server.raft: vote granted: from=143979ca-aa53-b50d-19d5-746ddb87413c term=2 tally=1 +49:16.816 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.816 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23659 [Leader]" +49:16.816 [INFO] TestAgent.server: cluster leadership acquired +49:16.816 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23659 +49:16.816 [INFO] TestAgent.server: New leader elected: payload=Node-143979ca-aa53-b50d-19d5-746ddb87413c +49:16.817 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.818 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.818 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.818 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.818 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.818 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.818 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-143979ca-aa53-b50d-19d5-746ddb87413c +49:16.818 [INFO] TestAgent.server: member joined, marking health alive: member=Node-143979ca-aa53-b50d-19d5-746ddb87413c +49:16.819 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.881 [INFO] TestAgent: Requesting shutdown +49:16.881 [INFO] TestAgent.server: shutting down server +49:16.881 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.881 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.881 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.881 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.881 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.881 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:16.881 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.881 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.881 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.881 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.881 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:16.881 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:16.881Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.881Z [INFO] agent.router.manager: shutting down +49:16.881 [INFO] TestAgent: consul server down +49:16.881 [INFO] TestAgent: shutdown complete +49:16.881 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23675 network=tcp +49:16.881 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23675 network=udp +49:16.881 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23676 network=tcp +49:17.382 [INFO] TestAgent: Waiting for endpoints to shut down +49:17.382 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_LoadKeyrings/server_with_keys +2022-12-04T05:49:16.522Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.549Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.551 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:552b78cb-5335-1d0a-83ee-8da0577b95d0 Address:127.0.0.1:23638}]" +49:16.551 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23638 [Follower]" leader= +49:16.551 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-552b78cb-5335-1d0a-83ee-8da0577b95d0.dc1 127.0.0.1 +49:16.552 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-552b78cb-5335-1d0a-83ee-8da0577b95d0 127.0.0.1 +2022-12-04T05:49:16.552Z [INFO] agent.router: Initializing LAN area manager +49:16.552 [INFO] TestAgent.server: Adding LAN server: server="Node-552b78cb-5335-1d0a-83ee-8da0577b95d0 (Addr: tcp/127.0.0.1:23638) (DC: dc1)" +49:16.552 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-552b78cb-5335-1d0a-83ee-8da0577b95d0.dc1 area=wan +49:16.552 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23619 network=udp +49:16.552 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23619 network=tcp +49:16.553 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23620 network=tcp +49:16.553 [INFO] TestAgent: started state syncer +49:16.553 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23639 network=tcp +49:16.614 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.614 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23638 [Candidate]" term=2 +49:16.614 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.614 [DEBUG] TestAgent.server.raft: vote granted: from=552b78cb-5335-1d0a-83ee-8da0577b95d0 term=2 tally=1 +49:16.614 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.614 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23638 [Leader]" +49:16.614 [INFO] TestAgent.server: cluster leadership acquired +49:16.614 [INFO] TestAgent.server: New leader elected: payload=Node-552b78cb-5335-1d0a-83ee-8da0577b95d0 +49:16.614 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23638 +49:16.615 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.616 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.616 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.616 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.616 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.616 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.616 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-552b78cb-5335-1d0a-83ee-8da0577b95d0 +49:16.616 [INFO] TestAgent.server: member joined, marking health alive: member=Node-552b78cb-5335-1d0a-83ee-8da0577b95d0 +49:16.617 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.805 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:16.805 [INFO] TestAgent: Synced node info +49:16.882 [INFO] TestAgent: Requesting shutdown +49:16.882 [INFO] TestAgent.server: shutting down server +49:16.882 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.882 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.882 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.882 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.882 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.882 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.882 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.882 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.882 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:16.882 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.882 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:16.883Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.883Z [INFO] agent.router.manager: shutting down +49:16.883 [INFO] TestAgent: consul server down +49:16.883 [INFO] TestAgent: shutdown complete +49:16.883 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23619 network=tcp +49:16.883 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23619 network=udp +49:16.883 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23620 network=tcp +49:17.383 [INFO] TestAgent: Waiting for endpoints to shut down +49:17.383 [INFO] TestAgent: Endpoints down +=== RUN TestTxnEndpoint_KV_Actions/#01 +2022-12-04T05:49:16.724Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.734Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.735 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9a3c1daf-b4f4-64e1-cf24-e6171f5deb8d Address:127.0.0.1:23666}]" +49:16.735 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23666 [Follower]" leader= +49:16.736 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9a3c1daf-b4f4-64e1-cf24-e6171f5deb8d.dc1 127.0.0.1 +49:16.736 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9a3c1daf-b4f4-64e1-cf24-e6171f5deb8d 127.0.0.1 +2022-12-04T05:49:16.736Z [INFO] agent.router: Initializing LAN area manager +49:16.736 [INFO] TestAgent.server: Adding LAN server: server="Node-9a3c1daf-b4f4-64e1-cf24-e6171f5deb8d (Addr: tcp/127.0.0.1:23666) (DC: dc1)" +49:16.737 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9a3c1daf-b4f4-64e1-cf24-e6171f5deb8d.dc1 area=wan +49:16.737 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23654 network=udp +49:16.737 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23654 network=tcp +49:16.737 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23655 network=tcp +49:16.737 [INFO] TestAgent: started state syncer +49:16.737 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23667 network=tcp +49:16.772 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.772 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23666 [Candidate]" term=2 +49:16.773 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.773 [DEBUG] TestAgent.server.raft: vote granted: from=9a3c1daf-b4f4-64e1-cf24-e6171f5deb8d term=2 tally=1 +49:16.773 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.773 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23666 [Leader]" +49:16.773 [INFO] TestAgent.server: cluster leadership acquired +49:16.773 [INFO] TestAgent.server: New leader elected: payload=Node-9a3c1daf-b4f4-64e1-cf24-e6171f5deb8d +49:16.773 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23666 +49:16.774 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.775 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.775 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.775 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.775 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.775 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.775 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9a3c1daf-b4f4-64e1-cf24-e6171f5deb8d +49:16.775 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9a3c1daf-b4f4-64e1-cf24-e6171f5deb8d +49:16.776 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.880 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:16.881 [INFO] TestAgent: Synced node info +49:16.942 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:16.942 [INFO] TestAgent: Requesting shutdown +49:16.942 [INFO] TestAgent.server: shutting down server +49:16.942 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.942 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.942 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.942 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.942 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.942 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.942 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.943 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.943 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.943 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:16.943 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:16.943Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.943Z [INFO] agent.router.manager: shutting down +49:16.943 [INFO] TestAgent: consul server down +49:16.943 [INFO] TestAgent: shutdown complete +49:16.943 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23654 network=tcp +49:16.943 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23654 network=udp +49:16.943 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23655 network=tcp +49:17.443 [INFO] TestAgent: Waiting for endpoints to shut down +49:17.443 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_Explain (2.52s) + --- PASS: TestPreparedQuery_Explain/#00 (0.86s) + --- PASS: TestPreparedQuery_Explain/#01 (0.93s) + --- PASS: TestPreparedQuery_Explain/#02 (0.73s) +=== CONT TestStatusPeers +2022-12-04T05:49:16.674Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.687Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.688 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f6287635-1fac-97b1-19d6-0168e5f2e128 Address:127.0.0.1:23652}]" +49:16.688 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23652 [Follower]" leader= +49:16.689 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f6287635-1fac-97b1-19d6-0168e5f2e128.dc1 127.0.0.1 +49:16.689 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f6287635-1fac-97b1-19d6-0168e5f2e128 127.0.0.1 +2022-12-04T05:49:16.689Z [INFO] agent.router: Initializing LAN area manager +49:16.689 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f6287635-1fac-97b1-19d6-0168e5f2e128.dc1 area=wan +49:16.689 [INFO] TestAgent.server: Adding LAN server: server="Node-f6287635-1fac-97b1-19d6-0168e5f2e128 (Addr: tcp/127.0.0.1:23652) (DC: dc1)" +49:16.689 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23633 network=udp +49:16.690 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23633 network=tcp +49:16.690 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23634 network=tcp +49:16.690 [INFO] TestAgent: started state syncer +49:16.690 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23653 network=tcp +49:16.755 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.755 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23652 [Candidate]" term=2 +49:16.756 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.756 [DEBUG] TestAgent.server.raft: vote granted: from=f6287635-1fac-97b1-19d6-0168e5f2e128 term=2 tally=1 +49:16.756 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.756 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23652 [Leader]" +49:16.756 [INFO] TestAgent.server: cluster leadership acquired +49:16.756 [INFO] TestAgent.server: New leader elected: payload=Node-f6287635-1fac-97b1-19d6-0168e5f2e128 +49:16.756 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23652 +49:16.757 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.758 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.758 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.758 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.758 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.758 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.758 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f6287635-1fac-97b1-19d6-0168e5f2e128 +49:16.758 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f6287635-1fac-97b1-19d6-0168e5f2e128 +49:16.759 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.953 [INFO] TestAgent: Requesting shutdown +49:16.953 [INFO] TestAgent.server: shutting down server +49:16.953 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.953 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.953 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.953 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.953 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.953 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:16.953 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.953 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.953 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.953 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.953 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:16.953 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:16.953Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.953Z [INFO] agent.router.manager: shutting down +49:16.953 [INFO] TestAgent: consul server down +49:16.953 [INFO] TestAgent: shutdown complete +49:16.953 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23633 network=tcp +49:16.953 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23633 network=udp +49:16.953 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23634 network=tcp +49:17.456 [INFO] TestAgent: Waiting for endpoints to shut down +49:17.456 [INFO] TestAgent: Endpoints down +--- PASS: TestTxnEndpoint_Bad_Size_Ops (0.80s) +=== CONT TestStatusLeaderSecondary +2022-12-04T05:49:16.773Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.790Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.798 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:364acb74-474f-937b-303c-b197d7f86455 Address:127.0.0.1:23680}]" +49:16.799 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23680 [Follower]" leader= +49:16.800 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-364acb74-474f-937b-303c-b197d7f86455.dc1 127.0.0.1 +49:16.801 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-364acb74-474f-937b-303c-b197d7f86455 127.0.0.1 +2022-12-04T05:49:16.801Z [INFO] agent.router: Initializing LAN area manager +49:16.801 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23661 network=udp +49:16.801 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-364acb74-474f-937b-303c-b197d7f86455.dc1 area=wan +49:16.801 [INFO] TestAgent.server: Adding LAN server: server="Node-364acb74-474f-937b-303c-b197d7f86455 (Addr: tcp/127.0.0.1:23680) (DC: dc1)" +49:16.801 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23661 network=tcp +49:16.802 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23662 network=tcp +49:16.802 [INFO] TestAgent: started state syncer +49:16.802 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23681 network=tcp +49:16.847 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.847 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23680 [Candidate]" term=2 +49:16.847 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.847 [DEBUG] TestAgent.server.raft: vote granted: from=364acb74-474f-937b-303c-b197d7f86455 term=2 tally=1 +49:16.847 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.847 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23680 [Leader]" +49:16.847 [INFO] TestAgent.server: cluster leadership acquired +49:16.847 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23680 +49:16.847 [INFO] TestAgent.server: New leader elected: payload=Node-364acb74-474f-937b-303c-b197d7f86455 +49:16.848 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.849 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.849 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.849 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.849 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.849 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.849 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-364acb74-474f-937b-303c-b197d7f86455 +49:16.849 [INFO] TestAgent.server: member joined, marking health alive: member=Node-364acb74-474f-937b-303c-b197d7f86455 +49:16.850 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.969 [INFO] TestAgent: Requesting shutdown +49:16.970 [INFO] TestAgent.server: shutting down server +49:16.970 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.970 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.970 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.970 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.970 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.970 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.970 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.970 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:16.970 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:16.970 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.970 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.970 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:16.970Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.970Z [INFO] agent.router.manager: shutting down +49:16.970 [INFO] TestAgent: consul server down +49:16.970 [INFO] TestAgent: shutdown complete +49:16.970 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23661 network=tcp +49:16.970 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23661 network=udp +49:16.970 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23662 network=tcp +49:17.470 [INFO] TestAgent: Waiting for endpoints to shut down +49:17.470 [INFO] TestAgent: Endpoints down +=== RUN TestTxnEndpoint_Bad_Size_Item/exceeds_configured_max_txn_len +[WARN] freeport: 10 out of 66 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:16.810Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.823Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.824 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e9eb8e89-132e-6e7e-aee1-ee160ff5c85d Address:127.0.0.1:23645}]" +49:16.825 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e9eb8e89-132e-6e7e-aee1-ee160ff5c85d.dc1 127.0.0.1 +49:16.826 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e9eb8e89-132e-6e7e-aee1-ee160ff5c85d 127.0.0.1 +2022-12-04T05:49:16.826Z [INFO] agent.router: Initializing LAN area manager +49:16.826 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23682 network=udp +49:16.826 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23645 [Follower]" leader= +49:16.826 [INFO] TestAgent.server: Adding LAN server: server="Node-e9eb8e89-132e-6e7e-aee1-ee160ff5c85d (Addr: tcp/127.0.0.1:23645) (DC: dc1)" +49:16.826 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e9eb8e89-132e-6e7e-aee1-ee160ff5c85d.dc1 area=wan +49:16.826 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23682 network=tcp +49:16.827 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23683 network=tcp +49:16.827 [INFO] TestAgent: started state syncer +49:16.827 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23646 network=tcp +49:16.889 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.889 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23645 [Candidate]" term=2 +49:16.890 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.890 [DEBUG] TestAgent.server.raft: vote granted: from=e9eb8e89-132e-6e7e-aee1-ee160ff5c85d term=2 tally=1 +49:16.890 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.890 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23645 [Leader]" +49:16.890 [INFO] TestAgent.server: cluster leadership acquired +49:16.890 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23645 +49:16.890 [INFO] TestAgent.server: New leader elected: payload=Node-e9eb8e89-132e-6e7e-aee1-ee160ff5c85d +49:16.890 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.892 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.892 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.892 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.892 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.892 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.892 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e9eb8e89-132e-6e7e-aee1-ee160ff5c85d +49:16.892 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e9eb8e89-132e-6e7e-aee1-ee160ff5c85d +49:16.892 [INFO] TestAgent.server: federation state anti-entropy synced +49:16.980 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:16.980 [INFO] TestAgent: Synced node info +49:16.980 [DEBUG] TestAgent: Node info in sync +49:17.058 [INFO] TestAgent: Requesting shutdown +49:17.058 [INFO] TestAgent.server: shutting down server +49:17.058 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:17.058 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:17.058 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:17.058 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:17.058 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.058 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:17.058 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:17.058 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:17.058 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:17.058 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:17.058 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:17.058Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.058Z [INFO] agent.router.manager: shutting down +49:17.058 [INFO] TestAgent: consul server down +49:17.058 [INFO] TestAgent: shutdown complete +49:17.058 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23682 network=tcp +49:17.058 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23682 network=udp +49:17.058 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23683 network=tcp +49:17.558 [INFO] TestAgent: Waiting for endpoints to shut down +49:17.558 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsCreate_good (0.77s) +=== CONT TestIntentionsSpecificGet_invalidId +2022-12-04T05:49:16.930Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.939Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.941 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:11222eb3-8cea-f85b-9101-726940083e87 Address:127.0.0.1:23694}]" +49:16.941 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23694 [Follower]" leader= +49:16.941 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-11222eb3-8cea-f85b-9101-726940083e87.dc1 127.0.0.1 +49:16.942 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-11222eb3-8cea-f85b-9101-726940083e87 127.0.0.1 +2022-12-04T05:49:16.942Z [INFO] agent.router: Initializing LAN area manager +49:16.942 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23647 network=udp +49:16.942 [INFO] TestAgent.server: Adding LAN server: server="Node-11222eb3-8cea-f85b-9101-726940083e87 (Addr: tcp/127.0.0.1:23694) (DC: dc1)" +49:16.942 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-11222eb3-8cea-f85b-9101-726940083e87.dc1 area=wan +49:16.942 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23647 network=tcp +49:16.943 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23648 network=tcp +49:16.943 [INFO] TestAgent: started state syncer +49:16.943 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23695 network=tcp +49:16.983 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.983 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23694 [Candidate]" term=2 +49:16.983 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.983 [DEBUG] TestAgent.server.raft: vote granted: from=11222eb3-8cea-f85b-9101-726940083e87 term=2 tally=1 +49:16.983 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.983 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23694 [Leader]" +49:16.983 [INFO] TestAgent.server: cluster leadership acquired +49:16.983 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23694 +49:16.984 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.985 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.985 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.985 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.985 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.985 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.985 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-11222eb3-8cea-f85b-9101-726940083e87 +49:16.985 [INFO] TestAgent.server: member joined, marking health alive: member=Node-11222eb3-8cea-f85b-9101-726940083e87 +49:16.986 [INFO] TestAgent.server: New leader elected: payload=Node-11222eb3-8cea-f85b-9101-726940083e87 +49:16.986 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.068 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:17.069 [INFO] TestAgent: Synced node info +49:17.069 [DEBUG] TestAgent: Node info in sync +49:17.196 [INFO] TestAgent: Requesting shutdown +49:17.196 [INFO] TestAgent.server: shutting down server +49:17.196 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:17.196 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:17.196 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:17.196 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:17.196 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.197 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:17.197 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:17.197 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:17.197 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:17.197 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:17.197 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:17.197Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.197Z [INFO] agent.router.manager: shutting down +49:17.197 [INFO] TestAgent: consul server down +49:17.197 [INFO] TestAgent: shutdown complete +49:17.197 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23647 network=tcp +49:17.197 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23647 network=udp +49:17.197 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23648 network=tcp +49:17.697 [INFO] TestAgent: Waiting for endpoints to shut down +49:17.697 [INFO] TestAgent: Endpoints down +--- PASS: TestTxnEndpoint_Bad_JSON (0.78s) +=== CONT TestStatusLeader +2022-12-04T05:49:15.138Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.161Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.163 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:057e0c78-34ab-9491-f6bf-d7dbcbaaf544 Address:127.0.0.1:23477}]" +49:15.163 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23477 [Follower]" leader= +49:15.164 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-057e0c78-34ab-9491-f6bf-d7dbcbaaf544.dc1 127.0.0.1 +49:15.164 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-057e0c78-34ab-9491-f6bf-d7dbcbaaf544 127.0.0.1 +2022-12-04T05:49:15.164Z [INFO] agent.router: Initializing LAN area manager +49:15.164 [INFO] TestAgent.server: Adding LAN server: server="Node-057e0c78-34ab-9491-f6bf-d7dbcbaaf544 (Addr: tcp/127.0.0.1:23477) (DC: dc1)" +49:15.164 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-057e0c78-34ab-9491-f6bf-d7dbcbaaf544.dc1 area=wan +49:15.165 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23486 network=udp +49:15.165 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23486 network=tcp +49:15.165 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23487 network=tcp +49:15.165 [INFO] TestAgent: started state syncer +49:15.165 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23478 network=tcp +49:15.217 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.217 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23477 [Candidate]" term=2 +49:15.217 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.217 [DEBUG] TestAgent.server.raft: vote granted: from=057e0c78-34ab-9491-f6bf-d7dbcbaaf544 term=2 tally=1 +49:15.217 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.217 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23477 [Leader]" +49:15.217 [INFO] TestAgent.server: cluster leadership acquired +49:15.218 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23477 +49:15.219 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.219 [INFO] TestAgent.server: New leader elected: payload=Node-057e0c78-34ab-9491-f6bf-d7dbcbaaf544 +49:15.221 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.221 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.221 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.221 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.221 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.221 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-057e0c78-34ab-9491-f6bf-d7dbcbaaf544 +49:15.221 [INFO] TestAgent.server: member joined, marking health alive: member=Node-057e0c78-34ab-9491-f6bf-d7dbcbaaf544 +49:15.222 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.314 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:15.314 [INFO] TestAgent: Synced node info +49:16.776 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:16.776 [DEBUG] TestAgent: Node info in sync +49:16.776 [DEBUG] TestAgent: Node info in sync +49:17.228 [INFO] TestAgent: Requesting shutdown +49:17.228 [INFO] TestAgent.server: shutting down server +49:17.228 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:17.228 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:17.228 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:17.228 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:17.228 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.228 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:17.228 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:17.228 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:17.228 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:17.228 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:17.228 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:17.228Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.228Z [INFO] agent.router.manager: shutting down +49:17.228 [INFO] TestAgent: consul server down +49:17.228 [INFO] TestAgent: shutdown complete +49:17.228 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23486 network=tcp +49:17.228 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23486 network=udp +49:17.228 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23487 network=tcp +49:17.728 [INFO] TestAgent: Waiting for endpoints to shut down +49:17.728 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_ServerHealth (2.63s) +=== CONT TestIntentionsSpecificGet_good +2022-12-04T05:49:15.120Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.151Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.158 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:61a661bd-018d-6ebb-97e4-5615ae7d569d Address:127.0.0.1:23484}]" +49:15.159 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23484 [Follower]" leader= +49:15.159 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-61a661bd-018d-6ebb-97e4-5615ae7d569d.dc1 127.0.0.1 +49:15.160 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-61a661bd-018d-6ebb-97e4-5615ae7d569d 127.0.0.1 +2022-12-04T05:49:15.160Z [INFO] agent.router: Initializing LAN area manager +49:15.160 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-61a661bd-018d-6ebb-97e4-5615ae7d569d.dc1 area=wan +49:15.160 [INFO] TestAgent.server: Adding LAN server: server="Node-61a661bd-018d-6ebb-97e4-5615ae7d569d (Addr: tcp/127.0.0.1:23484) (DC: dc1)" +49:15.160 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23472 network=udp +49:15.160 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23472 network=tcp +49:15.161 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23473 network=tcp +49:15.161 [INFO] TestAgent: started state syncer +49:15.161 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23485 network=tcp +49:15.212 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.216 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23484 [Candidate]" term=2 +49:15.216 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.216 [DEBUG] TestAgent.server.raft: vote granted: from=61a661bd-018d-6ebb-97e4-5615ae7d569d term=2 tally=1 +49:15.216 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.216 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23484 [Leader]" +49:15.218 [INFO] TestAgent.server: cluster leadership acquired +49:15.219 [INFO] TestAgent.server: New leader elected: payload=Node-61a661bd-018d-6ebb-97e4-5615ae7d569d +49:15.221 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23484 +49:15.222 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.224 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.228 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.228 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.228 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.228 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.228 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-61a661bd-018d-6ebb-97e4-5615ae7d569d +49:15.228 [INFO] TestAgent.server: member joined, marking health alive: member=Node-61a661bd-018d-6ebb-97e4-5615ae7d569d +49:15.237 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.584 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:15.585 [INFO] TestAgent: Synced node info +49:17.228 [INFO] TestAgent: Requesting shutdown +49:17.228 [INFO] TestAgent.server: shutting down server +49:17.228 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:17.228 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:17.228 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:17.228 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:17.228 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.228 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:17.228 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:17.228 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:17.228 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:17.228 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:17.228 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:17.228Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.228Z [INFO] agent.router.manager: shutting down +49:17.228 [INFO] TestAgent: consul server down +49:17.228 [INFO] TestAgent: shutdown complete +49:17.228 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23472 network=tcp +49:17.228 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23472 network=udp +49:17.228 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23473 network=tcp +49:17.732 [INFO] TestAgent: Waiting for endpoints to shut down +49:17.732 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_ServerHealth_Unhealthy (2.65s) +=== CONT TestSnapshot_Options +=== RUN TestSnapshot_Options/GET +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:16.737Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.746Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.748 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:910a549e-cbce-086e-736d-c7fb991400ec Address:127.0.0.1:23673}]" +49:16.748 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23673 [Follower]" leader= +49:16.748 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-910a549e-cbce-086e-736d-c7fb991400ec.dc1 127.0.0.1 +49:16.749 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-910a549e-cbce-086e-736d-c7fb991400ec 127.0.0.1 +2022-12-04T05:49:16.749Z [INFO] agent.router: Initializing LAN area manager +49:16.749 [INFO] TestAgent.server: Adding LAN server: server="Node-910a549e-cbce-086e-736d-c7fb991400ec (Addr: tcp/127.0.0.1:23673) (DC: dc1)" +49:16.749 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-910a549e-cbce-086e-736d-c7fb991400ec.dc1 area=wan +49:16.749 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23668 network=udp +49:16.749 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23668 network=tcp +49:16.750 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23669 network=tcp +49:16.750 [INFO] TestAgent: started state syncer +49:16.750 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23674 network=tcp +49:16.788 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:16.788 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23673 [Candidate]" term=2 +49:16.788 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:16.788 [DEBUG] TestAgent.server.raft: vote granted: from=910a549e-cbce-086e-736d-c7fb991400ec term=2 tally=1 +49:16.788 [INFO] TestAgent.server.raft: election won: tally=1 +49:16.788 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23673 [Leader]" +49:16.788 [INFO] TestAgent.server: cluster leadership acquired +49:16.789 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23673 +49:16.790 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:16.792 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:16.792 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:16.792 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:16.792 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:16.792 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:16.792 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-910a549e-cbce-086e-736d-c7fb991400ec +49:16.792 [INFO] TestAgent.server: member joined, marking health alive: member=Node-910a549e-cbce-086e-736d-c7fb991400ec +49:16.792 [INFO] TestAgent.server: New leader elected: payload=Node-910a549e-cbce-086e-736d-c7fb991400ec +49:16.792 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.087 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:17.087 [INFO] TestAgent: Synced node info +49:17.360 [INFO] TestAgent: Requesting shutdown +49:17.360 [INFO] TestAgent.server: shutting down server +49:17.360 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:17.360 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:17.360 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:17.360 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:17.360 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.360 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:17.360 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:17.360 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:17.360 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:17.360 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:17.360 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:17.360Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.360Z [INFO] agent.router.manager: shutting down +49:17.360 [INFO] TestAgent: consul server down +49:17.360 [INFO] TestAgent: shutdown complete +49:17.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23668 network=tcp +49:17.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23668 network=udp +49:17.360 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23669 network=tcp +49:17.860 [INFO] TestAgent: Waiting for endpoints to shut down +49:17.860 [INFO] TestAgent: Endpoints down +=== RUN TestTxnEndpoint_Bad_Size_Net/exceeds_configured_max_txn_len +2022-12-04T05:49:17.362Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.371Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.373 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fa4c52ce-fdc5-2a1d-887e-2e689e1fe708 Address:127.0.0.1:23729}]" +49:17.373 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23729 [Follower]" leader= +49:17.373 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fa4c52ce-fdc5-2a1d-887e-2e689e1fe708.dc1 127.0.0.1 +49:17.373 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fa4c52ce-fdc5-2a1d-887e-2e689e1fe708 127.0.0.1 +2022-12-04T05:49:17.373Z [INFO] agent.router: Initializing LAN area manager +49:17.374 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23724 network=udp +49:17.374 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fa4c52ce-fdc5-2a1d-887e-2e689e1fe708.dc1 area=wan +49:17.374 [INFO] TestAgent.server: Adding LAN server: server="Node-fa4c52ce-fdc5-2a1d-887e-2e689e1fe708 (Addr: tcp/127.0.0.1:23729) (DC: dc1)" +49:17.374 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23724 network=tcp +49:17.374 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23725 network=tcp +49:17.374 [INFO] TestAgent: started state syncer +49:17.374 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23730 network=tcp +49:17.417 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.417 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23729 [Candidate]" term=2 +49:17.418 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.418 [DEBUG] TestAgent.server.raft: vote granted: from=fa4c52ce-fdc5-2a1d-887e-2e689e1fe708 term=2 tally=1 +49:17.418 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.418 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23729 [Leader]" +49:17.418 [INFO] TestAgent.server: cluster leadership acquired +49:17.419 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23729 +49:17.419 [INFO] TestAgent.server: New leader elected: payload=Node-fa4c52ce-fdc5-2a1d-887e-2e689e1fe708 +49:17.421 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.421 [INFO] TestAgent: Requesting shutdown +49:17.421 [INFO] TestAgent.server: shutting down server +49:17.421 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.421 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:17.422 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:17.422Z [INFO] agent.router.manager: shutting down +49:17.422 [INFO] TestAgent: consul server down +49:17.422 [INFO] TestAgent: shutdown complete +49:17.422 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23724 network=tcp +49:17.422 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23724 network=udp +49:17.422 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23725 network=tcp +2022-12-04T05:49:17.422Z [INFO] agent.router.manager: shutting down +49:17.422 [ERROR] TestAgent.server: failed to establish leadership: error="error generating CA root certificate: raft is already shutdown" +49:17.422 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=0 retry_limit=3 error="raft is already shutdown" +49:17.922 [INFO] TestAgent: Waiting for endpoints to shut down +49:17.922 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsSpecificUpdate_good (0.57s) +=== CONT TestSessionDeleteDestroy +2022-12-04T05:49:17.394Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.411Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.414 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:10f75341-6721-9ef8-191e-dc8e26e04371 Address:127.0.0.1:23743}]" +49:17.415 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-10f75341-6721-9ef8-191e-dc8e26e04371.dc1 127.0.0.1 +49:17.416 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-10f75341-6721-9ef8-191e-dc8e26e04371 127.0.0.1 +2022-12-04T05:49:17.416Z [INFO] agent.router: Initializing LAN area manager +49:17.416 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23743 [Follower]" leader= +49:17.417 [INFO] TestAgent.server: Adding LAN server: server="Node-10f75341-6721-9ef8-191e-dc8e26e04371 (Addr: tcp/127.0.0.1:23743) (DC: dc1)" +49:17.417 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-10f75341-6721-9ef8-191e-dc8e26e04371.dc1 area=wan +49:17.418 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23731 network=tcp +49:17.419 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23731 network=udp +49:17.421 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23732 network=tcp +49:17.421 [INFO] TestAgent: started state syncer +49:17.421 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23744 network=tcp +49:17.481 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.481 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23743 [Candidate]" term=2 +49:17.481 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.481 [DEBUG] TestAgent.server.raft: vote granted: from=10f75341-6721-9ef8-191e-dc8e26e04371 term=2 tally=1 +49:17.481 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.481 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23743 [Leader]" +49:17.481 [INFO] TestAgent.server: cluster leadership acquired +49:17.481 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23743 +49:17.482 [INFO] TestAgent.server: New leader elected: payload=Node-10f75341-6721-9ef8-191e-dc8e26e04371 +49:17.482 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.483 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:17.483 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:17.483 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:17.483 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:17.483 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:17.483 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-10f75341-6721-9ef8-191e-dc8e26e04371 +49:17.484 [INFO] TestAgent.server: member joined, marking health alive: member=Node-10f75341-6721-9ef8-191e-dc8e26e04371 +49:17.484 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.515 [INFO] TestAgent: Requesting shutdown +49:17.515 [INFO] TestAgent.server: shutting down server +49:17.515 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:17.515 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:17.515 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:17.515 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:17.515 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.515 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:17.515 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:17.515 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:17.515 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:17.515 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:17.515 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:17.515 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:17.515Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.515Z [INFO] agent.router.manager: shutting down +49:17.515 [INFO] TestAgent: consul server down +49:17.515 [INFO] TestAgent: shutdown complete +49:17.515 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23731 network=tcp +49:17.515 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23731 network=udp +49:17.515 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23732 network=tcp +49:18.015 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.015 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_LoadKeyrings/client_with_keys +2022-12-04T05:49:17.395Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.413Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.418 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0064728f-7a9d-f537-5829-16b18b0d0fcf Address:127.0.0.1:23736}]" +49:17.418 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23736 [Follower]" leader= +49:17.419 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0064728f-7a9d-f537-5829-16b18b0d0fcf.dc1 127.0.0.1 +49:17.421 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0064728f-7a9d-f537-5829-16b18b0d0fcf 127.0.0.1 +2022-12-04T05:49:17.421Z [INFO] agent.router: Initializing LAN area manager +49:17.421 [INFO] TestAgent.server: Adding LAN server: server="Node-0064728f-7a9d-f537-5829-16b18b0d0fcf (Addr: tcp/127.0.0.1:23736) (DC: dc1)" +49:17.421 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0064728f-7a9d-f537-5829-16b18b0d0fcf.dc1 area=wan +49:17.422 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23745 network=udp +49:17.422 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23745 network=tcp +49:17.422 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23746 network=tcp +49:17.422 [INFO] TestAgent: started state syncer +49:17.422 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23737 network=tcp +49:17.464 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.464 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23736 [Candidate]" term=2 +49:17.464 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.464 [DEBUG] TestAgent.server.raft: vote granted: from=0064728f-7a9d-f537-5829-16b18b0d0fcf term=2 tally=1 +49:17.464 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.464 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23736 [Leader]" +49:17.464 [INFO] TestAgent.server: cluster leadership acquired +49:17.464 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23736 +49:17.464 [INFO] TestAgent.server: New leader elected: payload=Node-0064728f-7a9d-f537-5829-16b18b0d0fcf +49:17.465 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.466 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:17.466 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:17.466 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:17.466 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:17.466 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:17.466 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0064728f-7a9d-f537-5829-16b18b0d0fcf +49:17.466 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0064728f-7a9d-f537-5829-16b18b0d0fcf +49:17.467 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.535 [INFO] TestAgent: Requesting shutdown +49:17.535 [INFO] TestAgent.server: shutting down server +49:17.535 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:17.535 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:17.535 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:17.535 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:17.535 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.535 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:17.535 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:17.535 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:17.535 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:17.535 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:17.535 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:17.535 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:17.535Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.535Z [INFO] agent.router.manager: shutting down +49:17.536 [INFO] TestAgent: consul server down +49:17.536 [INFO] TestAgent: shutdown complete +49:17.536 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23745 network=tcp +49:17.536 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23745 network=udp +49:17.536 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23746 network=tcp +49:18.036 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.036 [INFO] TestAgent: Endpoints down +--- PASS: TestTxnEndpoint_KV_Actions (1.54s) + --- PASS: TestTxnEndpoint_KV_Actions/#00 (0.89s) + --- PASS: TestTxnEndpoint_KV_Actions/#01 (0.65s) +=== CONT TestIntentionsCreate_noBody +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:17.119Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.128Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.130 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:60a0a0e7-f212-d96c-5f35-68762b082180 Address:127.0.0.1:23701}]" +49:17.130 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23701 [Follower]" leader= +49:17.130 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-60a0a0e7-f212-d96c-5f35-68762b082180.dc1 127.0.0.1 +49:17.131 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-60a0a0e7-f212-d96c-5f35-68762b082180 127.0.0.1 +2022-12-04T05:49:17.131Z [INFO] agent.router: Initializing LAN area manager +49:17.131 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23689 network=udp +49:17.131 [INFO] TestAgent.server: Adding LAN server: server="Node-60a0a0e7-f212-d96c-5f35-68762b082180 (Addr: tcp/127.0.0.1:23701) (DC: dc1)" +49:17.131 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-60a0a0e7-f212-d96c-5f35-68762b082180.dc1 area=wan +49:17.131 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23689 network=tcp +49:17.132 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23690 network=tcp +49:17.132 [INFO] TestAgent: started state syncer +49:17.132 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23702 network=tcp +49:17.195 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.195 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23701 [Candidate]" term=2 +49:17.195 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.195 [DEBUG] TestAgent.server.raft: vote granted: from=60a0a0e7-f212-d96c-5f35-68762b082180 term=2 tally=1 +49:17.195 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.195 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23701 [Leader]" +49:17.195 [INFO] TestAgent.server: cluster leadership acquired +49:17.195 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23701 +49:17.195 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.196 [INFO] TestAgent.server: New leader elected: payload=Node-60a0a0e7-f212-d96c-5f35-68762b082180 +49:17.197 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:17.197 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:17.197 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:17.197 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:17.197 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:17.197 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-60a0a0e7-f212-d96c-5f35-68762b082180 +49:17.197 [INFO] TestAgent.server: member joined, marking health alive: member=Node-60a0a0e7-f212-d96c-5f35-68762b082180 +49:17.197 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.483 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:17.484 [INFO] TestAgent: Synced node info +49:17.484 [DEBUG] TestAgent: Node info in sync +49:17.551 [INFO] TestAgent: Requesting shutdown +49:17.551 [INFO] TestAgent.server: shutting down server +49:17.551 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:17.551 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:17.551 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:17.551 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:17.551 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.551 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:17.551 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:17.551 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:17.551 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:17.551 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:17.551 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:17.551Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.551Z [INFO] agent.router.manager: shutting down +49:17.551 [INFO] TestAgent: consul server down +49:17.551 [INFO] TestAgent: shutdown complete +49:17.551 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23689 network=tcp +49:17.552 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23689 network=udp +49:17.552 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23690 network=tcp +49:18.052 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.052 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsSpecificDelete_good (0.94s) +=== CONT TestSessionGet +=== RUN TestSessionGet/#00 +2022-12-04T05:49:17.341Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.351Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.353 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:75300537-ed28-2ba5-f221-3e4266f2d786 Address:127.0.0.1:23722}]" +49:17.353 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23722 [Follower]" leader= +49:17.353 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-75300537-ed28-2ba5-f221-3e4266f2d786.dc1 127.0.0.1 +49:17.354 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-75300537-ed28-2ba5-f221-3e4266f2d786 127.0.0.1 +2022-12-04T05:49:17.354Z [INFO] agent.router: Initializing LAN area manager +49:17.354 [INFO] TestAgent.server: Adding LAN server: server="Node-75300537-ed28-2ba5-f221-3e4266f2d786 (Addr: tcp/127.0.0.1:23722) (DC: dc1)" +49:17.354 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-75300537-ed28-2ba5-f221-3e4266f2d786.dc1 area=wan +49:17.354 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23717 network=udp +49:17.354 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23717 network=tcp +49:17.355 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23718 network=tcp +49:17.355 [INFO] TestAgent: started state syncer +49:17.355 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23723 network=tcp +49:17.422 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.422 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23722 [Candidate]" term=2 +49:17.422 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.422 [DEBUG] TestAgent.server.raft: vote granted: from=75300537-ed28-2ba5-f221-3e4266f2d786 term=2 tally=1 +49:17.422 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.422 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23722 [Leader]" +49:17.422 [INFO] TestAgent.server: cluster leadership acquired +49:17.423 [INFO] TestAgent.server: New leader elected: payload=Node-75300537-ed28-2ba5-f221-3e4266f2d786 +49:17.423 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23722 +49:17.423 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.424 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:17.424 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:17.424 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:17.424 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:17.424 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:17.424 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-75300537-ed28-2ba5-f221-3e4266f2d786 +49:17.424 [INFO] TestAgent.server: member joined, marking health alive: member=Node-75300537-ed28-2ba5-f221-3e4266f2d786 +49:17.425 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.592 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:17.592 [INFO] TestAgent: Requesting shutdown +49:17.592 [INFO] TestAgent.server: shutting down server +49:17.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:17.592 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:17.592 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:17.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:17.592 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.592 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:17.592 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:17.592 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:17.592 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:17.592 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:17.592 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:17.592 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:17.592Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.592Z [INFO] agent.router.manager: shutting down +49:17.592 [INFO] TestAgent: consul server down +49:17.592 [INFO] TestAgent: shutdown complete +49:17.592 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23717 network=tcp +49:17.592 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23717 network=udp +49:17.592 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23718 network=tcp +49:18.093 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.093 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Execute/#04 +2022-12-04T05:49:17.325Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.336Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.338 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4f1a8aef-647a-a059-10dd-566b47ad8c95 Address:127.0.0.1:23715}]" +49:17.338 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23715 [Follower]" leader= +49:17.338 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4f1a8aef-647a-a059-10dd-566b47ad8c95.dc1 127.0.0.1 +49:17.339 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4f1a8aef-647a-a059-10dd-566b47ad8c95 127.0.0.1 +2022-12-04T05:49:17.339Z [INFO] agent.router: Initializing LAN area manager +49:17.339 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23710 network=udp +49:17.339 [INFO] TestAgent.server: Adding LAN server: server="Node-4f1a8aef-647a-a059-10dd-566b47ad8c95 (Addr: tcp/127.0.0.1:23715) (DC: dc1)" +49:17.339 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4f1a8aef-647a-a059-10dd-566b47ad8c95.dc1 area=wan +49:17.339 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23710 network=tcp +49:17.340 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23711 network=tcp +49:17.340 [INFO] TestAgent: started state syncer +49:17.340 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23716 network=tcp +49:17.401 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.401 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23715 [Candidate]" term=2 +49:17.401 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.401 [DEBUG] TestAgent.server.raft: vote granted: from=4f1a8aef-647a-a059-10dd-566b47ad8c95 term=2 tally=1 +49:17.401 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.401 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23715 [Leader]" +49:17.401 [INFO] TestAgent.server: cluster leadership acquired +49:17.401 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23715 +49:17.402 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.403 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:17.403 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:17.403 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:17.403 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:17.403 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:17.403 [INFO] TestAgent.server: New leader elected: payload=Node-4f1a8aef-647a-a059-10dd-566b47ad8c95 +49:17.404 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4f1a8aef-647a-a059-10dd-566b47ad8c95 +49:17.404 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4f1a8aef-647a-a059-10dd-566b47ad8c95 +49:17.408 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.652 [INFO] TestAgent: Requesting shutdown +49:17.652 [INFO] TestAgent.server: shutting down server +49:17.652 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:17.652 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:17.652 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:17.652 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:17.652 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.652 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:17.652 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:17.652 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:17.652 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:17.652 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:17.652 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:17.652 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:17.652Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.652Z [INFO] agent.router.manager: shutting down +49:17.652 [INFO] TestAgent: consul server down +49:17.652 [INFO] TestAgent: shutdown complete +49:17.652 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23710 network=tcp +49:17.652 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23710 network=udp +49:17.652 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23711 network=tcp +49:18.153 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.153 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_InmemKeyrings/client_with_keys +2022-12-04T05:49:17.483Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.493Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.494 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:79482683-fc42-01c2-6b09-d63b0ec77517 Address:127.0.0.1:23766}]" +49:17.494 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23766 [Follower]" leader= +49:17.494 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-79482683-fc42-01c2-6b09-d63b0ec77517.dc1 127.0.0.1 +49:17.495 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-79482683-fc42-01c2-6b09-d63b0ec77517 127.0.0.1 +2022-12-04T05:49:17.495Z [INFO] agent.router: Initializing LAN area manager +49:17.495 [INFO] TestAgent.server: Adding LAN server: server="Node-79482683-fc42-01c2-6b09-d63b0ec77517 (Addr: tcp/127.0.0.1:23766) (DC: dc1)" +49:17.495 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-79482683-fc42-01c2-6b09-d63b0ec77517.dc1 area=wan +49:17.495 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23759 network=udp +49:17.495 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23759 network=tcp +49:17.496 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23760 network=tcp +49:17.496 [INFO] TestAgent: started state syncer +49:17.496 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23778 network=tcp +49:17.551 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.551 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23766 [Candidate]" term=2 +49:17.551 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.551 [DEBUG] TestAgent.server.raft: vote granted: from=79482683-fc42-01c2-6b09-d63b0ec77517 term=2 tally=1 +49:17.551 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.551 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23766 [Leader]" +49:17.551 [INFO] TestAgent.server: cluster leadership acquired +49:17.551 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23766 +49:17.551 [INFO] TestAgent.server: New leader elected: payload=Node-79482683-fc42-01c2-6b09-d63b0ec77517 +49:17.552 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.553 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:17.553 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:17.553 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:17.553 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:17.553 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:17.553 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-79482683-fc42-01c2-6b09-d63b0ec77517 +49:17.553 [INFO] TestAgent.server: member joined, marking health alive: member=Node-79482683-fc42-01c2-6b09-d63b0ec77517 +49:17.554 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.693 [INFO] TestAgent: Requesting shutdown +49:17.693 [INFO] TestAgent.server: shutting down server +49:17.693 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:17.693 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:17.693 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:17.693 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:17.693 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.694 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:17.694 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:17.694 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:17.694 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:17.694 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:17.694 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:17.694 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:17.694Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.694Z [INFO] agent.router.manager: shutting down +49:17.694 [INFO] TestAgent: consul server down +49:17.694 [INFO] TestAgent: shutdown complete +49:17.694 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23759 network=tcp +49:17.694 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23759 network=udp +49:17.694 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23760 network=tcp +49:18.194 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.194 [INFO] TestAgent: Endpoints down +=== RUN TestTxnEndpoint_Bad_Size_Item/exceeds_default_max_kv_value_size +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:17.568Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.577Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.578 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fb5a6d28-d668-9b74-04fc-faeb8a63e3b7 Address:127.0.0.1:23784}]" +49:17.579 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23784 [Follower]" leader= +49:17.579 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fb5a6d28-d668-9b74-04fc-faeb8a63e3b7.dc1 127.0.0.1 +49:17.579 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fb5a6d28-d668-9b74-04fc-faeb8a63e3b7 127.0.0.1 +2022-12-04T05:49:17.579Z [INFO] agent.router: Initializing LAN area manager +49:17.579 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23779 network=udp +49:17.580 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fb5a6d28-d668-9b74-04fc-faeb8a63e3b7.dc1 area=wan +49:17.580 [INFO] TestAgent.server: Adding LAN server: server="Node-fb5a6d28-d668-9b74-04fc-faeb8a63e3b7 (Addr: tcp/127.0.0.1:23784) (DC: dc1)" +49:17.580 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23779 network=tcp +49:17.580 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23780 network=tcp +49:17.580 [INFO] TestAgent: started state syncer +49:17.580 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23771 network=tcp +49:17.644 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.644 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23784 [Candidate]" term=2 +49:17.646 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.646 [DEBUG] TestAgent.server.raft: vote granted: from=fb5a6d28-d668-9b74-04fc-faeb8a63e3b7 term=2 tally=1 +49:17.646 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.646 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23784 [Leader]" +49:17.646 [INFO] TestAgent.server: cluster leadership acquired +49:17.646 [INFO] TestAgent.server: New leader elected: payload=Node-fb5a6d28-d668-9b74-04fc-faeb8a63e3b7 +49:17.646 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23784 +49:17.647 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.648 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:17.648 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:17.648 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:17.648 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:17.648 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:17.648 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fb5a6d28-d668-9b74-04fc-faeb8a63e3b7 +49:17.648 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fb5a6d28-d668-9b74-04fc-faeb8a63e3b7 +49:17.649 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.841 [INFO] TestAgent: Requesting shutdown +49:17.841 [INFO] TestAgent.server: shutting down server +49:17.841 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:17.841 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:17.841 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:17.841 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:17.841 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.841 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:17.841 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:17.841 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:17.841 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:17.841 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:17.841 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:17.841 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:17.841Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.841Z [INFO] agent.router.manager: shutting down +49:17.841 [INFO] TestAgent: consul server down +49:17.841 [INFO] TestAgent: shutdown complete +49:17.841 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23779 network=tcp +49:17.841 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23779 network=udp +49:17.841 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23780 network=tcp +49:18.342 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.342 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsSpecificGet_invalidId (0.78s) +=== CONT TestSessionCustomTTL +2022-12-04T05:49:17.453Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.469Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.471 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:04008698-a4b4-2d0c-ce71-94c8ec35b0b3 Address:127.0.0.1:23750}]" +49:17.471 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23750 [Follower]" leader= +49:17.471 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-04008698-a4b4-2d0c-ce71-94c8ec35b0b3.dc1 127.0.0.1 +49:17.472 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-04008698-a4b4-2d0c-ce71-94c8ec35b0b3 127.0.0.1 +2022-12-04T05:49:17.472Z [INFO] agent.router: Initializing LAN area manager +49:17.472 [INFO] TestAgent.server: Adding LAN server: server="Node-04008698-a4b4-2d0c-ce71-94c8ec35b0b3 (Addr: tcp/127.0.0.1:23750) (DC: dc1)" +49:17.472 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-04008698-a4b4-2d0c-ce71-94c8ec35b0b3.dc1 area=wan +49:17.472 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23738 network=udp +49:17.472 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23738 network=tcp +49:17.473 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23739 network=tcp +49:17.473 [INFO] TestAgent: started state syncer +49:17.473 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23751 network=tcp +49:17.519 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.519 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23750 [Candidate]" term=2 +49:17.519 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.519 [DEBUG] TestAgent.server.raft: vote granted: from=04008698-a4b4-2d0c-ce71-94c8ec35b0b3 term=2 tally=1 +49:17.519 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.519 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23750 [Leader]" +49:17.519 [INFO] TestAgent.server: cluster leadership acquired +49:17.519 [INFO] TestAgent.server: New leader elected: payload=Node-04008698-a4b4-2d0c-ce71-94c8ec35b0b3 +49:17.519 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23750 +49:17.520 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.521 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:17.521 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:17.521 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:17.521 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:17.521 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:17.521 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-04008698-a4b4-2d0c-ce71-94c8ec35b0b3 +49:17.521 [INFO] TestAgent.server: member joined, marking health alive: member=Node-04008698-a4b4-2d0c-ce71-94c8ec35b0b3 +49:17.522 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.651 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:17.651 [INFO] TestAgent: Synced node info +49:17.873 [INFO] TestAgent: Requesting shutdown +49:17.874 [INFO] TestAgent.server: shutting down server +49:17.874 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:17.874 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:17.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:17.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:17.874 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.874 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:17.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:17.874 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:17.874 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:17.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:17.874 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:17.874Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.874Z [INFO] agent.router.manager: shutting down +49:17.874 [INFO] TestAgent: consul server down +49:17.874 [INFO] TestAgent: shutdown complete +49:17.874 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23738 network=tcp +49:17.874 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23738 network=udp +49:17.874 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23739 network=tcp +49:18.375 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.375 [INFO] TestAgent: Endpoints down +--- PASS: TestStatusPeers (0.93s) +=== CONT TestSessionDestroy +2022-12-04T05:49:17.709Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.718Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.720 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:118d30dc-cb9c-5c8a-0596-017a6386fb0b Address:127.0.0.1:23777}]" +49:17.720 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23777 [Follower]" leader= +49:17.720 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-118d30dc-cb9c-5c8a-0596-017a6386fb0b.dc1 127.0.0.1 +49:17.721 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-118d30dc-cb9c-5c8a-0596-017a6386fb0b 127.0.0.1 +2022-12-04T05:49:17.721Z [INFO] agent.router: Initializing LAN area manager +49:17.721 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-118d30dc-cb9c-5c8a-0596-017a6386fb0b.dc1 area=wan +49:17.721 [INFO] TestAgent.server: Adding LAN server: server="Node-118d30dc-cb9c-5c8a-0596-017a6386fb0b (Addr: tcp/127.0.0.1:23777) (DC: dc1)" +49:17.721 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23772 network=udp +49:17.721 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23772 network=tcp +49:17.721 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23773 network=tcp +49:17.721 [INFO] TestAgent: started state syncer +49:17.722 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23785 network=tcp +49:17.756 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.757 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23777 [Candidate]" term=2 +49:17.757 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.757 [DEBUG] TestAgent.server.raft: vote granted: from=118d30dc-cb9c-5c8a-0596-017a6386fb0b term=2 tally=1 +49:17.757 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.757 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23777 [Leader]" +49:17.757 [INFO] TestAgent.server: cluster leadership acquired +49:17.757 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23777 +49:17.757 [INFO] TestAgent.server: New leader elected: payload=Node-118d30dc-cb9c-5c8a-0596-017a6386fb0b +49:17.758 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.759 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:17.759 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:17.759 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:17.759 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:17.759 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:17.759 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-118d30dc-cb9c-5c8a-0596-017a6386fb0b +49:17.759 [INFO] TestAgent.server: member joined, marking health alive: member=Node-118d30dc-cb9c-5c8a-0596-017a6386fb0b +49:17.760 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.925 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:17.925 [INFO] TestAgent: Synced node info +49:17.927 [INFO] TestAgent: Requesting shutdown +49:17.928 [INFO] TestAgent.server: shutting down server +49:17.928 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:17.928 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:17.928 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:17.928 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:17.928 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.928 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:17.928 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:17.928 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:17.928 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:17.928 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:17.928 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:17.928Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.928Z [INFO] agent.router.manager: shutting down +49:17.928 [INFO] TestAgent: consul server down +49:17.928 [INFO] TestAgent: shutdown complete +49:17.928 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23772 network=tcp +49:17.928 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23772 network=udp +49:17.928 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23773 network=tcp +49:18.429 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.429 [INFO] TestAgent: Endpoints down +--- PASS: TestStatusLeader (0.73s) +=== CONT TestIntentionsCheck_noDestination +2022-12-04T05:49:17.743Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.752Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.753 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:df9ef3fa-8383-5492-fb2a-dd61bef014ad Address:127.0.0.1:23791}]" +49:17.754 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23791 [Follower]" leader= +49:17.754 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-df9ef3fa-8383-5492-fb2a-dd61bef014ad.dc1 127.0.0.1 +49:17.754 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-df9ef3fa-8383-5492-fb2a-dd61bef014ad 127.0.0.1 +2022-12-04T05:49:17.754Z [INFO] agent.router: Initializing LAN area manager +49:17.754 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-df9ef3fa-8383-5492-fb2a-dd61bef014ad.dc1 area=wan +49:17.755 [INFO] TestAgent.server: Adding LAN server: server="Node-df9ef3fa-8383-5492-fb2a-dd61bef014ad (Addr: tcp/127.0.0.1:23791) (DC: dc1)" +49:17.755 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23786 network=udp +49:17.755 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23786 network=tcp +49:17.755 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23787 network=tcp +49:17.755 [INFO] TestAgent: started state syncer +49:17.755 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23792 network=tcp +49:17.813 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.813 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23791 [Candidate]" term=2 +49:17.814 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.814 [DEBUG] TestAgent.server.raft: vote granted: from=df9ef3fa-8383-5492-fb2a-dd61bef014ad term=2 tally=1 +49:17.814 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.814 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23791 [Leader]" +49:17.814 [INFO] TestAgent.server: cluster leadership acquired +49:17.814 [INFO] TestAgent.server: New leader elected: payload=Node-df9ef3fa-8383-5492-fb2a-dd61bef014ad +49:17.814 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23791 +49:17.815 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.816 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:17.816 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:17.816 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:17.816 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:17.816 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:17.816 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-df9ef3fa-8383-5492-fb2a-dd61bef014ad +49:17.816 [INFO] TestAgent.server: member joined, marking health alive: member=Node-df9ef3fa-8383-5492-fb2a-dd61bef014ad +49:17.817 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.977 [INFO] TestAgent: Requesting shutdown +49:17.977 [INFO] TestAgent.server: shutting down server +49:17.977 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:17.977 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:17.977 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:17.977 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:17.977 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.977 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:17.977 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:17.977 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:17.977 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:17.977 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:17.977 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:17.977 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:17.977Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.977Z [INFO] agent.router.manager: shutting down +49:17.977 [INFO] TestAgent: consul server down +49:17.977 [INFO] TestAgent: shutdown complete +49:17.977 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23786 network=tcp +49:17.977 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23786 network=udp +49:17.977 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23787 network=tcp +49:18.478 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.478 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsSpecificGet_good (0.75s) +=== CONT TestIntentionsCheck_noSource +49:18.036 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-95606ce2-8e8c-40eb-e09d-2873ed43ef06 127.0.0.1 +2022-12-04T05:49:18.036Z [INFO] agent.router: Initializing LAN area manager +49:18.037 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23821 network=udp +49:18.037 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23821 network=tcp +49:18.037 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23822 network=tcp +49:18.037 [INFO] TestAgent: started state syncer +49:18.038 [INFO] TestAgent: Requesting shutdown +49:18.038 [INFO] TestAgent.client: shutting down client +49:18.038 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +49:18.038 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23827 network=tcp +49:18.038 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" +49:18.038 [INFO] TestAgent: consul client down +49:18.038 [INFO] TestAgent: shutdown complete +49:18.038 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23821 network=tcp +2022-12-04T05:49:18.038Z [WARN] agent.router.manager: No servers available +49:18.038 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23821 network=udp +49:18.038 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +49:18.038 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23822 network=tcp +49:18.538 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.538 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_LoadKeyrings (1.80s) + --- PASS: TestAgent_LoadKeyrings/no_keys (0.64s) + --- PASS: TestAgent_LoadKeyrings/server_with_keys (0.63s) + --- PASS: TestAgent_LoadKeyrings/client_with_keys (0.52s) +=== CONT TestSessionCreate_NoCheck +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:17.136Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.149Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.150 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d48df046-88b3-7f96-4c4c-b9557dc047b0 Address:127.0.0.1:23708}]" +49:17.151 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23708 [Follower]" leader= +49:17.151 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d48df046-88b3-7f96-4c4c-b9557dc047b0.secondary 127.0.0.1 +49:17.151 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d48df046-88b3-7f96-4c4c-b9557dc047b0 127.0.0.1 +2022-12-04T05:49:17.151Z [INFO] agent.router: Initializing LAN area manager +49:17.152 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23703 network=udp +49:17.152 [INFO] TestAgent.server: Adding LAN server: server="Node-d48df046-88b3-7f96-4c4c-b9557dc047b0 (Addr: tcp/127.0.0.1:23708) (DC: secondary)" +49:17.152 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d48df046-88b3-7f96-4c4c-b9557dc047b0.secondary area=wan +49:17.152 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23703 network=tcp +49:17.152 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23704 network=tcp +49:17.152 [INFO] TestAgent: started state syncer +49:17.152 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23709 network=tcp +49:17.208 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.208 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23708 [Candidate]" term=2 +49:17.208 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.208 [DEBUG] TestAgent.server.raft: vote granted: from=d48df046-88b3-7f96-4c4c-b9557dc047b0 term=2 tally=1 +49:17.208 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.208 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23708 [Leader]" +49:17.208 [INFO] TestAgent.server: cluster leadership acquired +49:17.208 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23708 +49:17.208 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.210 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:17.210 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:17.210 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:17.210 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:17.210 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:17.210 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d48df046-88b3-7f96-4c4c-b9557dc047b0 +49:17.210 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d48df046-88b3-7f96-4c4c-b9557dc047b0 +49:17.210 [INFO] TestAgent.server: New leader elected: payload=Node-d48df046-88b3-7f96-4c4c-b9557dc047b0 +49:17.210 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.315 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:17.315 [INFO] TestAgent: Synced node info +49:17.315 [DEBUG] TestAgent: Node info in sync +49:17.573 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:23700] +49:17.574 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:23700 +49:17.574 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-caee452f-f04e-57b4-63fa-3614a9334176.primary 127.0.0.1 +49:17.574 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +49:17.574 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-caee452f-f04e-57b4-63fa-3614a9334176.primary area=wan +49:17.576 [INFO] TestAgent: Requesting shutdown +49:17.576 [INFO] TestAgent.server: shutting down server +49:17.576 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:17.576 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:17.576 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:17.576 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:17.576 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:17.576 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:17.576 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:17.576 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:17.576 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:17.576 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:17.576 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:17.576Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.577Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:17.577Z [INFO] agent.router.manager: shutting down +49:17.577 [INFO] TestAgent: consul server down +49:17.577 [INFO] TestAgent: shutdown complete +49:17.577 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23703 network=tcp +49:17.577 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23703 network=udp +49:17.577 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23704 network=tcp +49:18.077 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.077 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:16.940Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:16.955Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:16.956 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:caee452f-f04e-57b4-63fa-3614a9334176 Address:127.0.0.1:23687}]" +49:16.956 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23687 [Follower]" leader= +49:16.957 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-caee452f-f04e-57b4-63fa-3614a9334176.primary 127.0.0.1 +49:16.957 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-caee452f-f04e-57b4-63fa-3614a9334176 127.0.0.1 +2022-12-04T05:49:16.957Z [INFO] agent.router: Initializing LAN area manager +49:16.958 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23696 network=udp +49:16.958 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-caee452f-f04e-57b4-63fa-3614a9334176.primary area=wan +49:16.958 [INFO] TestAgent.server: Adding LAN server: server="Node-caee452f-f04e-57b4-63fa-3614a9334176 (Addr: tcp/127.0.0.1:23687) (DC: primary)" +49:16.958 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23696 network=tcp +49:16.958 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23697 network=tcp +49:16.958 [INFO] TestAgent: started state syncer +49:16.958 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23688 network=tcp +49:17.002 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.002 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23687 [Candidate]" term=2 +49:17.002 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.002 [DEBUG] TestAgent.server.raft: vote granted: from=caee452f-f04e-57b4-63fa-3614a9334176 term=2 tally=1 +49:17.002 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.002 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23687 [Leader]" +49:17.002 [INFO] TestAgent.server: cluster leadership acquired +49:17.002 [INFO] TestAgent.server: New leader elected: payload=Node-caee452f-f04e-57b4-63fa-3614a9334176 +49:17.002 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23687 +49:17.003 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.005 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:17.005 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:17.005 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:17.005 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:17.005 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:17.005 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-caee452f-f04e-57b4-63fa-3614a9334176 +49:17.005 [INFO] TestAgent.server: member joined, marking health alive: member=Node-caee452f-f04e-57b4-63fa-3614a9334176 +49:17.005 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.345 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:17.345 [INFO] TestAgent: Synced node info +49:17.574 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:59804 +49:17.574 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d48df046-88b3-7f96-4c4c-b9557dc047b0.secondary 127.0.0.1 +49:17.574 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d48df046-88b3-7f96-4c4c-b9557dc047b0.secondary area=wan +49:18.077 [INFO] TestAgent: Requesting shutdown +49:18.077 [INFO] TestAgent.server: shutting down server +49:18.077 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.077 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.077 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.077 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.077 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.077 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.077 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.077 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.077 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:18.077 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.077 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:49:18.077Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.077Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.077Z [INFO] agent.router.manager: shutting down +49:18.077 [INFO] TestAgent: consul server down +49:18.077 [INFO] TestAgent: shutdown complete +49:18.077 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23696 network=tcp +49:18.078 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23696 network=udp +49:18.078 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23697 network=tcp +49:18.580 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.580 [INFO] TestAgent: Endpoints down +--- PASS: TestStatusPeersSecondary (1.65s) +=== CONT TestSessionCreate_DefaultCheck +2022-12-04T05:49:17.742Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:17.742Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.752Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:17.752Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.753 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aedccd3c-8206-5adb-30cb-dbf6daebc94d Address:127.0.0.1:23798}]" +49:17.753 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23798 [Follower]" leader= +49:17.754 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aedccd3c-8206-5adb-30cb-dbf6daebc94d.dc1 127.0.0.1 +49:17.754 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aedccd3c-8206-5adb-30cb-dbf6daebc94d 127.0.0.1 +2022-12-04T05:49:17.754Z [INFO] agent.router: Initializing LAN area manager +49:17.754 [INFO] TestAgent.server: Adding LAN server: server="Node-aedccd3c-8206-5adb-30cb-dbf6daebc94d (Addr: tcp/127.0.0.1:23798) (DC: dc1)" +49:17.754 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aedccd3c-8206-5adb-30cb-dbf6daebc94d.dc1 area=wan +49:17.754 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23793 network=udp +49:17.754 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23793 network=tcp +49:17.755 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23794 network=tcp +49:17.755 [INFO] TestAgent: started state syncer +49:17.755 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23799 network=tcp +49:17.805 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:17.806 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.806 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23798 [Candidate]" term=2 +49:17.806 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.806 [DEBUG] TestAgent.server.raft: vote granted: from=aedccd3c-8206-5adb-30cb-dbf6daebc94d term=2 tally=1 +49:17.806 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.806 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23798 [Leader]" +49:17.806 [INFO] TestAgent.server: cluster leadership acquired +49:17.806 [INFO] TestAgent.server: New leader elected: payload=Node-aedccd3c-8206-5adb-30cb-dbf6daebc94d +49:17.806 [INFO] TestAgent.server: initializing acls +49:17.807 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:17.807 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:17.807 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:17.807 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:17.807 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:17.807 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:17.807 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-aedccd3c-8206-5adb-30cb-dbf6daebc94d +49:17.807 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-aedccd3c-8206-5adb-30cb-dbf6daebc94d.dc1 +49:17.807 [INFO] TestAgent.server: Updating LAN server: server="Node-aedccd3c-8206-5adb-30cb-dbf6daebc94d (Addr: tcp/127.0.0.1:23798) (DC: dc1)" +49:17.807 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-aedccd3c-8206-5adb-30cb-dbf6daebc94d.dc1 area=wan +49:17.808 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.810 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:17.810 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:17.810 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:17.810 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:17.810 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:17.810 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aedccd3c-8206-5adb-30cb-dbf6daebc94d +49:17.810 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aedccd3c-8206-5adb-30cb-dbf6daebc94d +49:17.811 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.811 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aedccd3c-8206-5adb-30cb-dbf6daebc94d +49:18.001 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:18.001 [INFO] TestAgent: Synced node info +49:18.001 [DEBUG] TestAgent: Node info in sync +49:18.143 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-aedccd3c-8206-5adb-30cb-dbf6daebc94d +49:18.143 [INFO] TestAgent: Requesting shutdown +49:18.143 [INFO] TestAgent.server: shutting down server +49:18.143 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:18.143 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:18.143 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.143 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.143 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.143 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.143 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.143 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.143 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.143 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:18.143 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:18.143 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.143 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.144 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.144 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.144Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.144Z [INFO] agent.router.manager: shutting down +49:18.144 [INFO] TestAgent: consul server down +49:18.144 [INFO] TestAgent: shutdown complete +49:18.144 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23793 network=tcp +49:18.144 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23793 network=udp +49:18.144 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23794 network=tcp +49:18.644 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.644 [INFO] TestAgent: Endpoints down +=== RUN TestSnapshot_Options/GET#01 +49:18.176 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-33646e96-838f-11c3-b5d5-23c81b7545d2 127.0.0.1 +2022-12-04T05:49:18.176Z [INFO] agent.router: Initializing LAN area manager +49:18.176 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23849 network=udp +49:18.176 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23849 network=tcp +49:18.177 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23850 network=tcp +49:18.177 [INFO] TestAgent: started state syncer +49:18.177 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23869 network=tcp +2022-12-04T05:49:18.177Z [WARN] agent.router.manager: No servers available +49:18.177 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +49:18.177 [INFO] TestAgent: Requesting shutdown +49:18.177 [INFO] TestAgent.client: shutting down client +49:18.177 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +49:18.177 [INFO] TestAgent: consul client down +49:18.177 [INFO] TestAgent: shutdown complete +49:18.177 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23849 network=tcp +49:18.177 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23849 network=udp +49:18.177 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23850 network=tcp +49:18.678 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.678 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_InmemKeyrings/ignore_files +2022-12-04T05:49:18.046Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:18.055Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:18.057 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e0042682-758c-43b6-46bf-021193acd27f Address:127.0.0.1:23833}]" +49:18.057 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23833 [Follower]" leader= +49:18.057 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e0042682-758c-43b6-46bf-021193acd27f.dc1 127.0.0.1 +49:18.058 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e0042682-758c-43b6-46bf-021193acd27f 127.0.0.1 +2022-12-04T05:49:18.058Z [INFO] agent.router: Initializing LAN area manager +49:18.058 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23828 network=udp +49:18.058 [INFO] TestAgent.server: Adding LAN server: server="Node-e0042682-758c-43b6-46bf-021193acd27f (Addr: tcp/127.0.0.1:23833) (DC: dc1)" +49:18.058 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e0042682-758c-43b6-46bf-021193acd27f.dc1 area=wan +49:18.058 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23828 network=tcp +49:18.058 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23829 network=tcp +49:18.059 [INFO] TestAgent: started state syncer +49:18.059 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23834 network=tcp +49:18.126 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.126 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23833 [Candidate]" term=2 +49:18.126 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.126 [DEBUG] TestAgent.server.raft: vote granted: from=e0042682-758c-43b6-46bf-021193acd27f term=2 tally=1 +49:18.126 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.126 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23833 [Leader]" +49:18.126 [INFO] TestAgent.server: cluster leadership acquired +49:18.127 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23833 +49:18.127 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.128 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.128 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.128 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.128 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.128 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.128 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e0042682-758c-43b6-46bf-021193acd27f +49:18.128 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e0042682-758c-43b6-46bf-021193acd27f +49:18.129 [INFO] TestAgent.server: New leader elected: payload=Node-e0042682-758c-43b6-46bf-021193acd27f +49:18.129 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.204 [INFO] TestAgent: Requesting shutdown +49:18.204 [INFO] TestAgent.server: shutting down server +49:18.204 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.204 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.204 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.204 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.204 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.204 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.204 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.204 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.204 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.204 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.204 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:18.204 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.204Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.204Z [INFO] agent.router.manager: shutting down +49:18.204 [INFO] TestAgent: consul server down +49:18.205 [INFO] TestAgent: shutdown complete +49:18.205 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23828 network=tcp +49:18.205 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23828 network=udp +49:18.205 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23829 network=tcp +49:18.705 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.705 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsCreate_noBody (0.67s) +=== CONT TestIntentionsCheck_basic +2022-12-04T05:49:18.062Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:18.071Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:18.073 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0bc275e7-545b-e442-10e4-cea658e36784 Address:127.0.0.1:23840}]" +49:18.073 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23840 [Follower]" leader= +49:18.073 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0bc275e7-545b-e442-10e4-cea658e36784.dc1 127.0.0.1 +49:18.073 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0bc275e7-545b-e442-10e4-cea658e36784 127.0.0.1 +2022-12-04T05:49:18.074Z [INFO] agent.router: Initializing LAN area manager +49:18.074 [INFO] TestAgent.server: Adding LAN server: server="Node-0bc275e7-545b-e442-10e4-cea658e36784 (Addr: tcp/127.0.0.1:23840) (DC: dc1)" +49:18.074 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0bc275e7-545b-e442-10e4-cea658e36784.dc1 area=wan +49:18.074 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23835 network=udp +49:18.074 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23835 network=tcp +49:18.074 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23836 network=tcp +49:18.074 [INFO] TestAgent: started state syncer +49:18.074 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23841 network=tcp +49:18.118 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.118 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23840 [Candidate]" term=2 +49:18.119 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.119 [DEBUG] TestAgent.server.raft: vote granted: from=0bc275e7-545b-e442-10e4-cea658e36784 term=2 tally=1 +49:18.119 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.119 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23840 [Leader]" +49:18.119 [INFO] TestAgent.server: cluster leadership acquired +49:18.119 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23840 +49:18.119 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.120 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.120 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.120 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.120 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.120 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.120 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0bc275e7-545b-e442-10e4-cea658e36784 +49:18.120 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0bc275e7-545b-e442-10e4-cea658e36784 +49:18.121 [INFO] TestAgent.server: New leader elected: payload=Node-0bc275e7-545b-e442-10e4-cea658e36784 +49:18.121 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.269 [INFO] TestAgent: Requesting shutdown +49:18.270 [INFO] TestAgent.server: shutting down server +49:18.270 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.270 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.270 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.270 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.270 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.270 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.270 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.270 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:18.270 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.270 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.270 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.270 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.270Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.270Z [INFO] agent.router.manager: shutting down +49:18.270 [INFO] TestAgent: consul server down +49:18.270 [INFO] TestAgent: shutdown complete +49:18.270 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23835 network=tcp +49:18.270 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23835 network=udp +49:18.270 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23836 network=tcp +49:18.770 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.770 [INFO] TestAgent: Endpoints down +=== RUN TestSessionGet/#01 +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:18.103Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:18.112Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:18.113 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:47c3a6d1-91fb-4fa2-d351-858ddb731209 Address:127.0.0.1:23847}]" +49:18.113 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23847 [Follower]" leader= +49:18.114 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-47c3a6d1-91fb-4fa2-d351-858ddb731209.dc1 127.0.0.1 +49:18.114 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-47c3a6d1-91fb-4fa2-d351-858ddb731209 127.0.0.1 +2022-12-04T05:49:18.114Z [INFO] agent.router: Initializing LAN area manager +49:18.114 [INFO] TestAgent.server: Adding LAN server: server="Node-47c3a6d1-91fb-4fa2-d351-858ddb731209 (Addr: tcp/127.0.0.1:23847) (DC: dc1)" +49:18.114 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-47c3a6d1-91fb-4fa2-d351-858ddb731209.dc1 area=wan +49:18.114 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23842 network=udp +49:18.114 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23842 network=tcp +49:18.115 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23843 network=tcp +49:18.115 [INFO] TestAgent: started state syncer +49:18.115 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23848 network=tcp +49:18.172 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.172 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23847 [Candidate]" term=2 +49:18.172 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.172 [DEBUG] TestAgent.server.raft: vote granted: from=47c3a6d1-91fb-4fa2-d351-858ddb731209 term=2 tally=1 +49:18.172 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.172 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23847 [Leader]" +49:18.172 [INFO] TestAgent.server: cluster leadership acquired +49:18.172 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23847 +49:18.173 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.174 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.174 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.174 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.174 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.174 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.174 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-47c3a6d1-91fb-4fa2-d351-858ddb731209 +49:18.174 [INFO] TestAgent.server: member joined, marking health alive: member=Node-47c3a6d1-91fb-4fa2-d351-858ddb731209 +49:18.174 [INFO] TestAgent.server: New leader elected: payload=Node-47c3a6d1-91fb-4fa2-d351-858ddb731209 +49:18.175 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.281 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:18.281 [INFO] TestAgent: Synced node info +49:18.281 [DEBUG] TestAgent: Node info in sync +49:18.315 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:18.315 [INFO] TestAgent: Requesting shutdown +49:18.315 [INFO] TestAgent.server: shutting down server +49:18.315 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.315 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.315 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.315 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.315 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.315 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.315 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.315 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.315 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.315 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.315 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.315Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.315Z [INFO] agent.router.manager: shutting down +49:18.315 [INFO] TestAgent: consul server down +49:18.315 [INFO] TestAgent: shutdown complete +49:18.315 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23842 network=tcp +49:18.315 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23842 network=udp +49:18.316 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23843 network=tcp +49:18.816 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.816 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Execute/#05 +2022-12-04T05:49:17.932Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.956Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.958 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5683693d-9eb9-e1b4-1c79-861ec25b744b Address:127.0.0.1:23819}]" +49:17.958 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23819 [Follower]" leader= +49:17.958 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5683693d-9eb9-e1b4-1c79-861ec25b744b.dc1 127.0.0.1 +49:17.959 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5683693d-9eb9-e1b4-1c79-861ec25b744b 127.0.0.1 +2022-12-04T05:49:17.959Z [INFO] agent.router: Initializing LAN area manager +49:17.959 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23814 network=udp +49:17.959 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5683693d-9eb9-e1b4-1c79-861ec25b744b.dc1 area=wan +49:17.959 [INFO] TestAgent.server: Adding LAN server: server="Node-5683693d-9eb9-e1b4-1c79-861ec25b744b (Addr: tcp/127.0.0.1:23819) (DC: dc1)" +49:17.959 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23814 network=tcp +49:17.960 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23815 network=tcp +49:17.960 [INFO] TestAgent: started state syncer +49:17.960 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23820 network=tcp +49:18.027 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.027 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23819 [Candidate]" term=2 +49:18.027 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.027 [DEBUG] TestAgent.server.raft: vote granted: from=5683693d-9eb9-e1b4-1c79-861ec25b744b term=2 tally=1 +49:18.027 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.027 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23819 [Leader]" +49:18.027 [INFO] TestAgent.server: cluster leadership acquired +49:18.027 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23819 +49:18.028 [INFO] TestAgent.server: New leader elected: payload=Node-5683693d-9eb9-e1b4-1c79-861ec25b744b +49:18.028 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.029 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.029 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.029 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.029 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.029 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.029 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5683693d-9eb9-e1b4-1c79-861ec25b744b +49:18.029 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5683693d-9eb9-e1b4-1c79-861ec25b744b +49:18.030 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.123 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:18.123 [INFO] TestAgent: Synced node info +49:18.123 [DEBUG] TestAgent: Node info in sync +49:18.331 [INFO] TestAgent: Requesting shutdown +49:18.331 [INFO] TestAgent.server: shutting down server +49:18.331 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.331 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.331 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.331 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.331 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.331 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.332 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.332 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.332 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.332 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.332 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.332Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.332Z [INFO] agent.router.manager: shutting down +49:18.332 [INFO] TestAgent: consul server down +49:18.332 [INFO] TestAgent: shutdown complete +49:18.332 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23814 network=tcp +49:18.332 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23814 network=udp +49:18.332 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23815 network=tcp +49:18.833 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.833 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionDeleteDestroy (0.91s) +=== CONT TestIntentionsMatch_noName +=== RUN TestSessionCreate_NoCheck/no_check_fields_should_yield_default_serfHealth +=== RUN TestSessionCreate_NoCheck/overwrite_nodechecks_to_associate_with_no_checks +=== RUN TestSessionCreate_NoCheck/overwrite_checks_to_associate_with_no_checks +2022-12-04T05:49:17.772Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.785Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.787 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a8a98642-a2da-cc26-9fc4-f8add2fee395 Address:127.0.0.1:23805}]" +49:17.787 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a8a98642-a2da-cc26-9fc4-f8add2fee395.secondary 127.0.0.1 +49:17.788 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23805 [Follower]" leader= +49:17.788 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a8a98642-a2da-cc26-9fc4-f8add2fee395 127.0.0.1 +2022-12-04T05:49:17.788Z [INFO] agent.router: Initializing LAN area manager +49:17.788 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a8a98642-a2da-cc26-9fc4-f8add2fee395.secondary area=wan +49:17.788 [INFO] TestAgent.server: Adding LAN server: server="Node-a8a98642-a2da-cc26-9fc4-f8add2fee395 (Addr: tcp/127.0.0.1:23805) (DC: secondary)" +49:17.788 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23800 network=tcp +49:17.788 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23800 network=udp +49:17.789 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23801 network=tcp +49:17.789 [INFO] TestAgent: started state syncer +49:17.789 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23806 network=tcp +49:17.843 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.843 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23805 [Candidate]" term=2 +49:17.843 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.843 [DEBUG] TestAgent.server.raft: vote granted: from=a8a98642-a2da-cc26-9fc4-f8add2fee395 term=2 tally=1 +49:17.843 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.843 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23805 [Leader]" +49:17.843 [INFO] TestAgent.server: cluster leadership acquired +49:17.843 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23805 +49:17.844 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.844 [INFO] TestAgent.server: New leader elected: payload=Node-a8a98642-a2da-cc26-9fc4-f8add2fee395 +49:17.845 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:17.845 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:17.845 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:17.845 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:17.845 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:17.845 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a8a98642-a2da-cc26-9fc4-f8add2fee395 +49:17.845 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a8a98642-a2da-cc26-9fc4-f8add2fee395 +49:17.846 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.987 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:17.988 [INFO] TestAgent: Synced node info +49:17.988 [DEBUG] TestAgent: Node info in sync +49:18.016 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:23756] +49:18.016 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:23756 +49:18.017 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6109dc4c-a41e-724a-84d9-b039341a38d5.primary 127.0.0.1 +49:18.017 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +49:18.017 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6109dc4c-a41e-724a-84d9-b039341a38d5.primary area=wan +49:18.019 [INFO] TestAgent: Requesting shutdown +49:18.019 [INFO] TestAgent.server: shutting down server +49:18.019 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.019 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.019 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.019 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.019 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.019 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.019 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.019 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.019 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.019 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.019 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.019Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.019Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.019Z [INFO] agent.router.manager: shutting down +49:18.019 [INFO] TestAgent: consul server down +49:18.019 [INFO] TestAgent: shutdown complete +49:18.019 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23800 network=tcp +49:18.019 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23800 network=udp +49:18.019 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23801 network=tcp +49:18.519 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.519 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:17.469Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.478Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.480 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6109dc4c-a41e-724a-84d9-b039341a38d5 Address:127.0.0.1:23757}]" +49:17.480 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23757 [Follower]" leader= +49:17.480 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6109dc4c-a41e-724a-84d9-b039341a38d5.primary 127.0.0.1 +49:17.481 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6109dc4c-a41e-724a-84d9-b039341a38d5 127.0.0.1 +2022-12-04T05:49:17.481Z [INFO] agent.router: Initializing LAN area manager +49:17.481 [INFO] TestAgent.server: Adding LAN server: server="Node-6109dc4c-a41e-724a-84d9-b039341a38d5 (Addr: tcp/127.0.0.1:23757) (DC: primary)" +49:17.481 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6109dc4c-a41e-724a-84d9-b039341a38d5.primary area=wan +49:17.481 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23752 network=udp +49:17.481 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23752 network=tcp +49:17.482 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23753 network=tcp +49:17.482 [INFO] TestAgent: started state syncer +49:17.482 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23758 network=tcp +49:17.526 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.526 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23757 [Candidate]" term=2 +49:17.526 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.526 [DEBUG] TestAgent.server.raft: vote granted: from=6109dc4c-a41e-724a-84d9-b039341a38d5 term=2 tally=1 +49:17.526 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.526 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23757 [Leader]" +49:17.526 [INFO] TestAgent.server: cluster leadership acquired +49:17.526 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23757 +49:17.526 [INFO] TestAgent.server: New leader elected: payload=Node-6109dc4c-a41e-724a-84d9-b039341a38d5 +49:17.527 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.528 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:17.528 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:17.528 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:17.528 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:17.528 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:17.528 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6109dc4c-a41e-724a-84d9-b039341a38d5 +49:17.528 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6109dc4c-a41e-724a-84d9-b039341a38d5 +49:17.529 [INFO] TestAgent.server: federation state anti-entropy synced +49:17.852 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:17.853 [INFO] TestAgent: Synced node info +49:18.016 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:41706 +49:18.017 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a8a98642-a2da-cc26-9fc4-f8add2fee395.secondary 127.0.0.1 +49:18.017 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a8a98642-a2da-cc26-9fc4-f8add2fee395.secondary area=wan +49:18.519 [INFO] TestAgent: Requesting shutdown +49:18.519 [INFO] TestAgent.server: shutting down server +49:18.519 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.519 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.519 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.519 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.519 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.519 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.519 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.519 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.520 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.520 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.520 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.520Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.520Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.520Z [INFO] agent.router.manager: shutting down +49:18.520 [INFO] TestAgent: consul server down +49:18.520 [INFO] TestAgent: shutdown complete +49:18.520 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23752 network=tcp +49:18.520 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23752 network=udp +49:18.520 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23753 network=tcp +49:19.020 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.020 [INFO] TestAgent: Endpoints down +--- PASS: TestStatusLeaderSecondary (1.56s) +=== CONT TestIntentionsMatch_byInvalid +2022-12-04T05:49:17.871Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:17.881Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:17.882 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f4a7e4f6-e4eb-afa6-ef7d-a7c405cb101b Address:127.0.0.1:23812}]" +49:17.882 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23812 [Follower]" leader= +49:17.882 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f4a7e4f6-e4eb-afa6-ef7d-a7c405cb101b.dc1 127.0.0.1 +49:17.883 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f4a7e4f6-e4eb-afa6-ef7d-a7c405cb101b 127.0.0.1 +2022-12-04T05:49:17.883Z [INFO] agent.router: Initializing LAN area manager +49:17.883 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23807 network=udp +49:17.883 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f4a7e4f6-e4eb-afa6-ef7d-a7c405cb101b.dc1 area=wan +49:17.883 [INFO] TestAgent.server: Adding LAN server: server="Node-f4a7e4f6-e4eb-afa6-ef7d-a7c405cb101b (Addr: tcp/127.0.0.1:23812) (DC: dc1)" +49:17.883 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23807 network=tcp +49:17.883 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23808 network=tcp +49:17.883 [INFO] TestAgent: started state syncer +49:17.883 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23813 network=tcp +49:17.932 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:17.932 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23812 [Candidate]" term=2 +49:17.932 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:17.932 [DEBUG] TestAgent.server.raft: vote granted: from=f4a7e4f6-e4eb-afa6-ef7d-a7c405cb101b term=2 tally=1 +49:17.932 [INFO] TestAgent.server.raft: election won: tally=1 +49:17.932 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23812 [Leader]" +49:17.932 [INFO] TestAgent.server: cluster leadership acquired +49:17.932 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23812 +49:17.933 [INFO] TestAgent.server: New leader elected: payload=Node-f4a7e4f6-e4eb-afa6-ef7d-a7c405cb101b +49:17.941 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:17.942 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:17.942 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:17.942 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:17.942 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:17.942 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:17.942 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f4a7e4f6-e4eb-afa6-ef7d-a7c405cb101b +49:17.942 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f4a7e4f6-e4eb-afa6-ef7d-a7c405cb101b +49:17.943 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.317 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:18.319 [INFO] TestAgent: Synced node info +49:18.522 [INFO] TestAgent: Requesting shutdown +49:18.522 [INFO] TestAgent.server: shutting down server +49:18.522 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.522 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.522 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.522 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.522 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.522 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.522 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.522 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.522 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.522 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.523 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.523Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.523Z [INFO] agent.router.manager: shutting down +49:18.523 [INFO] TestAgent: consul server down +49:18.523 [INFO] TestAgent: shutdown complete +49:18.523 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23807 network=tcp +49:18.523 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23807 network=udp +49:18.523 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23808 network=tcp +49:19.023 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.023 [INFO] TestAgent: Endpoints down +=== RUN TestTxnEndpoint_Bad_Size_Net/exceeds_default_max_kv_value_size +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:18.489Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:18.518Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:18.520 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f16b30a3-9610-7cdc-cac4-e04317dd31a1 Address:127.0.0.1:23896}]" +49:18.520 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23896 [Follower]" leader= +49:18.520 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f16b30a3-9610-7cdc-cac4-e04317dd31a1.dc1 127.0.0.1 +49:18.521 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f16b30a3-9610-7cdc-cac4-e04317dd31a1 127.0.0.1 +2022-12-04T05:49:18.521Z [INFO] agent.router: Initializing LAN area manager +49:18.521 [INFO] TestAgent.server: Adding LAN server: server="Node-f16b30a3-9610-7cdc-cac4-e04317dd31a1 (Addr: tcp/127.0.0.1:23896) (DC: dc1)" +49:18.521 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f16b30a3-9610-7cdc-cac4-e04317dd31a1.dc1 area=wan +49:18.521 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23891 network=udp +49:18.521 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23891 network=tcp +49:18.522 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23892 network=tcp +49:18.522 [INFO] TestAgent: started state syncer +49:18.522 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23883 network=tcp +49:18.560 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.560 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23896 [Candidate]" term=2 +49:18.560 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.560 [DEBUG] TestAgent.server.raft: vote granted: from=f16b30a3-9610-7cdc-cac4-e04317dd31a1 term=2 tally=1 +49:18.560 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.560 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23896 [Leader]" +49:18.560 [INFO] TestAgent.server: cluster leadership acquired +49:18.560 [INFO] TestAgent.server: New leader elected: payload=Node-f16b30a3-9610-7cdc-cac4-e04317dd31a1 +49:18.560 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23896 +49:18.561 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.562 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.562 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.562 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.562 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.562 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.562 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f16b30a3-9610-7cdc-cac4-e04317dd31a1 +49:18.562 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f16b30a3-9610-7cdc-cac4-e04317dd31a1 +49:18.562 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.624 [INFO] TestAgent: Requesting shutdown +49:18.624 [INFO] TestAgent.server: shutting down server +49:18.624 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.624 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.624 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.624 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.624 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.624 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.624 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.624 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:18.624 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.624 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.624 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.624 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.624Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.625Z [INFO] agent.router.manager: shutting down +49:18.625 [INFO] TestAgent: consul server down +49:18.625 [INFO] TestAgent: shutdown complete +49:18.625 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23891 network=tcp +49:18.625 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23891 network=udp +49:18.625 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23892 network=tcp +49:19.125 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.125 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsCheck_noSource (0.65s) +=== CONT TestParseConsistency_Invalid +2022-12-04T05:49:18.204Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:18.214Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:18.215 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d2bd3d80-0537-3f81-a664-9286407646e0 Address:127.0.0.1:23875}]" +49:18.215 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23875 [Follower]" leader= +49:18.216 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d2bd3d80-0537-3f81-a664-9286407646e0.dc1 127.0.0.1 +49:18.216 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d2bd3d80-0537-3f81-a664-9286407646e0 127.0.0.1 +2022-12-04T05:49:18.216Z [INFO] agent.router: Initializing LAN area manager +49:18.216 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23870 network=udp +49:18.216 [INFO] TestAgent.server: Adding LAN server: server="Node-d2bd3d80-0537-3f81-a664-9286407646e0 (Addr: tcp/127.0.0.1:23875) (DC: dc1)" +49:18.216 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d2bd3d80-0537-3f81-a664-9286407646e0.dc1 area=wan +49:18.216 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23870 network=tcp +49:18.217 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23871 network=tcp +49:18.217 [INFO] TestAgent: started state syncer +49:18.217 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23855 network=tcp +49:18.264 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.265 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23875 [Candidate]" term=2 +49:18.265 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.265 [DEBUG] TestAgent.server.raft: vote granted: from=d2bd3d80-0537-3f81-a664-9286407646e0 term=2 tally=1 +49:18.265 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.265 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23875 [Leader]" +49:18.265 [INFO] TestAgent.server: cluster leadership acquired +49:18.265 [INFO] TestAgent.server: New leader elected: payload=Node-d2bd3d80-0537-3f81-a664-9286407646e0 +49:18.265 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23875 +49:18.266 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.267 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.267 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.267 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.267 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.267 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.267 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d2bd3d80-0537-3f81-a664-9286407646e0 +49:18.267 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d2bd3d80-0537-3f81-a664-9286407646e0 +49:18.267 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.359 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:18.360 [INFO] TestAgent: Synced node info +49:18.360 [DEBUG] TestAgent: Node info in sync +49:18.656 [INFO] TestAgent: Requesting shutdown +49:18.656 [INFO] TestAgent.server: shutting down server +49:18.656 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.656 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.656 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.656 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.656 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.656 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.656 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.656 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.656 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.656 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.656 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.656Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.656Z [INFO] agent.router.manager: shutting down +49:18.656 [INFO] TestAgent: consul server down +49:18.656 [INFO] TestAgent: shutdown complete +49:18.656 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23870 network=tcp +49:18.656 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23870 network=udp +49:18.656 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23871 network=tcp +49:19.157 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.157 [INFO] TestAgent: Endpoints down +=== RUN TestTxnEndpoint_Bad_Size_Item/allowed +2022-12-04T05:49:18.446Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:18.458Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:18.460 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:81f1bbeb-856c-2537-aef1-bb59c882703e Address:127.0.0.1:23868}]" +49:18.460 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23868 [Follower]" leader= +49:18.460 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-81f1bbeb-856c-2537-aef1-bb59c882703e.dc1 127.0.0.1 +49:18.461 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-81f1bbeb-856c-2537-aef1-bb59c882703e 127.0.0.1 +2022-12-04T05:49:18.461Z [INFO] agent.router: Initializing LAN area manager +49:18.461 [INFO] TestAgent.server: Adding LAN server: server="Node-81f1bbeb-856c-2537-aef1-bb59c882703e (Addr: tcp/127.0.0.1:23868) (DC: dc1)" +49:18.461 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-81f1bbeb-856c-2537-aef1-bb59c882703e.dc1 area=wan +49:18.461 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23863 network=udp +49:18.461 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23863 network=tcp +49:18.461 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23864 network=tcp +49:18.461 [INFO] TestAgent: started state syncer +49:18.462 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23890 network=tcp +49:18.500 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.500 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23868 [Candidate]" term=2 +49:18.500 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.500 [DEBUG] TestAgent.server.raft: vote granted: from=81f1bbeb-856c-2537-aef1-bb59c882703e term=2 tally=1 +49:18.500 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.500 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23868 [Leader]" +49:18.500 [INFO] TestAgent.server: cluster leadership acquired +49:18.501 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23868 +49:18.502 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.504 [INFO] TestAgent.server: New leader elected: payload=Node-81f1bbeb-856c-2537-aef1-bb59c882703e +49:18.516 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.516 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.516 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.516 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.516 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.516 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-81f1bbeb-856c-2537-aef1-bb59c882703e +49:18.516 [INFO] TestAgent.server: member joined, marking health alive: member=Node-81f1bbeb-856c-2537-aef1-bb59c882703e +49:18.517 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.680 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:18.680 [INFO] TestAgent: Synced node info +49:18.680 [DEBUG] TestAgent: Node info in sync +49:18.690 [INFO] TestAgent: Requesting shutdown +49:18.690 [INFO] TestAgent.server: shutting down server +49:18.690 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.690 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.690 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.690 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.690 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.690 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.690 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.690 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.690 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.690 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.690 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.690Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.690Z [INFO] agent.router.manager: shutting down +49:18.690 [INFO] TestAgent: consul server down +49:18.690 [INFO] TestAgent: shutdown complete +49:18.690 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23863 network=tcp +49:18.690 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23863 network=udp +49:18.690 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23864 network=tcp +49:19.190 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.190 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsCheck_noDestination (0.76s) +=== CONT TestParseConsistency +2022-12-04T05:49:18.387Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:18.398Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:18.400 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:34d43a83-824a-b969-6f46-526b03508ae9 Address:127.0.0.1:23882}]" +49:18.400 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23882 [Follower]" leader= +49:18.401 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-34d43a83-824a-b969-6f46-526b03508ae9.dc1 127.0.0.1 +49:18.401 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-34d43a83-824a-b969-6f46-526b03508ae9 127.0.0.1 +2022-12-04T05:49:18.401Z [INFO] agent.router: Initializing LAN area manager +49:18.401 [INFO] TestAgent.server: Adding LAN server: server="Node-34d43a83-824a-b969-6f46-526b03508ae9 (Addr: tcp/127.0.0.1:23882) (DC: dc1)" +49:18.401 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-34d43a83-824a-b969-6f46-526b03508ae9.dc1 area=wan +49:18.401 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23877 network=udp +49:18.401 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23877 network=tcp +49:18.402 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23878 network=tcp +49:18.402 [INFO] TestAgent: started state syncer +49:18.402 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23862 network=tcp +49:18.440 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.440 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23882 [Candidate]" term=2 +49:18.440 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.440 [DEBUG] TestAgent.server.raft: vote granted: from=34d43a83-824a-b969-6f46-526b03508ae9 term=2 tally=1 +49:18.440 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.440 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23882 [Leader]" +49:18.440 [INFO] TestAgent.server: cluster leadership acquired +49:18.440 [INFO] TestAgent.server: New leader elected: payload=Node-34d43a83-824a-b969-6f46-526b03508ae9 +49:18.440 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23882 +49:18.441 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.443 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.443 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.443 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.443 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.443 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.443 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-34d43a83-824a-b969-6f46-526b03508ae9 +49:18.443 [INFO] TestAgent.server: member joined, marking health alive: member=Node-34d43a83-824a-b969-6f46-526b03508ae9 +49:18.443 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.694 [INFO] TestAgent: Requesting shutdown +49:18.694 [INFO] TestAgent.server: shutting down server +49:18.694 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.694 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.694 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.694 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.694 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.694 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.694 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.694 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:18.694 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.694 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.694 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.694 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.694Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.694Z [INFO] agent.router.manager: shutting down +49:18.694 [INFO] TestAgent: consul server down +49:18.694 [INFO] TestAgent: shutdown complete +49:18.694 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23877 network=tcp +49:18.694 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23877 network=udp +49:18.694 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23878 network=tcp +49:19.195 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.195 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionDestroy (0.82s) +=== CONT TestIntentionsMatch_noBy +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:18.548Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:18.557Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:18.559 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f865d454-55e1-632a-94c3-910928e92043 Address:127.0.0.1:23889}]" +49:18.559 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23889 [Follower]" leader= +49:18.559 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f865d454-55e1-632a-94c3-910928e92043.dc1 127.0.0.1 +49:18.560 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f865d454-55e1-632a-94c3-910928e92043 127.0.0.1 +2022-12-04T05:49:18.560Z [INFO] agent.router: Initializing LAN area manager +49:18.560 [INFO] TestAgent.server: Adding LAN server: server="Node-f865d454-55e1-632a-94c3-910928e92043 (Addr: tcp/127.0.0.1:23889) (DC: dc1)" +49:18.560 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f865d454-55e1-632a-94c3-910928e92043.dc1 area=wan +49:18.560 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23884 network=udp +49:18.560 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23884 network=tcp +49:18.560 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23885 network=tcp +49:18.560 [INFO] TestAgent: started state syncer +49:18.560 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23897 network=tcp +49:18.602 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.602 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23889 [Candidate]" term=2 +49:18.602 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.602 [DEBUG] TestAgent.server.raft: vote granted: from=f865d454-55e1-632a-94c3-910928e92043 term=2 tally=1 +49:18.602 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.602 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23889 [Leader]" +49:18.602 [INFO] TestAgent.server: cluster leadership acquired +49:18.602 [INFO] TestAgent.server: New leader elected: payload=Node-f865d454-55e1-632a-94c3-910928e92043 +49:18.602 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23889 +49:18.603 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.604 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.604 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.604 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.604 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.604 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.604 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f865d454-55e1-632a-94c3-910928e92043 +49:18.604 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f865d454-55e1-632a-94c3-910928e92043 +49:18.605 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.852 [INFO] TestAgent: Requesting shutdown +49:18.852 [INFO] TestAgent.server: shutting down server +49:18.852 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.852 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.852 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.852 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.852 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.853 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.853 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.853 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:18.853 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.853 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.853 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.853 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.853Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.853Z [INFO] agent.router.manager: shutting down +49:18.853 [INFO] TestAgent: consul server down +49:18.853 [INFO] TestAgent: shutdown complete +49:18.853 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23884 network=tcp +49:18.853 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23884 network=udp +49:18.853 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23885 network=tcp +49:19.353 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.353 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionCreate_NoCheck (0.82s) + --- PASS: TestSessionCreate_NoCheck/no_check_fields_should_yield_default_serfHealth (0.00s) + --- PASS: TestSessionCreate_NoCheck/overwrite_nodechecks_to_associate_with_no_checks (0.00s) + --- PASS: TestSessionCreate_NoCheck/overwrite_checks_to_associate_with_no_checks (0.00s) +=== CONT TestParseWait_InvalidIndex +--- PASS: TestParseWait_InvalidIndex (0.00s) +=== CONT TestParseWait_InvalidTime +--- PASS: TestParseWait_InvalidTime (0.00s) +=== CONT TestIntentionsMatch_basic +2022-12-04T05:49:18.781Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:18.789Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:18.791 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b92aaf14-69c5-0120-4bc0-e54b404bd664 Address:127.0.0.1:23938}]" +49:18.791 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23938 [Follower]" leader= +49:18.791 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b92aaf14-69c5-0120-4bc0-e54b404bd664.dc1 127.0.0.1 +49:18.792 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b92aaf14-69c5-0120-4bc0-e54b404bd664 127.0.0.1 +2022-12-04T05:49:18.792Z [INFO] agent.router: Initializing LAN area manager +49:18.792 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23933 network=udp +49:18.792 [INFO] TestAgent.server: Adding LAN server: server="Node-b92aaf14-69c5-0120-4bc0-e54b404bd664 (Addr: tcp/127.0.0.1:23938) (DC: dc1)" +49:18.792 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b92aaf14-69c5-0120-4bc0-e54b404bd664.dc1 area=wan +49:18.792 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23933 network=tcp +49:18.792 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23934 network=tcp +49:18.792 [INFO] TestAgent: started state syncer +49:18.793 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23918 network=tcp +49:18.840 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.840 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23938 [Candidate]" term=2 +49:18.841 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.841 [DEBUG] TestAgent.server.raft: vote granted: from=b92aaf14-69c5-0120-4bc0-e54b404bd664 term=2 tally=1 +49:18.841 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.841 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23938 [Leader]" +49:18.841 [INFO] TestAgent.server: cluster leadership acquired +49:18.841 [INFO] TestAgent.server: New leader elected: payload=Node-b92aaf14-69c5-0120-4bc0-e54b404bd664 +49:18.841 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23938 +49:18.841 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.843 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.843 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.843 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.843 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.843 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.843 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b92aaf14-69c5-0120-4bc0-e54b404bd664 +49:18.843 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b92aaf14-69c5-0120-4bc0-e54b404bd664 +49:18.843 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.866 [INFO] TestAgent: Requesting shutdown +49:18.866 [INFO] TestAgent.server: shutting down server +49:18.866 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.866 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.866 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.866 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.866 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.866 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.866 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.866 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:18.866 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.866 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.866 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.866 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.867Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.867Z [INFO] agent.router.manager: shutting down +49:18.867 [INFO] TestAgent: consul server down +49:18.867 [INFO] TestAgent: shutdown complete +49:18.867 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23933 network=tcp +49:18.867 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23933 network=udp +49:18.867 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23934 network=tcp +49:19.368 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.368 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionGet (1.32s) + --- PASS: TestSessionGet/#00 (0.72s) + --- PASS: TestSessionGet/#01 (0.60s) +=== CONT TestPProfHandlers_ACLs +2022-12-04T05:49:18.826Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:18.836Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:18.837 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:69a685a8-4923-1692-f4f5-bc2a197125ff Address:127.0.0.1:23924}]" +49:18.837 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23924 [Follower]" leader= +49:18.837 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-69a685a8-4923-1692-f4f5-bc2a197125ff.dc1 127.0.0.1 +49:18.838 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-69a685a8-4923-1692-f4f5-bc2a197125ff 127.0.0.1 +2022-12-04T05:49:18.838Z [INFO] agent.router: Initializing LAN area manager +49:18.838 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-69a685a8-4923-1692-f4f5-bc2a197125ff.dc1 area=wan +49:18.838 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23919 network=udp +49:18.838 [INFO] TestAgent.server: Adding LAN server: server="Node-69a685a8-4923-1692-f4f5-bc2a197125ff (Addr: tcp/127.0.0.1:23924) (DC: dc1)" +49:18.838 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23919 network=tcp +49:18.839 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23920 network=tcp +49:18.839 [INFO] TestAgent: started state syncer +49:18.839 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23946 network=tcp +49:18.888 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.888 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23924 [Candidate]" term=2 +49:18.888 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.888 [DEBUG] TestAgent.server.raft: vote granted: from=69a685a8-4923-1692-f4f5-bc2a197125ff term=2 tally=1 +49:18.888 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.888 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23924 [Leader]" +49:18.889 [INFO] TestAgent.server: cluster leadership acquired +49:18.889 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23924 +49:18.889 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.890 [INFO] TestAgent.server: New leader elected: payload=Node-69a685a8-4923-1692-f4f5-bc2a197125ff +49:18.890 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.890 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.890 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.890 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.890 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.890 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-69a685a8-4923-1692-f4f5-bc2a197125ff +49:18.890 [INFO] TestAgent.server: member joined, marking health alive: member=Node-69a685a8-4923-1692-f4f5-bc2a197125ff +49:18.891 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.929 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:18.930 [INFO] TestAgent: Requesting shutdown +49:18.930 [INFO] TestAgent.server: shutting down server +49:18.930 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.930 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.930 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.930 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.930 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.930 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.930 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.930 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:18.930 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.930 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.930 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.930 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.930Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.930Z [INFO] agent.router.manager: shutting down +49:18.930 [INFO] TestAgent: consul server down +49:18.930 [INFO] TestAgent: shutdown complete +49:18.930 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23919 network=tcp +49:18.930 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23919 network=udp +49:18.930 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23920 network=tcp +49:19.432 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.432 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:18.659Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:18.659Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:18.669Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:18.669Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:18.671 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1af6242c-d8f7-68d4-8000-cf57ea124f68 Address:127.0.0.1:23910}]" +49:18.671 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23910 [Follower]" leader= +49:18.671 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1af6242c-d8f7-68d4-8000-cf57ea124f68.dc1 127.0.0.1 +49:18.672 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1af6242c-d8f7-68d4-8000-cf57ea124f68 127.0.0.1 +2022-12-04T05:49:18.672Z [INFO] agent.router: Initializing LAN area manager +49:18.672 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23905 network=udp +49:18.672 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1af6242c-d8f7-68d4-8000-cf57ea124f68.dc1 area=wan +49:18.672 [INFO] TestAgent.server: Adding LAN server: server="Node-1af6242c-d8f7-68d4-8000-cf57ea124f68 (Addr: tcp/127.0.0.1:23910) (DC: dc1)" +49:18.672 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23905 network=tcp +49:18.673 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23906 network=tcp +49:18.673 [INFO] TestAgent: started state syncer +49:18.673 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23925 network=tcp +49:18.722 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:18.725 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.725 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23910 [Candidate]" term=2 +49:18.725 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.725 [DEBUG] TestAgent.server.raft: vote granted: from=1af6242c-d8f7-68d4-8000-cf57ea124f68 term=2 tally=1 +49:18.725 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.725 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23910 [Leader]" +49:18.725 [INFO] TestAgent.server: cluster leadership acquired +49:18.725 [INFO] TestAgent.server: New leader elected: payload=Node-1af6242c-d8f7-68d4-8000-cf57ea124f68 +49:18.725 [INFO] TestAgent.server: initializing acls +49:18.725 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:18.725 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:18.726 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:18.726 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:18.726 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:18.726 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:18.726 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1af6242c-d8f7-68d4-8000-cf57ea124f68 +49:18.726 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1af6242c-d8f7-68d4-8000-cf57ea124f68.dc1 +49:18.727 [INFO] TestAgent.server: Updating LAN server: server="Node-1af6242c-d8f7-68d4-8000-cf57ea124f68 (Addr: tcp/127.0.0.1:23910) (DC: dc1)" +49:18.727 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.727 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1af6242c-d8f7-68d4-8000-cf57ea124f68.dc1 area=wan +49:18.728 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.728 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.728 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.728 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.728 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.728 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1af6242c-d8f7-68d4-8000-cf57ea124f68 +49:18.728 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1af6242c-d8f7-68d4-8000-cf57ea124f68 +49:18.728 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.728 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1af6242c-d8f7-68d4-8000-cf57ea124f68 +49:18.742 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:18.743 [INFO] TestAgent: Synced node info +49:18.743 [DEBUG] TestAgent: Node info in sync +49:18.928 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-1af6242c-d8f7-68d4-8000-cf57ea124f68 +49:18.928 [INFO] TestAgent: Requesting shutdown +49:18.928 [INFO] TestAgent.server: shutting down server +49:18.928 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.928 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:18.928 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:18.928 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.928 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.928 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.928 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.929 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.929 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.929 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.929 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:18.929 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:18.929 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.929 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.929 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.929Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.929Z [INFO] agent.router.manager: shutting down +49:18.929 [INFO] TestAgent: consul server down +49:18.929 [INFO] TestAgent: shutdown complete +49:18.929 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23905 network=tcp +49:18.929 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23905 network=udp +49:18.929 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23906 network=tcp +49:19.432 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.432 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Execute/#06 +=== RUN TestSnapshot_Options/GET#02 +2022-12-04T05:49:18.714Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:18.723Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:18.725 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3d402505-73ec-fd68-3dc8-b2b6bfbbe700 Address:127.0.0.1:23917}]" +49:18.725 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23917 [Follower]" leader= +49:18.725 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3d402505-73ec-fd68-3dc8-b2b6bfbbe700.dc1 127.0.0.1 +49:18.726 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3d402505-73ec-fd68-3dc8-b2b6bfbbe700 127.0.0.1 +2022-12-04T05:49:18.726Z [INFO] agent.router: Initializing LAN area manager +49:18.726 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23912 network=udp +49:18.726 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3d402505-73ec-fd68-3dc8-b2b6bfbbe700.dc1 area=wan +49:18.726 [INFO] TestAgent.server: Adding LAN server: server="Node-3d402505-73ec-fd68-3dc8-b2b6bfbbe700 (Addr: tcp/127.0.0.1:23917) (DC: dc1)" +49:18.726 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23912 network=tcp +49:18.726 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23913 network=tcp +49:18.726 [INFO] TestAgent: started state syncer +49:18.727 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23932 network=tcp +49:18.765 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.765 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23917 [Candidate]" term=2 +49:18.765 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.765 [DEBUG] TestAgent.server.raft: vote granted: from=3d402505-73ec-fd68-3dc8-b2b6bfbbe700 term=2 tally=1 +49:18.765 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.765 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23917 [Leader]" +49:18.765 [INFO] TestAgent.server: cluster leadership acquired +49:18.765 [INFO] TestAgent.server: New leader elected: payload=Node-3d402505-73ec-fd68-3dc8-b2b6bfbbe700 +49:18.766 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23917 +49:18.766 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.767 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.767 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.767 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.767 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.767 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.768 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3d402505-73ec-fd68-3dc8-b2b6bfbbe700 +49:18.768 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3d402505-73ec-fd68-3dc8-b2b6bfbbe700 +49:18.768 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.934 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:18.935 [INFO] TestAgent: Synced node info +49:18.937 [INFO] TestAgent: Requesting shutdown +49:18.937 [INFO] TestAgent.server: shutting down server +49:18.937 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.937 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.937 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.937 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.937 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.937 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.937 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.937 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.937 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.937 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.937 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.937Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.937Z [INFO] agent.router.manager: shutting down +49:18.937 [INFO] TestAgent: consul server down +49:18.937 [INFO] TestAgent: shutdown complete +49:18.937 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23912 network=tcp +49:18.937 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23912 network=udp +49:18.937 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23913 network=tcp +49:19.439 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.439 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsCheck_basic (0.73s) +=== CONT TestIntentionsList_values +2022-12-04T05:49:18.688Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:18.697Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:18.699 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4f3b724c-24f6-e010-c85f-6606d5829deb Address:127.0.0.1:23931}]" +49:18.699 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23931 [Follower]" leader= +49:18.699 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4f3b724c-24f6-e010-c85f-6606d5829deb.dc1 127.0.0.1 +49:18.700 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4f3b724c-24f6-e010-c85f-6606d5829deb 127.0.0.1 +2022-12-04T05:49:18.700Z [INFO] agent.router: Initializing LAN area manager +49:18.700 [INFO] TestAgent.server: Adding LAN server: server="Node-4f3b724c-24f6-e010-c85f-6606d5829deb (Addr: tcp/127.0.0.1:23931) (DC: dc1)" +49:18.700 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4f3b724c-24f6-e010-c85f-6606d5829deb.dc1 area=wan +49:18.700 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23926 network=udp +49:18.700 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23926 network=tcp +49:18.701 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23927 network=tcp +49:18.701 [INFO] TestAgent: started state syncer +49:18.701 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23911 network=tcp +49:18.745 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.745 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23931 [Candidate]" term=2 +49:18.745 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.745 [DEBUG] TestAgent.server.raft: vote granted: from=4f3b724c-24f6-e010-c85f-6606d5829deb term=2 tally=1 +49:18.745 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.745 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23931 [Leader]" +49:18.745 [INFO] TestAgent.server: cluster leadership acquired +49:18.745 [INFO] TestAgent.server: New leader elected: payload=Node-4f3b724c-24f6-e010-c85f-6606d5829deb +49:18.745 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23931 +49:18.746 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.747 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.747 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.747 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.747 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.747 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.747 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4f3b724c-24f6-e010-c85f-6606d5829deb +49:18.747 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4f3b724c-24f6-e010-c85f-6606d5829deb +49:18.748 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.950 [INFO] TestAgent: Requesting shutdown +49:18.950 [INFO] TestAgent.server: shutting down server +49:18.950 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.950 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.950 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.950 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.950 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.950 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:18.950 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.950 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.950 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.950 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.950 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.950 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.950Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.950Z [INFO] agent.router.manager: shutting down +49:18.950 [INFO] TestAgent: consul server down +49:18.950 [INFO] TestAgent: shutdown complete +49:18.950 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23926 network=tcp +49:18.950 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23926 network=udp +49:18.950 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23927 network=tcp +49:19.451 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.451 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_InmemKeyrings (2.93s) + --- PASS: TestAgent_InmemKeyrings/no_keys (0.79s) + --- PASS: TestAgent_InmemKeyrings/server_with_keys (0.84s) + --- PASS: TestAgent_InmemKeyrings/client_with_keys (0.52s) + --- PASS: TestAgent_InmemKeyrings/ignore_files (0.77s) +=== CONT TestPProfHandlers_DisableDebugNoACLs +2022-12-04T05:49:18.590Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:18.599Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:18.600 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a19e3367-258d-b8b3-7f64-5ac008c9c0f1 Address:127.0.0.1:23903}]" +49:18.600 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23903 [Follower]" leader= +49:18.601 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a19e3367-258d-b8b3-7f64-5ac008c9c0f1.dc1 127.0.0.1 +49:18.601 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a19e3367-258d-b8b3-7f64-5ac008c9c0f1 127.0.0.1 +2022-12-04T05:49:18.601Z [INFO] agent.router: Initializing LAN area manager +49:18.601 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a19e3367-258d-b8b3-7f64-5ac008c9c0f1.dc1 area=wan +49:18.601 [INFO] TestAgent.server: Adding LAN server: server="Node-a19e3367-258d-b8b3-7f64-5ac008c9c0f1 (Addr: tcp/127.0.0.1:23903) (DC: dc1)" +49:18.601 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23898 network=udp +49:18.601 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23898 network=tcp +49:18.602 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23899 network=tcp +49:18.602 [INFO] TestAgent: started state syncer +49:18.602 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23904 network=tcp +49:18.664 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.664 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23903 [Candidate]" term=2 +49:18.664 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.664 [DEBUG] TestAgent.server.raft: vote granted: from=a19e3367-258d-b8b3-7f64-5ac008c9c0f1 term=2 tally=1 +49:18.664 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.664 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23903 [Leader]" +49:18.664 [INFO] TestAgent.server: cluster leadership acquired +49:18.665 [INFO] TestAgent.server: New leader elected: payload=Node-a19e3367-258d-b8b3-7f64-5ac008c9c0f1 +49:18.665 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23903 +49:18.665 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.667 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.667 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.667 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.667 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.667 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.667 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a19e3367-258d-b8b3-7f64-5ac008c9c0f1 +49:18.667 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a19e3367-258d-b8b3-7f64-5ac008c9c0f1 +49:18.667 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.895 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:18.895 [INFO] TestAgent: Synced node info +49:18.954 [INFO] TestAgent: Requesting shutdown +49:18.954 [INFO] TestAgent.server: shutting down server +49:18.954 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.954 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.954 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.954 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.954 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.954 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.955 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.955 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.955 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.955 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:18.955 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:18.955Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.955Z [INFO] agent.router.manager: shutting down +49:18.955 [INFO] TestAgent: consul server down +49:18.955 [INFO] TestAgent: shutdown complete +49:18.955 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23898 network=tcp +49:18.955 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23898 network=udp +49:18.955 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23899 network=tcp +49:19.455 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.455 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionCreate_DefaultCheck (0.88s) === CONT TestIntentionsList_empty -2021-10-31T23:13:04.822Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:05.037Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:05.125 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6b4eb63a-e67b-37d1-78cb-dccfcf8a12bc Address:127.0.0.1:16204}]" -13:05.126 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6b4eb63a-e67b-37d1-78cb-dccfcf8a12bc.dc1 127.0.0.1 -13:05.127 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6b4eb63a-e67b-37d1-78cb-dccfcf8a12bc 127.0.0.1 -2021-10-31T23:13:05.127Z [INFO] agent.router: Initializing LAN area manager -13:05.127 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16199 network=udp -13:05.127 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16204 [Follower]" leader= -13:05.135 [INFO] TestAgent.server: Adding LAN server: server="Node-6b4eb63a-e67b-37d1-78cb-dccfcf8a12bc (Addr: tcp/127.0.0.1:16204) (DC: dc1)" -13:05.135 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6b4eb63a-e67b-37d1-78cb-dccfcf8a12bc.dc1 area=wan -13:05.138 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16199 network=tcp -13:05.173 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.173 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16204 [Candidate]" term=2 -13:05.186 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.186 [DEBUG] TestAgent.server.raft: vote granted: from=6b4eb63a-e67b-37d1-78cb-dccfcf8a12bc term=2 tally=1 -13:05.186 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.186 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16204 [Leader]" -13:05.186 [INFO] TestAgent.server: cluster leadership acquired -13:05.186 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16204 -13:05.187 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.188 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.188 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.188 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.188 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.188 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.188 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6b4eb63a-e67b-37d1-78cb-dccfcf8a12bc -13:05.188 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6b4eb63a-e67b-37d1-78cb-dccfcf8a12bc -13:05.188 [INFO] TestAgent.server: New leader elected: payload=Node-6b4eb63a-e67b-37d1-78cb-dccfcf8a12bc -13:05.189 [INFO] TestAgent.server: federation state anti-entropy synced -13:05.204 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16200 network=tcp -13:05.204 [INFO] TestAgent: started state syncer -13:05.204 [INFO] TestAgent: Requesting shutdown -13:05.204 [INFO] TestAgent.server: shutting down server -13:05.204 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:05.204 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:05.204 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:05.204 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:05.204 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.204 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16205 network=tcp -13:05.204 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -13:05.204 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:05.204 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:05.204 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:05.204 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:05.204 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:05.204 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:05.205 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:05.205Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:05.205Z [INFO] agent.router.manager: shutting down -13:05.211 [WARN] TestAgent: Syncing node info failed.: error="leadership lost while committing log" -13:05.211 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="leadership lost while committing log" -13:05.225 [INFO] TestAgent: consul server down -13:05.226 [INFO] TestAgent: shutdown complete -13:05.226 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16199 network=tcp -13:05.226 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16199 network=udp -13:05.226 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16200 network=tcp -13:05.726 [INFO] TestAgent: Waiting for endpoints to shut down -13:05.726 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsMatch_byInvalid (1.03s) +[WARN] freeport: 10 out of 59 pending ports are still in use; something probably didn't wait around for the port to be closed! +49:19.083 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-b70d157c-1fe5-8585-b8a9-9ab267aedf8c 127.0.0.1 +2022-12-04T05:49:19.083Z [INFO] agent.router: Initializing LAN area manager +49:19.084 [INFO] TestAgent.client.serf.lan: serf: Attempting re-join to previously known node: Node-1b14785c-82ad-eb62-72cf-3bb6d1e1b0d8: 127.0.0.1:23405 +49:19.084 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy +49:19.084 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:23405: connect: connection refused +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +49:19.084 [INFO] TestAgent.client.serf.lan: serf: Attempting re-join to previously known node: Node-b0069722-72bd-cd56-8f04-15043a1fef7b: 127.0.0.1:23342 +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +49:19.084 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23961 network=udp +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +49:19.084 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +49:19.084 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23961 network=tcp +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +49:19.084 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:23342: connect: connection refused +49:19.084 [WARN] TestAgent.client.serf.lan: serf: Failed to re-join any previously known node +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +49:19.084 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +49:19.084 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +49:19.084 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +49:19.084 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +49:19.084 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +49:19.084 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +49:19.084 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +49:19.085 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +49:19.085 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +49:19.085 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" +49:19.085 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +49:19.085 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23962 network=tcp +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +49:19.085 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +49:19.085 [INFO] TestAgent: started state syncer +49:19.085 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +2022-12-04T05:49:19.084Z [WARN] agent.router.manager: No servers available +49:19.085 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +49:19.085 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23967 network=tcp +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +49:19.085 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +49:19.085 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +49:19.085 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +49:19.085 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +49:19.085 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +49:19.085 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +49:19.085 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +49:19.085 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +49:19.085 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +49:19.085 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:19.085Z [WARN] agent.router.manager: No servers available +49:19.085 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy +49:19.085 [INFO] TestAgent: Requesting shutdown +49:19.085 [INFO] TestAgent.client: shutting down client +49:19.085 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +49:19.085 [INFO] TestAgent: consul client down +49:19.085 [INFO] TestAgent: shutdown complete +49:19.085 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23961 network=tcp +49:19.085 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23961 network=udp +49:19.085 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23962 network=tcp +49:19.586 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.586 [INFO] TestAgent: Endpoints down +49:14.717 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-1b14785c-82ad-eb62-72cf-3bb6d1e1b0d8 127.0.0.1 +2022-12-04T05:49:14.717Z [INFO] agent.router: Initializing LAN area manager +49:14.718 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23402 network=udp +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23402 network=tcp +49:14.718 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +49:14.718 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +49:14.718 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" +49:14.718 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" +49:14.718 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23403 network=tcp +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [INFO] TestAgent: started state syncer +49:14.718 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +49:14.718 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23415 network=tcp +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.718 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +49:14.718 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +49:14.719 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +2022-12-04T05:49:14.718Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.719Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.719Z [WARN] agent.router.manager: No servers available +49:14.719 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +49:14.719 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:14.719Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.719Z [WARN] agent.router.manager: No servers available +49:14.719 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +49:14.719 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:23342] +2022-12-04T05:49:14.719Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.719Z [WARN] agent.router.manager: No servers available +49:14.719 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:14.719Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:14.719Z [WARN] agent.router.manager: No servers available +49:14.719 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:23342 +49:14.720 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-b0069722-72bd-cd56-8f04-15043a1fef7b 127.0.0.1 +49:14.720 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +49:14.720 [DEBUG] TestAgent: systemd notify failed: error="No socket" +2022-12-04T05:49:14.720Z [WARN] agent.router.manager: No servers available +49:14.720 [INFO] TestAgent.client: adding server: server="Node-b0069722-72bd-cd56-8f04-15043a1fef7b (Addr: tcp/127.0.0.1:23358) (DC: dc1)" +49:14.725 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader +49:14.925 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader +49:14.925 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-1b14785c-82ad-eb62-72cf-3bb6d1e1b0d8 +49:15.128 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader +49:15.128 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-1b14785c-82ad-eb62-72cf-3bb6d1e1b0d8 +49:15.326 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader +49:15.326 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-1b14785c-82ad-eb62-72cf-3bb6d1e1b0d8 +49:15.327 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-1b14785c-82ad-eb62-72cf-3bb6d1e1b0d8 +49:17.025 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +49:17.219 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:17.219 [INFO] TestAgent: Synced node info +49:17.219 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +49:17.219 [DEBUG] TestAgent: Node info in sync +49:17.219 [DEBUG] TestAgent: Service in sync: service=web-sidecar-proxy +49:18.010 [DEBUG] TestAgent: Node info in sync +49:18.011 [INFO] TestAgent: Synced service: service=web-sidecar-proxy +49:18.011 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +49:18.011 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +49:18.011 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +49:18.011 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" +49:18.031 [INFO] TestAgent: Requesting shutdown +49:18.031 [INFO] TestAgent.client: shutting down client +49:18.031 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +49:18.031 [INFO] TestAgent: consul client down +49:18.031 [INFO] TestAgent: shutdown complete +49:18.031 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23402 network=tcp +49:18.031 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23402 network=udp +49:18.031 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23403 network=tcp +49:18.031 [ERROR] TestAgent.client: RPC failed to server: method=DiscoveryChain.Get server=127.0.0.1:23358 error="rpc error making call: EOF" +49:18.031 [ERROR] TestAgent.client: RPC failed to server: method=ConnectCA.Roots server=127.0.0.1:23358 error="rpc error making call: EOF" +49:18.031 [ERROR] TestAgent.client: RPC failed to server: method=Intention.Match server=127.0.0.1:23358 error="rpc error making call: EOF" +49:18.031 [ERROR] TestAgent.client: RPC failed to server: method=ConnectCA.Roots server=127.0.0.1:23358 error="rpc error making call: EOF" +49:18.031 [ERROR] TestAgent.client: RPC failed to server: method=DiscoveryChain.Get server=127.0.0.1:23358 error="rpc error making call: EOF" +49:18.031 [ERROR] TestAgent.client: RPC failed to server: method=Health.ServiceNodes server=127.0.0.1:23358 error="rpc error making call: EOF" +49:18.031 [ERROR] TestAgent.client: RPC failed to server: method=ConfigEntry.ResolveServiceConfig server=127.0.0.1:23358 error="rpc error making call: EOF" +49:18.531 [INFO] TestAgent: Waiting for endpoints to shut down +49:18.531 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:14.247Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.322Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.323 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b0069722-72bd-cd56-8f04-15043a1fef7b Address:127.0.0.1:23358}]" +49:14.323 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23358 [Follower]" leader= +49:14.324 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b0069722-72bd-cd56-8f04-15043a1fef7b.dc1 127.0.0.1 +49:14.324 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b0069722-72bd-cd56-8f04-15043a1fef7b 127.0.0.1 +2022-12-04T05:49:14.324Z [INFO] agent.router: Initializing LAN area manager +49:14.324 [INFO] TestAgent.server: Adding LAN server: server="Node-b0069722-72bd-cd56-8f04-15043a1fef7b (Addr: tcp/127.0.0.1:23358) (DC: dc1)" +49:14.324 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23339 network=udp +49:14.324 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b0069722-72bd-cd56-8f04-15043a1fef7b.dc1 area=wan +49:14.324 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23339 network=tcp +49:14.325 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23340 network=tcp +49:14.325 [INFO] TestAgent: started state syncer +49:14.325 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23359 network=tcp +49:14.375 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.375 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23358 [Candidate]" term=2 +49:14.375 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.375 [DEBUG] TestAgent.server.raft: vote granted: from=b0069722-72bd-cd56-8f04-15043a1fef7b term=2 tally=1 +49:14.375 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.375 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23358 [Leader]" +49:14.375 [INFO] TestAgent.server: cluster leadership acquired +49:14.375 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23358 +49:14.376 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.376 [INFO] TestAgent.server: New leader elected: payload=Node-b0069722-72bd-cd56-8f04-15043a1fef7b +49:14.377 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.377 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.377 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.377 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.377 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.377 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b0069722-72bd-cd56-8f04-15043a1fef7b +49:14.377 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b0069722-72bd-cd56-8f04-15043a1fef7b +49:14.377 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.577 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:14.578 [INFO] TestAgent: Synced node info +49:14.578 [DEBUG] TestAgent: Node info in sync +49:14.719 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:55680 +49:14.719 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1b14785c-82ad-eb62-72cf-3bb6d1e1b0d8 127.0.0.1 +49:14.720 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1b14785c-82ad-eb62-72cf-3bb6d1e1b0d8 +49:14.919 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-1b14785c-82ad-eb62-72cf-3bb6d1e1b0d8 +49:15.120 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-1b14785c-82ad-eb62-72cf-3bb6d1e1b0d8 +49:15.318 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-1b14785c-82ad-eb62-72cf-3bb6d1e1b0d8 +49:15.327 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-1b14785c-82ad-eb62-72cf-3bb6d1e1b0d8 +49:17.258 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:17.258 [DEBUG] TestAgent: Node info in sync +49:18.531 [INFO] TestAgent: Requesting shutdown +49:18.531 [INFO] TestAgent.server: shutting down server +49:18.531 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:18.531 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:18.531 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:18.531 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:18.531 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:18.531 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:18.531 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:18.531 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:18.531 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:18.531Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:18.531Z [INFO] agent.router.manager: shutting down +49:18.531 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:18.531 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:18.532 [INFO] TestAgent: consul server down +49:18.532 [INFO] TestAgent: shutdown complete +49:18.532 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23339 network=tcp +49:18.532 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23339 network=udp +49:18.532 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23340 network=tcp +49:19.048 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.048 [INFO] TestAgent: Endpoints down +--- PASS: TestServiceManager_PersistService_ConfigFiles (5.41s) +=== CONT TestPProfHandlers_EnableDebug +=== RUN TestPProfHandlers_ACLs/case_0_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:200,_token:"master",_endpoint:"/debug/pprof/heap",_nilResponse:false}) +=== RUN TestPProfHandlers_ACLs/case_1_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"agent",_endpoint:"/debug/pprof/heap",_nilResponse:true}) +=== RUN TestPProfHandlers_ACLs/case_2_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"agent",_endpoint:"/debug/pprof/",_nilResponse:true}) +=== RUN TestPProfHandlers_ACLs/case_3_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"",_endpoint:"/debug/pprof/",_nilResponse:true}) +=== RUN TestPProfHandlers_ACLs/case_4_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:200,_token:"master",_endpoint:"/debug/pprof/heap",_nilResponse:false}) +2022-12-04T05:49:18.843Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:18.852Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:18.854 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5c7a6641-c51f-895b-e831-a08d9a36c01a Address:127.0.0.1:23952}]" +49:18.854 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23952 [Follower]" leader= +49:18.854 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5c7a6641-c51f-895b-e831-a08d9a36c01a.dc1 127.0.0.1 +49:18.854 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5c7a6641-c51f-895b-e831-a08d9a36c01a 127.0.0.1 +2022-12-04T05:49:18.854Z [INFO] agent.router: Initializing LAN area manager +49:18.855 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23947 network=udp +49:18.855 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5c7a6641-c51f-895b-e831-a08d9a36c01a.dc1 area=wan +49:18.855 [INFO] TestAgent.server: Adding LAN server: server="Node-5c7a6641-c51f-895b-e831-a08d9a36c01a (Addr: tcp/127.0.0.1:23952) (DC: dc1)" +49:18.855 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23947 network=tcp +49:18.855 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23948 network=tcp +49:18.855 [INFO] TestAgent: started state syncer +49:18.855 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23939 network=tcp +49:18.928 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.928 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23952 [Candidate]" term=2 +49:18.928 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.928 [DEBUG] TestAgent.server.raft: vote granted: from=5c7a6641-c51f-895b-e831-a08d9a36c01a term=2 tally=1 +49:18.928 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.928 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23952 [Leader]" +49:18.928 [INFO] TestAgent.server: cluster leadership acquired +49:18.928 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23952 +49:18.929 [INFO] TestAgent.server: New leader elected: payload=Node-5c7a6641-c51f-895b-e831-a08d9a36c01a +49:18.929 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.931 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.931 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.931 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.931 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.931 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.931 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5c7a6641-c51f-895b-e831-a08d9a36c01a +49:18.931 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5c7a6641-c51f-895b-e831-a08d9a36c01a +49:18.932 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.054 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:19.056 [INFO] TestAgent: Synced node info +49:19.056 [DEBUG] TestAgent: Node info in sync +49:19.134 [INFO] TestAgent: Requesting shutdown +49:19.134 [INFO] TestAgent.server: shutting down server +49:19.134 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:19.134 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:19.134 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:19.134 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:19.134 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.134 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.134 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:19.134 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:19.134 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:19.134 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:19.134 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:19.134Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:19.134Z [INFO] agent.router.manager: shutting down +49:19.134 [INFO] TestAgent: consul server down +49:19.134 [INFO] TestAgent: shutdown complete +49:19.134 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23947 network=tcp +49:19.134 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23947 network=udp +49:19.134 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23948 network=tcp +49:19.635 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.635 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsMatch_noName (0.80s) === CONT TestRPC_HTTPSMaxConnsPerClient === RUN TestRPC_HTTPSMaxConnsPerClient/HTTP -2021-10-31T23:13:04.849Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:05.124Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:05.127 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6cf09786-2c3c-7372-711a-05b4a7d39a95 Address:127.0.0.1:16239}]" -13:05.129 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6cf09786-2c3c-7372-711a-05b4a7d39a95.dc1 127.0.0.1 -13:05.133 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16239 [Follower]" leader= -13:05.138 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6cf09786-2c3c-7372-711a-05b4a7d39a95 127.0.0.1 -2021-10-31T23:13:05.138Z [INFO] agent.router: Initializing LAN area manager -13:05.139 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16234 network=udp -13:05.139 [INFO] TestAgent.server: Adding LAN server: server="Node-6cf09786-2c3c-7372-711a-05b4a7d39a95 (Addr: tcp/127.0.0.1:16239) (DC: dc1)" -13:05.173 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.173 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16239 [Candidate]" term=2 -13:05.186 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.186 [DEBUG] TestAgent.server.raft: vote granted: from=6cf09786-2c3c-7372-711a-05b4a7d39a95 term=2 tally=1 -13:05.186 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.186 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16239 [Leader]" -13:05.186 [INFO] TestAgent.server: cluster leadership acquired -13:05.186 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16239 -13:05.187 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.188 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.189 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.189 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.189 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.189 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.189 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6cf09786-2c3c-7372-711a-05b4a7d39a95 -13:05.189 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6cf09786-2c3c-7372-711a-05b4a7d39a95 -13:05.189 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6cf09786-2c3c-7372-711a-05b4a7d39a95.dc1 area=wan -13:05.189 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16234 network=tcp -13:05.189 [INFO] TestAgent.server: New leader elected: payload=Node-6cf09786-2c3c-7372-711a-05b4a7d39a95 -13:05.194 [INFO] TestAgent.server: federation state anti-entropy synced -13:05.197 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16235 network=tcp -13:05.198 [INFO] TestAgent: started state syncer -13:05.201 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16240 network=tcp -13:05.202 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:05.205 [INFO] TestAgent: Synced node info -13:05.214 [INFO] TestAgent: Requesting shutdown -13:05.214 [INFO] TestAgent.server: shutting down server -13:05.214 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:05.214 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:05.214 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:05.214 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:05.214 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.214 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:05.214 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:05.214 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:05.214 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:05.214 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:05.214 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:05.215Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:05.215Z [INFO] agent.router.manager: shutting down -13:05.242 [INFO] TestAgent: consul server down -13:05.242 [INFO] TestAgent: shutdown complete -13:05.242 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16234 network=tcp -13:05.242 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16234 network=udp -13:05.242 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16235 network=tcp -13:05.743 [INFO] TestAgent: Waiting for endpoints to shut down -13:05.743 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_PUT_GET_DELETE (1.05s) +=== RUN TestPProfHandlers_ACLs/case_5_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"towel",_endpoint:"/debug/pprof/heap",_nilResponse:true}) +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:19.201Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.210Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.211 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d21ca614-76ea-74d7-eea4-a698385abaee Address:127.0.0.1:23994}]" +49:19.211 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23994 [Follower]" leader= +49:19.212 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d21ca614-76ea-74d7-eea4-a698385abaee.dc1 127.0.0.1 +49:19.212 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d21ca614-76ea-74d7-eea4-a698385abaee 127.0.0.1 +2022-12-04T05:49:19.212Z [INFO] agent.router: Initializing LAN area manager +49:19.212 [INFO] TestAgent.server: Adding LAN server: server="Node-d21ca614-76ea-74d7-eea4-a698385abaee (Addr: tcp/127.0.0.1:23994) (DC: dc1)" +49:19.212 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23989 network=udp +49:19.212 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d21ca614-76ea-74d7-eea4-a698385abaee.dc1 area=wan +49:19.213 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23989 network=tcp +49:19.213 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23990 network=tcp +49:19.213 [INFO] TestAgent: started state syncer +49:19.213 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23981 network=tcp +49:19.257 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.257 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23994 [Candidate]" term=2 +49:19.258 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.258 [DEBUG] TestAgent.server.raft: vote granted: from=d21ca614-76ea-74d7-eea4-a698385abaee term=2 tally=1 +49:19.258 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.258 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23994 [Leader]" +49:19.258 [INFO] TestAgent.server: cluster leadership acquired +49:19.258 [INFO] TestAgent.server: New leader elected: payload=Node-d21ca614-76ea-74d7-eea4-a698385abaee +49:19.258 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23994 +49:19.259 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.260 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.260 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.260 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.260 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.260 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.260 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d21ca614-76ea-74d7-eea4-a698385abaee +49:19.260 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d21ca614-76ea-74d7-eea4-a698385abaee +49:19.260 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.306 [INFO] TestAgent: Requesting shutdown +49:19.306 [INFO] TestAgent.server: shutting down server +49:19.306 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:19.306 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:19.306 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:19.306 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:19.306 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.306 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.306 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:19.306 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:19.306 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:19.306 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:19.306 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:19.306 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:19.306Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:19.306Z [INFO] agent.router.manager: shutting down +49:19.306 [INFO] TestAgent: consul server down +49:19.306 [INFO] TestAgent: shutdown complete +49:19.306 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23989 network=tcp +49:19.306 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23989 network=udp +49:19.306 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23990 network=tcp +49:19.807 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.807 [INFO] TestAgent: Endpoints down +--- PASS: TestParseConsistency (0.62s) +=== CONT TestParseWait +--- PASS: TestParseWait (0.00s) === CONT TestHTTPServer_HandshakeTimeout -2021-10-31T23:13:04.914Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:05.027Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:05.038 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5defa723-3aba-1eb5-cff2-f02194d23f24 Address:127.0.0.1:16232}]" -13:05.041 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16232 [Follower]" leader= -13:05.102 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.102 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16232 [Candidate]" term=2 -13:05.102 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=5defa723-3aba-1eb5-cff2-f02194d23f24 fallback=127.0.0.1:16232 error="Could not find address for server id 5defa723-3aba-1eb5-cff2-f02194d23f24" -13:05.102 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.102 [DEBUG] TestAgent.server.raft: vote granted: from=5defa723-3aba-1eb5-cff2-f02194d23f24 term=2 tally=1 -13:05.102 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.102 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16232 [Leader]" -13:05.126 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5defa723-3aba-1eb5-cff2-f02194d23f24.dc1 127.0.0.1 -13:05.128 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5defa723-3aba-1eb5-cff2-f02194d23f24 127.0.0.1 -2021-10-31T23:13:05.128Z [INFO] agent.router: Initializing LAN area manager -13:05.129 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16227 network=udp -13:05.134 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5defa723-3aba-1eb5-cff2-f02194d23f24.dc1 area=wan -13:05.135 [INFO] TestAgent.server: Adding LAN server: server="Node-5defa723-3aba-1eb5-cff2-f02194d23f24 (Addr: tcp/127.0.0.1:16232) (DC: dc1)" -13:05.136 [INFO] TestAgent.server: cluster leadership acquired -13:05.136 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16232 -13:05.137 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.138 [INFO] TestAgent.server: New leader elected: payload=Node-5defa723-3aba-1eb5-cff2-f02194d23f24 -13:05.138 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16227 network=tcp -13:05.139 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16228 network=tcp -13:05.139 [INFO] TestAgent: started state syncer -13:05.140 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16233 network=tcp -13:05.205 [INFO] TestAgent: Synced node info -13:05.205 [DEBUG] TestAgent: Node info in sync -13:05.212 [INFO] TestAgent: Requesting shutdown -13:05.212 [INFO] TestAgent.server: shutting down server -13:05.212 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.212 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:05.212Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:05.212Z [INFO] agent.router.manager: shutting down -13:05.212 [ERROR] TestAgent.server: failed to establish leadership: error="error generating CA root certificate: leadership lost while committing log" -13:05.212 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=0 retry_limit=3 error="raft is already shutdown" -13:05.234 [INFO] TestAgent: consul server down -13:05.234 [INFO] TestAgent: shutdown complete -13:05.234 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16227 network=tcp -13:05.234 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16227 network=udp -13:05.234 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16228 network=tcp -13:05.745 [INFO] TestAgent: Waiting for endpoints to shut down -13:05.745 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:05.017Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:05.131Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:05.134 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2ecc1caf-5bdd-9f81-009a-e4e100b06b10 Address:127.0.0.1:16211}]" -13:05.135 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16211 [Follower]" leader= -13:05.135 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2ecc1caf-5bdd-9f81-009a-e4e100b06b10.dc1 127.0.0.1 -13:05.136 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2ecc1caf-5bdd-9f81-009a-e4e100b06b10 127.0.0.1 -2021-10-31T23:13:05.136Z [INFO] agent.router: Initializing LAN area manager -13:05.137 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16206 network=udp -13:05.137 [INFO] TestAgent.server: Adding LAN server: server="Node-2ecc1caf-5bdd-9f81-009a-e4e100b06b10 (Addr: tcp/127.0.0.1:16211) (DC: dc1)" -13:05.137 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2ecc1caf-5bdd-9f81-009a-e4e100b06b10.dc1 area=wan -13:05.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16206 network=tcp -13:05.186 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16207 network=tcp -13:05.186 [INFO] TestAgent: started state syncer -13:05.190 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16212 network=tcp -13:05.195 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.195 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16211 [Candidate]" term=2 -13:05.195 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.195 [DEBUG] TestAgent.server.raft: vote granted: from=2ecc1caf-5bdd-9f81-009a-e4e100b06b10 term=2 tally=1 -13:05.195 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.195 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16211 [Leader]" -13:05.195 [INFO] TestAgent.server: cluster leadership acquired -13:05.196 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16211 -13:05.197 [INFO] TestAgent.server: New leader elected: payload=Node-2ecc1caf-5bdd-9f81-009a-e4e100b06b10 -13:05.210 [INFO] TestAgent: Requesting shutdown -13:05.210 [INFO] TestAgent.server: shutting down server -13:05.210 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.210 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:05.210 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:05.210Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:05.210Z [INFO] agent.router.manager: shutting down -13:05.212 [ERROR] TestAgent.server: failed to establish leadership: error="error configuring provider: leadership lost while committing log" -13:05.212 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=0 retry_limit=3 error="raft is already shutdown" -13:05.234 [INFO] TestAgent: consul server down -13:05.234 [INFO] TestAgent: shutdown complete -13:05.234 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16206 network=tcp -13:05.234 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16206 network=udp -13:05.234 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16207 network=tcp -13:05.745 [INFO] TestAgent: Waiting for endpoints to shut down -13:05.745 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsCheck_basic (1.05s) +2022-12-04T05:49:19.031Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.061Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.063 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:355c1f67-5c2b-4aa2-f785-039f87456725 Address:127.0.0.1:23945}]" +49:19.063 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-355c1f67-5c2b-4aa2-f785-039f87456725.dc1 127.0.0.1 +49:19.064 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-355c1f67-5c2b-4aa2-f785-039f87456725 127.0.0.1 +2022-12-04T05:49:19.064Z [INFO] agent.router: Initializing LAN area manager +49:19.064 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23945 [Follower]" leader= +49:19.064 [INFO] TestAgent.server: Adding LAN server: server="Node-355c1f67-5c2b-4aa2-f785-039f87456725 (Addr: tcp/127.0.0.1:23945) (DC: dc1)" +49:19.064 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-355c1f67-5c2b-4aa2-f785-039f87456725.dc1 area=wan +49:19.064 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23940 network=tcp +49:19.065 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23940 network=udp +49:19.065 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23941 network=tcp +49:19.065 [INFO] TestAgent: started state syncer +49:19.065 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23953 network=tcp +49:19.123 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.123 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23945 [Candidate]" term=2 +49:19.124 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.124 [DEBUG] TestAgent.server.raft: vote granted: from=355c1f67-5c2b-4aa2-f785-039f87456725 term=2 tally=1 +49:19.124 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.124 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23945 [Leader]" +49:19.124 [INFO] TestAgent.server: cluster leadership acquired +49:19.124 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23945 +49:19.124 [INFO] TestAgent.server: New leader elected: payload=Node-355c1f67-5c2b-4aa2-f785-039f87456725 +49:19.125 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.126 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.126 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.126 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.126 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.126 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.127 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-355c1f67-5c2b-4aa2-f785-039f87456725 +49:19.127 [INFO] TestAgent.server: member joined, marking health alive: member=Node-355c1f67-5c2b-4aa2-f785-039f87456725 +49:19.127 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.139 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:19.139 [INFO] TestAgent: Synced node info +49:19.307 [INFO] TestAgent: Requesting shutdown +49:19.307 [INFO] TestAgent.server: shutting down server +49:19.307 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:19.307 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:19.307 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:19.307 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:19.307 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.307 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.307 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:19.307 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:19.307 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:19.307 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:19.307 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:19.307Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:19.307Z [INFO] agent.router.manager: shutting down +49:19.308 [INFO] TestAgent: consul server down +49:19.308 [INFO] TestAgent: shutdown complete +49:19.308 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23940 network=tcp +49:19.308 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23940 network=udp +49:19.308 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23941 network=tcp +49:19.813 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.813 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsMatch_byInvalid (0.79s) +=== CONT TestParseSource +2022-12-04T05:49:19.137Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.148Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.150 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bfb7d346-e2fc-6b1a-7a14-9d7e02b203d4 Address:127.0.0.1:23973}]" +49:19.150 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23973 [Follower]" leader= +49:19.150 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bfb7d346-e2fc-6b1a-7a14-9d7e02b203d4.dc1 127.0.0.1 +49:19.151 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bfb7d346-e2fc-6b1a-7a14-9d7e02b203d4 127.0.0.1 +2022-12-04T05:49:19.151Z [INFO] agent.router: Initializing LAN area manager +49:19.151 [INFO] TestAgent.server: Adding LAN server: server="Node-bfb7d346-e2fc-6b1a-7a14-9d7e02b203d4 (Addr: tcp/127.0.0.1:23973) (DC: dc1)" +49:19.151 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bfb7d346-e2fc-6b1a-7a14-9d7e02b203d4.dc1 area=wan +49:19.151 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23968 network=udp +49:19.151 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23968 network=tcp +49:19.152 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23969 network=tcp +49:19.152 [INFO] TestAgent: started state syncer +49:19.152 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23974 network=tcp +49:19.209 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.209 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23973 [Candidate]" term=2 +49:19.209 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.209 [DEBUG] TestAgent.server.raft: vote granted: from=bfb7d346-e2fc-6b1a-7a14-9d7e02b203d4 term=2 tally=1 +49:19.209 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.209 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23973 [Leader]" +49:19.209 [INFO] TestAgent.server: cluster leadership acquired +49:19.209 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23973 +49:19.210 [INFO] TestAgent.server: New leader elected: payload=Node-bfb7d346-e2fc-6b1a-7a14-9d7e02b203d4 +49:19.210 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.212 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.212 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.212 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.212 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.212 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.212 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bfb7d346-e2fc-6b1a-7a14-9d7e02b203d4 +49:19.212 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bfb7d346-e2fc-6b1a-7a14-9d7e02b203d4 +49:19.212 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.353 [INFO] TestAgent: Requesting shutdown +49:19.353 [INFO] TestAgent.server: shutting down server +49:19.354 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:19.354 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:19.354 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:19.354 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:19.354 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.354 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.354 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:19.354 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:19.354 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:19.354 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:19.354 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:19.354 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:19.354Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:19.354Z [INFO] agent.router.manager: shutting down +49:19.354 [INFO] TestAgent: consul server down +49:19.354 [INFO] TestAgent: shutdown complete +49:19.354 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23968 network=tcp +49:19.354 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23968 network=udp +49:19.354 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23969 network=tcp +49:19.855 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.855 [INFO] TestAgent: Endpoints down +--- PASS: TestParseConsistency_Invalid (0.73s) === CONT TestEnableWebUI ---- PASS: TestIntentionsCheck_noSource (1.05s) +2022-12-04T05:49:18.357Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:18.370Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:18.371 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5028eeb7-ddb5-ec4c-4933-27114b1abadd Address:127.0.0.1:23861}]" +49:18.371 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23861 [Follower]" leader= +49:18.372 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5028eeb7-ddb5-ec4c-4933-27114b1abadd.dc1 127.0.0.1 +49:18.372 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5028eeb7-ddb5-ec4c-4933-27114b1abadd 127.0.0.1 +2022-12-04T05:49:18.372Z [INFO] agent.router: Initializing LAN area manager +49:18.372 [INFO] TestAgent.server: Adding LAN server: server="Node-5028eeb7-ddb5-ec4c-4933-27114b1abadd (Addr: tcp/127.0.0.1:23861) (DC: dc1)" +49:18.372 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5028eeb7-ddb5-ec4c-4933-27114b1abadd.dc1 area=wan +49:18.373 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23856 network=udp +49:18.373 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23856 network=tcp +49:18.373 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23857 network=tcp +49:18.373 [INFO] TestAgent: started state syncer +49:18.373 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23876 network=tcp +49:18.411 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:18.411 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23861 [Candidate]" term=2 +49:18.411 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:18.411 [DEBUG] TestAgent.server.raft: vote granted: from=5028eeb7-ddb5-ec4c-4933-27114b1abadd term=2 tally=1 +49:18.411 [INFO] TestAgent.server.raft: election won: tally=1 +49:18.411 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23861 [Leader]" +49:18.411 [INFO] TestAgent.server: cluster leadership acquired +49:18.411 [INFO] TestAgent.server: New leader elected: payload=Node-5028eeb7-ddb5-ec4c-4933-27114b1abadd +49:18.411 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23861 +49:18.412 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:18.413 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:18.413 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:18.413 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:18.413 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:18.413 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:18.413 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5028eeb7-ddb5-ec4c-4933-27114b1abadd +49:18.413 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5028eeb7-ddb5-ec4c-4933-27114b1abadd +49:18.414 [INFO] TestAgent.server: federation state anti-entropy synced +49:18.554 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:18.555 [INFO] TestAgent: Synced node info +49:18.555 [DEBUG] TestAgent: Node info in sync +49:19.108 [DEBUG] TestAgent.server: Session TTL expired: session=82bee117-9071-713b-c6c7-cc5f450f00af +49:19.358 [INFO] TestAgent: Requesting shutdown +49:19.358 [INFO] TestAgent.server: shutting down server +49:19.358 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:19.358 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:19.358 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:19.358 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:19.358 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.358 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.358 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:19.358 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:19.358 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:19.358 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:19.358 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:19.358Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:19.358Z [INFO] agent.router.manager: shutting down +49:19.358 [INFO] TestAgent: consul server down +49:19.358 [INFO] TestAgent: shutdown complete +49:19.358 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23856 network=tcp +49:19.358 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23856 network=udp +49:19.358 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23857 network=tcp +49:19.858 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.858 [INFO] TestAgent: Endpoints down +--- PASS: TestSessionCustomTTL (1.52s) +=== CONT TestPrettyPrintBare +2022-12-04T05:49:19.204Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.215Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.217 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c7db7da5-f7ef-4b23-a5ae-8230028121a9 Address:127.0.0.1:23987}]" +49:19.217 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23987 [Follower]" leader= +49:19.218 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c7db7da5-f7ef-4b23-a5ae-8230028121a9.dc1 127.0.0.1 +49:19.218 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c7db7da5-f7ef-4b23-a5ae-8230028121a9 127.0.0.1 +2022-12-04T05:49:19.218Z [INFO] agent.router: Initializing LAN area manager +49:19.218 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c7db7da5-f7ef-4b23-a5ae-8230028121a9.dc1 area=wan +49:19.218 [INFO] TestAgent.server: Adding LAN server: server="Node-c7db7da5-f7ef-4b23-a5ae-8230028121a9 (Addr: tcp/127.0.0.1:23987) (DC: dc1)" +49:19.218 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23982 network=udp +49:19.218 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23982 network=tcp +49:19.219 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23983 network=tcp +49:19.219 [INFO] TestAgent: started state syncer +49:19.219 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24002 network=tcp +49:19.261 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.261 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23987 [Candidate]" term=2 +49:19.261 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.261 [DEBUG] TestAgent.server.raft: vote granted: from=c7db7da5-f7ef-4b23-a5ae-8230028121a9 term=2 tally=1 +49:19.261 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.261 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23987 [Leader]" +49:19.262 [INFO] TestAgent.server: cluster leadership acquired +49:19.262 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23987 +49:19.262 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.262 [INFO] TestAgent.server: New leader elected: payload=Node-c7db7da5-f7ef-4b23-a5ae-8230028121a9 +49:19.263 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.263 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.263 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.263 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.263 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.263 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c7db7da5-f7ef-4b23-a5ae-8230028121a9 +49:19.263 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c7db7da5-f7ef-4b23-a5ae-8230028121a9 +49:19.264 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.346 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:19.346 [INFO] TestAgent: Synced node info +49:19.495 [INFO] TestAgent: Requesting shutdown +49:19.495 [INFO] TestAgent.server: shutting down server +49:19.495 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:19.495 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:19.495 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:19.495 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:19.495 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.496 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.496 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:19.496 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:19.496 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:19.496 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:19.496 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:19.498Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:19.498Z [INFO] agent.router.manager: shutting down +49:19.498 [INFO] TestAgent: consul server down +49:19.498 [INFO] TestAgent: shutdown complete +49:19.498 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23982 network=tcp +49:19.498 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23982 network=udp +49:19.499 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23983 network=tcp +49:19.999 [INFO] TestAgent: Waiting for endpoints to shut down +49:19.999 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsMatch_noBy (0.80s) === CONT TestACLResolution -2021-10-31T23:13:04.808Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:04.935Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:04.937 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4019ff61-b01e-e969-42ed-4659600c5766 Address:127.0.0.1:16176}]" -13:04.955 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4019ff61-b01e-e969-42ed-4659600c5766.dc1 127.0.0.1 -13:04.956 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4019ff61-b01e-e969-42ed-4659600c5766 127.0.0.1 -2021-10-31T23:13:04.956Z [INFO] agent.router: Initializing LAN area manager -13:04.956 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16185 network=udp -13:04.967 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4019ff61-b01e-e969-42ed-4659600c5766.dc1 area=wan -13:04.967 [INFO] TestAgent.server: Adding LAN server: server="Node-4019ff61-b01e-e969-42ed-4659600c5766 (Addr: tcp/127.0.0.1:16176) (DC: dc1)" -13:04.967 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16185 network=tcp -13:04.968 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16176 [Follower]" leader= -13:05.028 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.028 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16176 [Candidate]" term=2 -13:05.029 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16186 network=tcp -13:05.029 [INFO] TestAgent: started state syncer -13:05.029 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16177 network=tcp -13:05.034 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.034 [DEBUG] TestAgent.server.raft: vote granted: from=4019ff61-b01e-e969-42ed-4659600c5766 term=2 tally=1 -13:05.034 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.034 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16176 [Leader]" -13:05.041 [INFO] TestAgent.server: cluster leadership acquired -13:05.042 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16176 -13:05.042 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.044 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.044 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.044 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.044 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.044 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.044 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4019ff61-b01e-e969-42ed-4659600c5766 -13:05.044 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4019ff61-b01e-e969-42ed-4659600c5766 -13:05.044 [INFO] TestAgent.server: New leader elected: payload=Node-4019ff61-b01e-e969-42ed-4659600c5766 -13:05.045 [INFO] TestAgent.server: federation state anti-entropy synced -13:05.236 [INFO] TestAgent.server.serf.wan: serf: Received install-key query -13:05.242 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-4019ff61-b01e-e969-42ed-4659600c5766.dc1 -13:05.242 [INFO] TestAgent.server.serf.lan: serf: Received install-key query -13:05.242 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-4019ff61-b01e-e969-42ed-4659600c5766 -13:05.242 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -13:05.242 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-4019ff61-b01e-e969-42ed-4659600c5766.dc1 -13:05.242 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -13:05.243 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-4019ff61-b01e-e969-42ed-4659600c5766 -13:05.243 [INFO] TestAgent.server.serf.wan: serf: Received remove-key query -13:05.243 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-4019ff61-b01e-e969-42ed-4659600c5766.dc1 -13:05.243 [INFO] TestAgent.server.serf.lan: serf: Received remove-key query -13:05.243 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-4019ff61-b01e-e969-42ed-4659600c5766 -13:05.244 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -13:05.244 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-4019ff61-b01e-e969-42ed-4659600c5766.dc1 -13:05.244 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -13:05.245 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-4019ff61-b01e-e969-42ed-4659600c5766 -13:05.245 [INFO] TestAgent: Requesting shutdown -13:05.245 [INFO] TestAgent.server: shutting down server -13:05.245 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:05.245 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:05.245 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:05.245 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:05.245 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.245 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:05.245 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:05.245 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:05.245 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:05.245 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:05.245 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:05.245 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:05.245Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:05.245Z [INFO] agent.router.manager: shutting down -13:05.250 [INFO] TestAgent: consul server down -13:05.250 [INFO] TestAgent: shutdown complete -13:05.250 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16185 network=tcp -13:05.250 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16185 network=udp -13:05.250 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16186 network=tcp -13:05.753 [INFO] TestAgent: Waiting for endpoints to shut down -13:05.753 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_KeyringRemove (1.06s) +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:19.363Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.395Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.397 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a9978472-c016-b6a4-a48a-b6ca7b9fc7c3 Address:127.0.0.1:24008}]" +49:19.397 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a9978472-c016-b6a4-a48a-b6ca7b9fc7c3.dc1 127.0.0.1 +49:19.397 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a9978472-c016-b6a4-a48a-b6ca7b9fc7c3 127.0.0.1 +2022-12-04T05:49:19.397Z [INFO] agent.router: Initializing LAN area manager +49:19.398 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24003 network=udp +49:19.398 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24008 [Follower]" leader= +49:19.398 [INFO] TestAgent.server: Adding LAN server: server="Node-a9978472-c016-b6a4-a48a-b6ca7b9fc7c3 (Addr: tcp/127.0.0.1:24008) (DC: dc1)" +49:19.398 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a9978472-c016-b6a4-a48a-b6ca7b9fc7c3.dc1 area=wan +49:19.398 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24003 network=tcp +49:19.398 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24004 network=tcp +49:19.398 [INFO] TestAgent: started state syncer +49:19.398 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23995 network=tcp +49:19.464 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.464 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24008 [Candidate]" term=2 +49:19.464 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.464 [DEBUG] TestAgent.server.raft: vote granted: from=a9978472-c016-b6a4-a48a-b6ca7b9fc7c3 term=2 tally=1 +49:19.464 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.464 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24008 [Leader]" +49:19.464 [INFO] TestAgent.server: cluster leadership acquired +49:19.464 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24008 +49:19.468 [INFO] TestAgent.server: New leader elected: payload=Node-a9978472-c016-b6a4-a48a-b6ca7b9fc7c3 +49:19.481 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.496 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.496 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.496 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.496 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.496 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.496 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a9978472-c016-b6a4-a48a-b6ca7b9fc7c3 +49:19.496 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a9978472-c016-b6a4-a48a-b6ca7b9fc7c3 +49:19.497 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.573 [INFO] TestAgent: Requesting shutdown +49:19.574 [INFO] TestAgent.server: shutting down server +49:19.574 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:19.574 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:19.574 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:19.574 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:19.574 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.574 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.574 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:19.574 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:19.574 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:19.574 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:19.574 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:19.574 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:19.574Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:19.574Z [INFO] agent.router.manager: shutting down +49:19.574 [INFO] TestAgent: consul server down +49:19.574 [INFO] TestAgent: shutdown complete +49:19.574 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24003 network=tcp +49:19.574 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24003 network=udp +49:19.574 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24004 network=tcp +49:20.074 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.074 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsMatch_basic (0.72s) +=== CONT TestPrettyPrint +2022-12-04T05:49:19.167Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.176Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.177 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9dca7eb7-a67a-45a0-4b55-bdd8204ca7a1 Address:127.0.0.1:23980}]" +49:19.177 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23980 [Follower]" leader= +49:19.178 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9dca7eb7-a67a-45a0-4b55-bdd8204ca7a1.dc1 127.0.0.1 +49:19.178 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9dca7eb7-a67a-45a0-4b55-bdd8204ca7a1 127.0.0.1 +2022-12-04T05:49:19.178Z [INFO] agent.router: Initializing LAN area manager +49:19.178 [INFO] TestAgent.server: Adding LAN server: server="Node-9dca7eb7-a67a-45a0-4b55-bdd8204ca7a1 (Addr: tcp/127.0.0.1:23980) (DC: dc1)" +49:19.178 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9dca7eb7-a67a-45a0-4b55-bdd8204ca7a1.dc1 area=wan +49:19.178 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23975 network=udp +49:19.178 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23975 network=tcp +49:19.179 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23976 network=tcp +49:19.179 [INFO] TestAgent: started state syncer +49:19.179 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23988 network=tcp +49:19.242 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.242 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23980 [Candidate]" term=2 +49:19.242 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.242 [DEBUG] TestAgent.server.raft: vote granted: from=9dca7eb7-a67a-45a0-4b55-bdd8204ca7a1 term=2 tally=1 +49:19.242 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.242 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23980 [Leader]" +49:19.242 [INFO] TestAgent.server: cluster leadership acquired +49:19.242 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23980 +49:19.243 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.244 [INFO] TestAgent.server: New leader elected: payload=Node-9dca7eb7-a67a-45a0-4b55-bdd8204ca7a1 +49:19.244 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.244 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.244 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.244 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.245 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.245 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9dca7eb7-a67a-45a0-4b55-bdd8204ca7a1 +49:19.245 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9dca7eb7-a67a-45a0-4b55-bdd8204ca7a1 +49:19.245 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.307 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:19.308 [INFO] TestAgent: Synced node info +49:19.308 [DEBUG] TestAgent: Node info in sync +49:19.553 [WARN] TestAgent.server.rpc: Attempting to apply large raft entry: size_in_bytes=1573029 +49:19.586 [INFO] TestAgent: Requesting shutdown +49:19.586 [INFO] TestAgent.server: shutting down server +49:19.586 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:19.586 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:19.586 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:19.586 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:19.586 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.586 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.586 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:19.586 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:19.586 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:19.586 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:19.586 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:19.586Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:19.586Z [INFO] agent.router.manager: shutting down +49:19.587 [INFO] TestAgent: consul server down +49:19.587 [INFO] TestAgent: shutdown complete +49:19.587 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23975 network=tcp +49:19.587 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23975 network=udp +49:19.587 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23976 network=tcp +49:20.087 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.087 [INFO] TestAgent: Endpoints down +--- PASS: TestTxnEndpoint_Bad_Size_Item (3.33s) + --- PASS: TestTxnEndpoint_Bad_Size_Item/exceeds_default_limits (0.71s) + --- PASS: TestTxnEndpoint_Bad_Size_Item/exceeds_configured_max_txn_len (0.72s) + --- PASS: TestTxnEndpoint_Bad_Size_Item/exceeds_default_max_kv_value_size (0.96s) + --- PASS: TestTxnEndpoint_Bad_Size_Item/allowed (0.93s) === CONT TestHTTP_wrap_obfuscateLog -[WARN] freeport: 10 out of 59 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:04.824Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:04.937Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:05.018 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d89c713e-62a2-298f-801c-58d59cc47864 Address:127.0.0.1:16183}]" -13:05.019 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d89c713e-62a2-298f-801c-58d59cc47864.dc1 127.0.0.1 -13:05.019 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d89c713e-62a2-298f-801c-58d59cc47864 127.0.0.1 -2021-10-31T23:13:05.019Z [INFO] agent.router: Initializing LAN area manager -13:05.019 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16164 network=udp -13:05.019 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16183 [Follower]" leader= -13:05.020 [INFO] TestAgent.server: Adding LAN server: server="Node-d89c713e-62a2-298f-801c-58d59cc47864 (Addr: tcp/127.0.0.1:16183) (DC: dc1)" -13:05.020 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d89c713e-62a2-298f-801c-58d59cc47864.dc1 area=wan -13:05.020 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16164 network=tcp -13:05.021 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16165 network=tcp -13:05.021 [INFO] TestAgent: started state syncer -13:05.021 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16184 network=tcp -13:05.073 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.073 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16183 [Candidate]" term=2 -13:05.074 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.074 [DEBUG] TestAgent.server.raft: vote granted: from=d89c713e-62a2-298f-801c-58d59cc47864 term=2 tally=1 -13:05.074 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.074 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16183 [Leader]" -13:05.074 [INFO] TestAgent.server: cluster leadership acquired -13:05.074 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16183 -13:05.074 [INFO] TestAgent.server: New leader elected: payload=Node-d89c713e-62a2-298f-801c-58d59cc47864 -13:05.075 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.076 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.076 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.076 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.076 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.076 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.076 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d89c713e-62a2-298f-801c-58d59cc47864 -13:05.076 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d89c713e-62a2-298f-801c-58d59cc47864 -13:05.077 [INFO] TestAgent.server: federation state anti-entropy synced -13:05.195 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:05.195 [INFO] TestAgent: Synced node info -13:05.195 [DEBUG] TestAgent: Node info in sync -13:05.381 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:05.382 [INFO] TestAgent: Requesting shutdown -13:05.382 [INFO] TestAgent.server: shutting down server -13:05.382 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:05.382 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:05.382 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:05.382 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:05.382 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.382 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:05.382 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:05.382 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:05.382 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:05.382 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:05.382 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:05.382Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:05.382Z [INFO] agent.router.manager: shutting down -13:05.383 [INFO] TestAgent: consul server down -13:05.383 [INFO] TestAgent: shutdown complete -13:05.383 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16164 network=tcp -13:05.383 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16164 network=udp -13:05.383 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16165 network=tcp -13:05.885 [INFO] TestAgent: Waiting for endpoints to shut down -13:05.885 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:04.935Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:05.033Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:05.191 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:76a041e0-a052-b23b-e3f1-7592647c6cb4 Address:127.0.0.1:16197}]" -13:05.191 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-76a041e0-a052-b23b-e3f1-7592647c6cb4.dc1 127.0.0.1 -13:05.192 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-76a041e0-a052-b23b-e3f1-7592647c6cb4 127.0.0.1 -2021-10-31T23:13:05.192Z [INFO] agent.router: Initializing LAN area manager -13:05.194 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16197 [Follower]" leader= -13:05.194 [INFO] TestAgent.server: Adding LAN server: server="Node-76a041e0-a052-b23b-e3f1-7592647c6cb4 (Addr: tcp/127.0.0.1:16197) (DC: dc1)" -13:05.194 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-76a041e0-a052-b23b-e3f1-7592647c6cb4.dc1 area=wan -13:05.196 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16192 network=udp -13:05.196 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16192 network=tcp -13:05.197 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16193 network=tcp -13:05.197 [INFO] TestAgent: started state syncer -13:05.197 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16198 network=tcp -13:05.234 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.234 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16197 [Candidate]" term=2 -13:05.234 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.234 [DEBUG] TestAgent.server.raft: vote granted: from=76a041e0-a052-b23b-e3f1-7592647c6cb4 term=2 tally=1 -13:05.234 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.234 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16197 [Leader]" -13:05.234 [INFO] TestAgent.server: cluster leadership acquired -13:05.235 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16197 -13:05.235 [INFO] TestAgent.server: New leader elected: payload=Node-76a041e0-a052-b23b-e3f1-7592647c6cb4 -13:05.254 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.280 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.280 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.280 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.280 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.280 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.280 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-76a041e0-a052-b23b-e3f1-7592647c6cb4 -13:05.280 [INFO] TestAgent.server: member joined, marking health alive: member=Node-76a041e0-a052-b23b-e3f1-7592647c6cb4 -13:05.281 [INFO] TestAgent.server: federation state anti-entropy synced -13:05.382 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:05.382 [INFO] TestAgent: Requesting shutdown -13:05.382 [INFO] TestAgent.server: shutting down server -13:05.382 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:05.382 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:05.382 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:05.382 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:05.382 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.382 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:05.382 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:05.382 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:05.382 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:05.382 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:05.382 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:05.382 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:05.383Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:05.383Z [INFO] agent.router.manager: shutting down -13:05.383 [INFO] TestAgent: consul server down -13:05.383 [INFO] TestAgent: shutdown complete -13:05.383 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16192 network=tcp -13:05.383 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16192 network=udp -13:05.383 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16193 network=tcp -13:05.885 [INFO] TestAgent: Waiting for endpoints to shut down -13:05.885 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_ExecuteCached (1.19s) +2022/12/04 05:49:20 http: TLS handshake error from 127.0.0.1:42826: read tcp 127.0.0.1:24054->127.0.0.1:42826: i/o timeout +2022-12-04T05:49:19.396Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:19.396Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.409Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:19.409Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.411 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:779401fa-ced7-2e55-f2f8-1bcbe4eef4e2 Address:127.0.0.1:24001}]" +49:19.411 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-779401fa-ced7-2e55-f2f8-1bcbe4eef4e2.dc1 127.0.0.1 +49:19.411 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-779401fa-ced7-2e55-f2f8-1bcbe4eef4e2 127.0.0.1 +2022-12-04T05:49:19.411Z [INFO] agent.router: Initializing LAN area manager +49:19.412 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23996 network=udp +49:19.412 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24001 [Follower]" leader= +49:19.412 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-779401fa-ced7-2e55-f2f8-1bcbe4eef4e2.dc1 area=wan +49:19.412 [INFO] TestAgent.server: Adding LAN server: server="Node-779401fa-ced7-2e55-f2f8-1bcbe4eef4e2 (Addr: tcp/127.0.0.1:24001) (DC: dc1)" +49:19.412 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23996 network=tcp +49:19.413 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23997 network=tcp +49:19.413 [INFO] TestAgent: started state syncer +49:19.413 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24016 network=tcp +49:19.460 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.460 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24001 [Candidate]" term=2 +49:19.460 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.460 [DEBUG] TestAgent.server.raft: vote granted: from=779401fa-ced7-2e55-f2f8-1bcbe4eef4e2 term=2 tally=1 +49:19.460 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.460 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24001 [Leader]" +49:19.460 [INFO] TestAgent.server: cluster leadership acquired +49:19.460 [INFO] TestAgent.server: initializing acls +49:19.461 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:19.461 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:19.461 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:19.461 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:19.461 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:19.461 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:19.462 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-779401fa-ced7-2e55-f2f8-1bcbe4eef4e2 +49:19.462 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-779401fa-ced7-2e55-f2f8-1bcbe4eef4e2.dc1 +49:19.462 [INFO] TestAgent.server: New leader elected: payload=Node-779401fa-ced7-2e55-f2f8-1bcbe4eef4e2 +49:19.462 [INFO] TestAgent.server: Updating LAN server: server="Node-779401fa-ced7-2e55-f2f8-1bcbe4eef4e2 (Addr: tcp/127.0.0.1:24001) (DC: dc1)" +49:19.462 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-779401fa-ced7-2e55-f2f8-1bcbe4eef4e2.dc1 area=wan +49:19.480 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.488 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.488 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.488 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.488 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.488 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.495 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-779401fa-ced7-2e55-f2f8-1bcbe4eef4e2 +49:19.495 [INFO] TestAgent.server: member joined, marking health alive: member=Node-779401fa-ced7-2e55-f2f8-1bcbe4eef4e2 +49:19.498 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.499 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-779401fa-ced7-2e55-f2f8-1bcbe4eef4e2 +49:19.606 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-779401fa-ced7-2e55-f2f8-1bcbe4eef4e2 +49:19.606 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-779401fa-ced7-2e55-f2f8-1bcbe4eef4e2 +49:19.636 [INFO] TestAgent: Requesting shutdown +49:19.636 [INFO] TestAgent.server: shutting down server +49:19.636 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:19.636 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:19.636 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:19.636 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:19.636 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:19.636 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:19.636 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.636 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.636 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:19.636 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:19.636 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:19.636 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:19.636 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:19.636 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:19.636 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:19.636 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +2022-12-04T05:49:19.637Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:19.637Z [INFO] agent.router.manager: shutting down +49:19.637 [INFO] TestAgent: consul server down +49:19.637 [INFO] TestAgent: shutdown complete +49:19.637 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23996 network=tcp +49:19.637 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23996 network=udp +49:19.637 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23997 network=tcp +49:20.137 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.137 [INFO] TestAgent: Endpoints down +--- PASS: TestPProfHandlers_ACLs (0.77s) + --- PASS: TestPProfHandlers_ACLs/case_0_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:200,_token:"master",_endpoint:"/debug/pprof/heap",_nilResponse:false}) (0.01s) + --- PASS: TestPProfHandlers_ACLs/case_1_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"agent",_endpoint:"/debug/pprof/heap",_nilResponse:true}) (0.00s) + --- PASS: TestPProfHandlers_ACLs/case_2_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"agent",_endpoint:"/debug/pprof/",_nilResponse:true}) (0.00s) + --- PASS: TestPProfHandlers_ACLs/case_3_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"",_endpoint:"/debug/pprof/",_nilResponse:true}) (0.00s) + --- PASS: TestPProfHandlers_ACLs/case_4_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:200,_token:"master",_endpoint:"/debug/pprof/heap",_nilResponse:false}) (0.01s) + --- PASS: TestPProfHandlers_ACLs/case_5_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"towel",_endpoint:"/debug/pprof/heap",_nilResponse:true}) (0.00s) === CONT TestContentTypeIsJSON -=== RUN TestPreparedQuery_Get/#01 -2021-10-31T23:13:04.739Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:05.050Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:05.052 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:782ce70e-cb1e-9184-1e8d-1ee4a16f093e Address:127.0.0.1:16190}]" -13:05.052 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-782ce70e-cb1e-9184-1e8d-1ee4a16f093e.dc1 127.0.0.1 -13:05.053 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-782ce70e-cb1e-9184-1e8d-1ee4a16f093e 127.0.0.1 -2021-10-31T23:13:05.053Z [INFO] agent.router: Initializing LAN area manager -13:05.053 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16190 [Follower]" leader= -13:05.053 [INFO] TestAgent.server: Adding LAN server: server="Node-782ce70e-cb1e-9184-1e8d-1ee4a16f093e (Addr: tcp/127.0.0.1:16190) (DC: dc1)" -13:05.054 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-782ce70e-cb1e-9184-1e8d-1ee4a16f093e.dc1 area=wan -13:05.054 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16178 network=tcp -13:05.054 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16178 network=udp -13:05.054 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16179 network=tcp -13:05.054 [INFO] TestAgent: started state syncer -13:05.055 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16191 network=tcp -13:05.137 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.137 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16190 [Candidate]" term=2 -13:05.137 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.137 [DEBUG] TestAgent.server.raft: vote granted: from=782ce70e-cb1e-9184-1e8d-1ee4a16f093e term=2 tally=1 -13:05.137 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.137 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16190 [Leader]" -13:05.137 [INFO] TestAgent.server: cluster leadership acquired -13:05.137 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16190 -13:05.139 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.141 [INFO] TestAgent.server: New leader elected: payload=Node-782ce70e-cb1e-9184-1e8d-1ee4a16f093e -13:05.211 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.211 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.211 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.211 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.211 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.211 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-782ce70e-cb1e-9184-1e8d-1ee4a16f093e -13:05.211 [INFO] TestAgent.server: member joined, marking health alive: member=Node-782ce70e-cb1e-9184-1e8d-1ee4a16f093e -13:05.227 [INFO] TestAgent.server: federation state anti-entropy synced -13:05.362 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:05.363 [INFO] TestAgent: Synced node info -13:05.486 [INFO] TestAgent: Requesting shutdown -13:05.486 [INFO] TestAgent.server: shutting down server -13:05.486 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:05.486 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:05.486 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:05.486 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:05.486 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:05.486 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:05.486 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:05.486 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:05.486 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:05.486 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.486 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:05.486Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:05.486Z [INFO] agent.router.manager: shutting down -13:05.486 [INFO] TestAgent: consul server down -13:05.486 [INFO] TestAgent: shutdown complete -13:05.486 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16178 network=tcp -13:05.486 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16178 network=udp -13:05.486 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16179 network=tcp -13:05.989 [INFO] TestAgent: Waiting for endpoints to shut down -13:05.989 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsMatch_noName (1.29s) -=== CONT TestUIResponseHeaders -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:19.449Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.462Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.488 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:45287ae3-b0da-bd0d-9afd-ffa699bc7a5c Address:127.0.0.1:24022}]" +49:19.489 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24022 [Follower]" leader= +49:19.497 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-45287ae3-b0da-bd0d-9afd-ffa699bc7a5c.dc1 127.0.0.1 +49:19.498 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-45287ae3-b0da-bd0d-9afd-ffa699bc7a5c 127.0.0.1 +2022-12-04T05:49:19.498Z [INFO] agent.router: Initializing LAN area manager +49:19.498 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24017 network=udp +49:19.498 [INFO] TestAgent.server: Adding LAN server: server="Node-45287ae3-b0da-bd0d-9afd-ffa699bc7a5c (Addr: tcp/127.0.0.1:24022) (DC: dc1)" +49:19.498 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-45287ae3-b0da-bd0d-9afd-ffa699bc7a5c.dc1 area=wan +49:19.498 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24017 network=tcp +49:19.499 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24018 network=tcp +49:19.499 [INFO] TestAgent: started state syncer +49:19.499 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24009 network=tcp +49:19.565 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.565 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24022 [Candidate]" term=2 +49:19.565 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.565 [DEBUG] TestAgent.server.raft: vote granted: from=45287ae3-b0da-bd0d-9afd-ffa699bc7a5c term=2 tally=1 +49:19.565 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.565 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24022 [Leader]" +49:19.565 [INFO] TestAgent.server: cluster leadership acquired +49:19.565 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24022 +49:19.566 [INFO] TestAgent.server: New leader elected: payload=Node-45287ae3-b0da-bd0d-9afd-ffa699bc7a5c +49:19.567 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.573 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.573 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.573 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.573 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.573 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.573 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-45287ae3-b0da-bd0d-9afd-ffa699bc7a5c +49:19.573 [INFO] TestAgent.server: member joined, marking health alive: member=Node-45287ae3-b0da-bd0d-9afd-ffa699bc7a5c +49:19.574 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.647 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:19.647 [INFO] TestAgent: Requesting shutdown +49:19.647 [INFO] TestAgent.server: shutting down server +49:19.647 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:19.647 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:19.647 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:19.647 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:19.647 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.647 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.647 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:19.647 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:19.648 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:19.648 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:19.648 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:19.648 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:19.648Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:19.648Z [INFO] agent.router.manager: shutting down +49:19.648 [INFO] TestAgent: consul server down +49:19.648 [INFO] TestAgent: shutdown complete +49:19.648 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24017 network=tcp +49:19.648 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24017 network=udp +49:19.648 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24018 network=tcp +49:20.149 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.149 [INFO] TestAgent: Endpoints down +=== RUN TestPreparedQuery_Execute/#07 === RUN TestHTTP_wrap_obfuscateLog//some/url?token=secret1&token=secret2 === RUN TestHTTP_wrap_obfuscateLog//v1/acl/clone/secret1 === RUN TestHTTP_wrap_obfuscateLog//v1/acl/clone/secret1?token=secret2 @@ -22783,343 +28742,561 @@ === RUN TestHTTP_wrap_obfuscateLog//v1/acl/destroy/secret1?token=secret2 === RUN TestHTTP_wrap_obfuscateLog//v1/acl/info/secret1 === RUN TestHTTP_wrap_obfuscateLog//v1/acl/info/secret1?token=secret2 -2021/10/31 23:13:06 http: TLS handshake error from 127.0.0.1:33002: read tcp 127.0.0.1:16299->127.0.0.1:33002: i/o timeout -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:05.688Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:05.717Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:05.727 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:54e6848e-d65c-fd6a-e895-d00f36deda7d Address:127.0.0.1:16281}]" -13:05.728 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-54e6848e-d65c-fd6a-e895-d00f36deda7d.dc1 127.0.0.1 -13:05.728 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-54e6848e-d65c-fd6a-e895-d00f36deda7d 127.0.0.1 -2021-10-31T23:13:05.729Z [INFO] agent.router: Initializing LAN area manager -13:05.729 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16281 [Follower]" leader= -13:05.729 [INFO] TestAgent.server: Adding LAN server: server="Node-54e6848e-d65c-fd6a-e895-d00f36deda7d (Addr: tcp/127.0.0.1:16281) (DC: dc1)" -13:05.729 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-54e6848e-d65c-fd6a-e895-d00f36deda7d.dc1 area=wan -13:05.741 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16276 network=tcp -13:05.742 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16276 network=udp -13:05.742 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16277 network=tcp -13:05.742 [INFO] TestAgent: started state syncer -13:05.742 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16282 network=tcp -13:05.809 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.809 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16281 [Candidate]" term=2 -13:05.810 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.810 [DEBUG] TestAgent.server.raft: vote granted: from=54e6848e-d65c-fd6a-e895-d00f36deda7d term=2 tally=1 -13:05.810 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.810 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16281 [Leader]" -13:05.811 [INFO] TestAgent.server: cluster leadership acquired -13:05.811 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16281 -13:05.815 [INFO] TestAgent.server: New leader elected: payload=Node-54e6848e-d65c-fd6a-e895-d00f36deda7d -13:05.822 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.841 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.841 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.841 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.841 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.841 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.841 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-54e6848e-d65c-fd6a-e895-d00f36deda7d -13:05.841 [INFO] TestAgent.server: member joined, marking health alive: member=Node-54e6848e-d65c-fd6a-e895-d00f36deda7d -13:05.844 [INFO] TestAgent.server: federation state anti-entropy synced -13:05.886 [INFO] TestAgent: Requesting shutdown -13:05.886 [INFO] TestAgent.server: shutting down server -13:05.886 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:05.886 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:05.886 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:05.886 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:05.886 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.886 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:05.886 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:05.886 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:05.886 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:05.886 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:05.886 [WARN] TestAgent: Syncing node info failed.: error="leadership lost while committing log" -2021-10-31T23:13:05.886Z [INFO] agent.router.manager: shutting down -13:05.886 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="leadership lost while committing log" -2021-10-31T23:13:05.886Z [INFO] agent.router.manager: shutting down -13:05.886 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:05.886 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:05.901 [INFO] TestAgent: consul server down -13:05.901 [INFO] TestAgent: shutdown complete -13:05.901 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16276 network=tcp -13:05.901 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16276 network=udp -13:05.902 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16277 network=tcp -13:06.403 [INFO] TestAgent: Waiting for endpoints to shut down -13:06.403 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Explain/#02 -2021-10-31T23:13:05.621Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:05.682Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:05.684 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:647467e7-89ed-e5cc-307c-597bf52072c5 Address:127.0.0.1:16274}]" -13:05.684 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-647467e7-89ed-e5cc-307c-597bf52072c5.dc1 127.0.0.1 -13:05.685 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-647467e7-89ed-e5cc-307c-597bf52072c5 127.0.0.1 -2021-10-31T23:13:05.685Z [INFO] agent.router: Initializing LAN area manager -13:05.685 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16269 network=udp -13:05.685 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16274 [Follower]" leader= -13:05.685 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16269 network=tcp -13:05.685 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-647467e7-89ed-e5cc-307c-597bf52072c5.dc1 area=wan -13:05.685 [INFO] TestAgent.server: Adding LAN server: server="Node-647467e7-89ed-e5cc-307c-597bf52072c5 (Addr: tcp/127.0.0.1:16274) (DC: dc1)" -13:05.690 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16270 network=tcp -13:05.690 [INFO] TestAgent: started state syncer -13:05.690 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16275 network=tcp -13:05.753 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.754 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16274 [Candidate]" term=2 -13:05.754 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.754 [DEBUG] TestAgent.server.raft: vote granted: from=647467e7-89ed-e5cc-307c-597bf52072c5 term=2 tally=1 -13:05.754 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.754 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16274 [Leader]" -13:05.754 [INFO] TestAgent.server: cluster leadership acquired -13:05.754 [INFO] TestAgent.server: New leader elected: payload=Node-647467e7-89ed-e5cc-307c-597bf52072c5 -13:05.754 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16274 -13:05.755 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.756 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.756 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.756 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.756 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.756 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.757 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-647467e7-89ed-e5cc-307c-597bf52072c5 -13:05.757 [INFO] TestAgent.server: member joined, marking health alive: member=Node-647467e7-89ed-e5cc-307c-597bf52072c5 -13:05.757 [INFO] TestAgent.server: federation state anti-entropy synced -13:05.885 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:05.885 [INFO] TestAgent: Synced node info -13:05.885 [DEBUG] TestAgent: Node info in sync -13:05.891 [INFO] TestAgent: Requesting shutdown -13:05.891 [INFO] TestAgent.server: shutting down server -13:05.891 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:05.891 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:05.891 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:05.891 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:05.891 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.891 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:05.891 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:05.891 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:05.891 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:05.891 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:05.891 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:05.891Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:05.891Z [INFO] agent.router.manager: shutting down -13:05.910 [INFO] TestAgent: consul server down -13:05.910 [INFO] TestAgent: shutdown complete -13:05.910 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16269 network=tcp -13:05.910 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16269 network=udp -13:05.910 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16270 network=tcp -13:06.413 [INFO] TestAgent: Waiting for endpoints to shut down -13:06.414 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsList_values (0.83s) +2022-12-04T05:49:19.500Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.512Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.513 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5cfd5398-d998-6866-d547-ea2ee4e22ab7 Address:127.0.0.1:24050}]" +49:19.513 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24050 [Follower]" leader= +49:19.514 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5cfd5398-d998-6866-d547-ea2ee4e22ab7.dc1 127.0.0.1 +49:19.514 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5cfd5398-d998-6866-d547-ea2ee4e22ab7 127.0.0.1 +2022-12-04T05:49:19.514Z [INFO] agent.router: Initializing LAN area manager +49:19.514 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5cfd5398-d998-6866-d547-ea2ee4e22ab7.dc1 area=wan +49:19.514 [INFO] TestAgent.server: Adding LAN server: server="Node-5cfd5398-d998-6866-d547-ea2ee4e22ab7 (Addr: tcp/127.0.0.1:24050) (DC: dc1)" +49:19.514 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24045 network=tcp +49:19.514 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24045 network=udp +49:19.515 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24046 network=tcp +49:19.515 [INFO] TestAgent: started state syncer +49:19.515 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24037 network=tcp +49:19.560 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.560 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24050 [Candidate]" term=2 +49:19.560 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.560 [DEBUG] TestAgent.server.raft: vote granted: from=5cfd5398-d998-6866-d547-ea2ee4e22ab7 term=2 tally=1 +49:19.560 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.560 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24050 [Leader]" +49:19.560 [INFO] TestAgent.server: cluster leadership acquired +49:19.561 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24050 +49:19.562 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.563 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.563 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.563 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.563 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.563 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.563 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5cfd5398-d998-6866-d547-ea2ee4e22ab7 +49:19.564 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5cfd5398-d998-6866-d547-ea2ee4e22ab7 +49:19.564 [INFO] TestAgent.server: New leader elected: payload=Node-5cfd5398-d998-6866-d547-ea2ee4e22ab7 +49:19.565 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.726 [INFO] TestAgent: Requesting shutdown +49:19.726 [INFO] TestAgent.server: shutting down server +49:19.726 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:19.726 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:19.726 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:19.726 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:19.726 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.726 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.726 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:19.726 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:19.726 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:19.726 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:19.726 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:19.726Z [INFO] agent.router.manager: shutting down +49:19.726 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +2022-12-04T05:49:19.726Z [INFO] agent.router.manager: shutting down +49:19.726 [DEBUG] TestAgent.server: shutting down leader loop +49:19.726 [INFO] TestAgent.server: cluster leadership lost +49:19.726 [INFO] TestAgent: consul server down +49:19.726 [INFO] TestAgent: shutdown complete +49:19.726 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24045 network=tcp +49:19.726 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24045 network=udp +49:19.726 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24046 network=tcp +49:20.226 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.226 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsList_empty (0.77s) +=== CONT TestUIResponseHeaders +2022-12-04T05:49:19.464Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.507Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.509 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c38b38e0-31b5-a953-acae-0c5b79fd3a1d Address:127.0.0.1:24036}]" +49:19.509 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24036 [Follower]" leader= +49:19.509 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c38b38e0-31b5-a953-acae-0c5b79fd3a1d.dc1 127.0.0.1 +49:19.510 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c38b38e0-31b5-a953-acae-0c5b79fd3a1d 127.0.0.1 +2022-12-04T05:49:19.510Z [INFO] agent.router: Initializing LAN area manager +49:19.510 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c38b38e0-31b5-a953-acae-0c5b79fd3a1d.dc1 area=wan +49:19.510 [INFO] TestAgent.server: Adding LAN server: server="Node-c38b38e0-31b5-a953-acae-0c5b79fd3a1d (Addr: tcp/127.0.0.1:24036) (DC: dc1)" +49:19.510 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24031 network=udp +49:19.510 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24031 network=tcp +49:19.511 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24032 network=tcp +49:19.511 [INFO] TestAgent: started state syncer +49:19.511 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24044 network=tcp +49:19.567 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.567 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24036 [Candidate]" term=2 +49:19.567 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.567 [DEBUG] TestAgent.server.raft: vote granted: from=c38b38e0-31b5-a953-acae-0c5b79fd3a1d term=2 tally=1 +49:19.567 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.567 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24036 [Leader]" +49:19.567 [INFO] TestAgent.server: cluster leadership acquired +49:19.567 [INFO] TestAgent.server: New leader elected: payload=Node-c38b38e0-31b5-a953-acae-0c5b79fd3a1d +49:19.569 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24036 +49:19.570 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.571 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.571 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.571 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.571 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.571 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.571 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c38b38e0-31b5-a953-acae-0c5b79fd3a1d +49:19.571 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c38b38e0-31b5-a953-acae-0c5b79fd3a1d +49:19.572 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.743 [INFO] TestAgent: Requesting shutdown +49:19.743 [INFO] TestAgent.server: shutting down server +49:19.743 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:19.743 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:19.743 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:19.743 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:19.743 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.743 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.743 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:19.743 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:19.743 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:19.743 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:19.743 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:19.743 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:19.743Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:19.743Z [INFO] agent.router.manager: shutting down +49:19.743 [INFO] TestAgent: consul server down +49:19.743 [INFO] TestAgent: shutdown complete +49:19.743 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24031 network=tcp +49:19.743 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24031 network=udp +49:19.743 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24032 network=tcp +49:20.244 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.244 [INFO] TestAgent: Endpoints down +--- PASS: TestPProfHandlers_DisableDebugNoACLs (0.79s) === CONT TestHTTPAPIResponseHeaders -2021-10-31T23:13:05.592Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:05.668Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:05.674 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:76678b5b-62a7-520c-88ef-525dac5bf6c0 Address:127.0.0.1:16260}]" -13:05.677 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16260 [Follower]" leader= -13:05.690 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-76678b5b-62a7-520c-88ef-525dac5bf6c0.dc1 127.0.0.1 -13:05.691 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-76678b5b-62a7-520c-88ef-525dac5bf6c0 127.0.0.1 -2021-10-31T23:13:05.691Z [INFO] agent.router: Initializing LAN area manager -13:05.691 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16255 network=udp -13:05.691 [INFO] TestAgent.server: Adding LAN server: server="Node-76678b5b-62a7-520c-88ef-525dac5bf6c0 (Addr: tcp/127.0.0.1:16260) (DC: dc1)" -13:05.691 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-76678b5b-62a7-520c-88ef-525dac5bf6c0.dc1 area=wan -13:05.691 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16255 network=tcp -13:05.692 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16256 network=tcp -13:05.692 [INFO] TestAgent: started state syncer -13:05.692 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16261 network=tcp -13:05.743 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.743 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16260 [Candidate]" term=2 -13:05.743 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.743 [DEBUG] TestAgent.server.raft: vote granted: from=76678b5b-62a7-520c-88ef-525dac5bf6c0 term=2 tally=1 -13:05.743 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.743 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16260 [Leader]" -13:05.743 [INFO] TestAgent.server: cluster leadership acquired -13:05.743 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16260 -13:05.744 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.745 [INFO] TestAgent.server: New leader elected: payload=Node-76678b5b-62a7-520c-88ef-525dac5bf6c0 -13:05.758 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.758 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.758 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.758 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.758 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.758 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-76678b5b-62a7-520c-88ef-525dac5bf6c0 -13:05.758 [INFO] TestAgent.server: member joined, marking health alive: member=Node-76678b5b-62a7-520c-88ef-525dac5bf6c0 -13:05.759 [INFO] TestAgent.server: federation state anti-entropy synced -13:05.950 [INFO] TestAgent: Requesting shutdown -13:05.950 [INFO] TestAgent.server: shutting down server -13:05.950 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:05.950 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:05.950 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:05.950 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:05.950 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.950 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:05.950 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:05.950 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:05.950 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:05.950 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:05.950 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:05.950 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:05.950Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:05.950Z [INFO] agent.router.manager: shutting down -13:05.962 [INFO] TestAgent: consul server down -13:05.962 [INFO] TestAgent: shutdown complete -13:05.962 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16255 network=tcp -13:05.962 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16255 network=udp -13:05.962 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16256 network=tcp -13:06.465 [INFO] TestAgent: Waiting for endpoints to shut down -13:06.465 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsMatch_noBy (0.92s) +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:19.447Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:19.447Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.461Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:19.461Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.468 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:63977d62-dacd-69e3-c3a0-77c21b3e71f9 Address:127.0.0.1:24015}]" +49:19.484 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-63977d62-dacd-69e3-c3a0-77c21b3e71f9.dc1 127.0.0.1 +49:19.487 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24015 [Follower]" leader= +49:19.495 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-63977d62-dacd-69e3-c3a0-77c21b3e71f9 127.0.0.1 +2022-12-04T05:49:19.495Z [INFO] agent.router: Initializing LAN area manager +49:19.496 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24010 network=udp +49:19.496 [INFO] TestAgent.server: Adding LAN server: server="Node-63977d62-dacd-69e3-c3a0-77c21b3e71f9 (Addr: tcp/127.0.0.1:24015) (DC: dc1)" +49:19.496 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-63977d62-dacd-69e3-c3a0-77c21b3e71f9.dc1 area=wan +49:19.496 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24010 network=tcp +49:19.497 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24011 network=tcp +49:19.497 [INFO] TestAgent: started state syncer +49:19.497 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24023 network=tcp +49:19.546 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:19.547 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.547 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24015 [Candidate]" term=2 +49:19.548 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.548 [DEBUG] TestAgent.server.raft: vote granted: from=63977d62-dacd-69e3-c3a0-77c21b3e71f9 term=2 tally=1 +49:19.548 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.548 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24015 [Leader]" +49:19.548 [INFO] TestAgent.server: cluster leadership acquired +49:19.548 [INFO] TestAgent.server: initializing acls +49:19.548 [INFO] TestAgent.server: New leader elected: payload=Node-63977d62-dacd-69e3-c3a0-77c21b3e71f9 +49:19.548 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:19.548 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:19.549 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:19.549 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:19.549 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:19.549 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:19.549 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-63977d62-dacd-69e3-c3a0-77c21b3e71f9 +49:19.549 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-63977d62-dacd-69e3-c3a0-77c21b3e71f9.dc1 +49:19.549 [INFO] TestAgent.server: Updating LAN server: server="Node-63977d62-dacd-69e3-c3a0-77c21b3e71f9 (Addr: tcp/127.0.0.1:24015) (DC: dc1)" +49:19.549 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-63977d62-dacd-69e3-c3a0-77c21b3e71f9.dc1 area=wan +49:19.550 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.551 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.551 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.551 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.551 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.551 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.551 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-63977d62-dacd-69e3-c3a0-77c21b3e71f9 +49:19.551 [INFO] TestAgent.server: member joined, marking health alive: member=Node-63977d62-dacd-69e3-c3a0-77c21b3e71f9 +49:19.552 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.552 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-63977d62-dacd-69e3-c3a0-77c21b3e71f9 +49:19.802 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-63977d62-dacd-69e3-c3a0-77c21b3e71f9 +49:19.803 [INFO] TestAgent.server.fsm: snapshot created: duration=40.988µs +49:19.803 [INFO] TestAgent.server.raft: starting snapshot up to: index=14 +49:19.803 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot_Options_GET#02-TestAgent-agent620912710/raft/snapshots/2-14-1670132959803.tmp +49:19.803 [INFO] TestAgent.server.raft: snapshot complete up to: index=14 +49:19.804 [INFO] TestAgent: Requesting shutdown +49:19.805 [INFO] TestAgent.server: shutting down server +49:19.805 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:19.805 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:19.805 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:19.805 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:19.805 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:19.805 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:19.805 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.805 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.805 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:19.805 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:19.805 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:19.805 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:19.805 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:19.805 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:19.805 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:19.805 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:19.805Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:19.805Z [INFO] agent.router.manager: shutting down +49:19.805 [INFO] TestAgent: consul server down +49:19.805 [INFO] TestAgent: shutdown complete +49:19.805 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24010 network=tcp +49:19.805 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24010 network=udp +49:19.805 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24011 network=tcp +49:20.305 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.305 [INFO] TestAgent: Endpoints down +=== RUN TestSnapshot_Options/PUT +2022-12-04T05:49:19.452Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.496Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.498 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3f312a7a-648e-8e9d-fc4b-6d734dd889fb Address:127.0.0.1:24029}]" +49:19.499 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3f312a7a-648e-8e9d-fc4b-6d734dd889fb.dc1 127.0.0.1 +49:19.499 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3f312a7a-648e-8e9d-fc4b-6d734dd889fb 127.0.0.1 +2022-12-04T05:49:19.499Z [INFO] agent.router: Initializing LAN area manager +49:19.499 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24029 [Follower]" leader= +49:19.499 [INFO] TestAgent.server: Adding LAN server: server="Node-3f312a7a-648e-8e9d-fc4b-6d734dd889fb (Addr: tcp/127.0.0.1:24029) (DC: dc1)" +49:19.500 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3f312a7a-648e-8e9d-fc4b-6d734dd889fb.dc1 area=wan +49:19.500 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24024 network=tcp +49:19.500 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24024 network=udp +49:19.500 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24025 network=tcp +49:19.500 [INFO] TestAgent: started state syncer +49:19.500 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24030 network=tcp +49:19.541 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.541 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24029 [Candidate]" term=2 +49:19.541 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.541 [DEBUG] TestAgent.server.raft: vote granted: from=3f312a7a-648e-8e9d-fc4b-6d734dd889fb term=2 tally=1 +49:19.541 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.541 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24029 [Leader]" +49:19.541 [INFO] TestAgent.server: cluster leadership acquired +49:19.541 [INFO] TestAgent.server: New leader elected: payload=Node-3f312a7a-648e-8e9d-fc4b-6d734dd889fb +49:19.542 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24029 +49:19.542 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.544 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.544 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.544 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.544 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.544 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.544 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3f312a7a-648e-8e9d-fc4b-6d734dd889fb +49:19.544 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3f312a7a-648e-8e9d-fc4b-6d734dd889fb +49:19.545 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.551 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:19.552 [INFO] TestAgent: Synced node info +49:19.813 [INFO] TestAgent: Requesting shutdown +49:19.813 [INFO] TestAgent.server: shutting down server +49:19.813 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:19.813 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:19.813 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:19.813 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:19.813 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.813 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.813 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:19.813 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:19.813 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:19.813 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:19.813 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:19.813Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:19.813Z [INFO] agent.router.manager: shutting down +49:19.813 [INFO] TestAgent: consul server down +49:19.813 [INFO] TestAgent: shutdown complete +49:19.813 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24024 network=tcp +49:19.813 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24024 network=udp +49:19.813 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24025 network=tcp +49:20.314 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.314 [INFO] TestAgent: Endpoints down +--- PASS: TestIntentionsList_values (0.87s) +=== CONT TestHealthConnectServiceNodes_PassingFilter +2022-12-04T05:49:19.053Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.067Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.068 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:739b39c5-b9b1-5d15-544b-995a2c90767c Address:127.0.0.1:23959}]" +49:19.068 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23959 [Follower]" leader= +49:19.069 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-739b39c5-b9b1-5d15-544b-995a2c90767c.dc1 127.0.0.1 +49:19.069 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-739b39c5-b9b1-5d15-544b-995a2c90767c 127.0.0.1 +2022-12-04T05:49:19.069Z [INFO] agent.router: Initializing LAN area manager +49:19.069 [INFO] TestAgent.server: Adding LAN server: server="Node-739b39c5-b9b1-5d15-544b-995a2c90767c (Addr: tcp/127.0.0.1:23959) (DC: dc1)" +49:19.069 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-739b39c5-b9b1-5d15-544b-995a2c90767c.dc1 area=wan +49:19.069 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23954 network=udp +49:19.069 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23954 network=tcp +49:19.070 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23955 network=tcp +49:19.070 [INFO] TestAgent: started state syncer +49:19.070 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23960 network=tcp +49:19.116 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.116 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23959 [Candidate]" term=2 +49:19.116 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.116 [DEBUG] TestAgent.server.raft: vote granted: from=739b39c5-b9b1-5d15-544b-995a2c90767c term=2 tally=1 +49:19.116 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.116 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23959 [Leader]" +49:19.116 [INFO] TestAgent.server: cluster leadership acquired +49:19.116 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23959 +49:19.116 [INFO] TestAgent.server: New leader elected: payload=Node-739b39c5-b9b1-5d15-544b-995a2c90767c +49:19.117 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.118 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.118 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.118 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.118 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.118 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.118 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-739b39c5-b9b1-5d15-544b-995a2c90767c +49:19.118 [INFO] TestAgent.server: member joined, marking health alive: member=Node-739b39c5-b9b1-5d15-544b-995a2c90767c +49:19.119 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.275 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:19.276 [INFO] TestAgent: Synced node info +49:19.276 [DEBUG] TestAgent: Node info in sync +49:19.824 [INFO] TestAgent: Requesting shutdown +49:19.824 [INFO] TestAgent.server: shutting down server +49:19.824 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:19.824 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:19.824 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:19.824 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:19.824 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.824 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.824 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:19.824 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:19.824 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:19.824 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:19.824 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:19.825Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:19.825Z [INFO] agent.router.manager: shutting down +49:19.825 [INFO] TestAgent: consul server down +49:19.825 [INFO] TestAgent: shutdown complete +49:19.825 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23954 network=tcp +49:19.825 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23954 network=udp +49:19.825 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23955 network=tcp +49:20.325 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.325 [INFO] TestAgent: Endpoints down +=== RUN TestTxnEndpoint_Bad_Size_Net/allowed +2022-12-04T05:49:20.315Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:20.315Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.326Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:20.326Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.327 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ebba2cf9-e0e8-2b52-c675-43cc8b5bf770 Address:127.0.0.1:24145}]" +49:20.327 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24145 [Follower]" leader= +49:20.328 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ebba2cf9-e0e8-2b52-c675-43cc8b5bf770.dc1 127.0.0.1 +49:20.328 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ebba2cf9-e0e8-2b52-c675-43cc8b5bf770 127.0.0.1 +2022-12-04T05:49:20.328Z [INFO] agent.router: Initializing LAN area manager +49:20.328 [INFO] TestAgent.server: Adding LAN server: server="Node-ebba2cf9-e0e8-2b52-c675-43cc8b5bf770 (Addr: tcp/127.0.0.1:24145) (DC: dc1)" +49:20.329 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ebba2cf9-e0e8-2b52-c675-43cc8b5bf770.dc1 area=wan +49:20.329 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24140 network=udp +49:20.329 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24140 network=tcp +49:20.329 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24141 network=tcp +49:20.329 [INFO] TestAgent: started state syncer +49:20.329 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24146 network=tcp +49:20.379 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:20.396 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.396 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24145 [Candidate]" term=2 +49:20.396 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.396 [DEBUG] TestAgent.server.raft: vote granted: from=ebba2cf9-e0e8-2b52-c675-43cc8b5bf770 term=2 tally=1 +49:20.396 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.396 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24145 [Leader]" +49:20.396 [INFO] TestAgent.server: cluster leadership acquired +49:20.396 [INFO] TestAgent.server: New leader elected: payload=Node-ebba2cf9-e0e8-2b52-c675-43cc8b5bf770 +49:20.396 [INFO] TestAgent.server: initializing acls +49:20.397 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:20.397 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:20.397 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:20.397 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:20.397 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:20.397 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:20.397 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ebba2cf9-e0e8-2b52-c675-43cc8b5bf770 +49:20.397 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ebba2cf9-e0e8-2b52-c675-43cc8b5bf770.dc1 +49:20.398 [INFO] TestAgent.server: Updating LAN server: server="Node-ebba2cf9-e0e8-2b52-c675-43cc8b5bf770 (Addr: tcp/127.0.0.1:24145) (DC: dc1)" +49:20.398 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ebba2cf9-e0e8-2b52-c675-43cc8b5bf770.dc1 area=wan +49:20.398 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.399 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.399 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.399 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.399 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.399 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.399 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ebba2cf9-e0e8-2b52-c675-43cc8b5bf770 +49:20.399 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ebba2cf9-e0e8-2b52-c675-43cc8b5bf770 +49:20.399 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.400 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ebba2cf9-e0e8-2b52-c675-43cc8b5bf770 +49:20.423 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ebba2cf9-e0e8-2b52-c675-43cc8b5bf770 +49:20.423 [INFO] TestAgent: Requesting shutdown +49:20.423 [INFO] TestAgent.server: shutting down server +49:20.423 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:20.423 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:20.423 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.423 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.423 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.423 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.423 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.423 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.423 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:20.423 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:20.423 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.423 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.423 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.423 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.423 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:20.423 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.424Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.424Z [INFO] agent.router.manager: shutting down +49:20.424 [INFO] TestAgent: consul server down +49:20.424 [INFO] TestAgent: shutdown complete +49:20.424 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24140 network=tcp +49:20.424 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24140 network=udp +49:20.424 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24141 network=tcp +49:20.424 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.424 [INFO] TestAgent: Endpoints down +=== RUN TestSnapshot_Options/PUT#01 +2022-12-04T05:49:19.827Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.839Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.841 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ac07e551-9843-51da-a0ea-45f16d633e32 Address:127.0.0.1:24078}]" +49:19.841 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24078 [Follower]" leader= +49:19.842 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ac07e551-9843-51da-a0ea-45f16d633e32.dc1 127.0.0.1 +49:19.842 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ac07e551-9843-51da-a0ea-45f16d633e32 127.0.0.1 +2022-12-04T05:49:19.842Z [INFO] agent.router: Initializing LAN area manager +49:19.842 [INFO] TestAgent.server: Adding LAN server: server="Node-ac07e551-9843-51da-a0ea-45f16d633e32 (Addr: tcp/127.0.0.1:24078) (DC: dc1)" +49:19.843 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24073 network=udp +49:19.843 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ac07e551-9843-51da-a0ea-45f16d633e32.dc1 area=wan +49:19.843 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24073 network=tcp +49:19.843 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24074 network=tcp +49:19.843 [INFO] TestAgent: started state syncer +49:19.843 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24065 network=tcp +49:19.901 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.901 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24078 [Candidate]" term=2 +49:19.901 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.901 [DEBUG] TestAgent.server.raft: vote granted: from=ac07e551-9843-51da-a0ea-45f16d633e32 term=2 tally=1 +49:19.901 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.901 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24078 [Leader]" +49:19.902 [INFO] TestAgent.server: cluster leadership acquired +49:19.902 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24078 +49:19.902 [INFO] TestAgent.server: New leader elected: payload=Node-ac07e551-9843-51da-a0ea-45f16d633e32 +49:19.902 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.903 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.903 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.903 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.903 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.903 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.903 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ac07e551-9843-51da-a0ea-45f16d633e32 +49:19.903 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ac07e551-9843-51da-a0ea-45f16d633e32 +49:19.904 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.932 [INFO] TestAgent: Requesting shutdown +49:19.932 [INFO] TestAgent.server: shutting down server +49:19.932 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:19.932 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:19.932 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:19.932 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:19.932 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.932 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.932 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:19.933 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:19.933 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:19.933 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:19.933 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:19.933 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:49:19.933Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:19.933Z [INFO] agent.router.manager: shutting down +49:19.933 [INFO] TestAgent: consul server down +49:19.933 [INFO] TestAgent: shutdown complete +49:19.933 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24073 network=tcp +49:19.933 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24073 network=udp +49:19.933 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24074 network=tcp +49:20.434 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.434 [INFO] TestAgent: Endpoints down +--- PASS: TestParseSource (0.62s) === CONT TestHTTPAPI_TranslateAddrHeader -2021-10-31T23:13:05.761Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:05.877Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:05.883 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f521a918-d068-d583-84a9-4f3af45258c2 Address:127.0.0.1:16295}]" -13:05.884 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f521a918-d068-d583-84a9-4f3af45258c2.dc1 127.0.0.1 -13:05.884 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16295 [Follower]" leader= -13:05.895 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f521a918-d068-d583-84a9-4f3af45258c2 127.0.0.1 -2021-10-31T23:13:05.895Z [INFO] agent.router: Initializing LAN area manager -13:05.897 [INFO] TestAgent.server: Adding LAN server: server="Node-f521a918-d068-d583-84a9-4f3af45258c2 (Addr: tcp/127.0.0.1:16295) (DC: dc1)" -13:05.898 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f521a918-d068-d583-84a9-4f3af45258c2.dc1 area=wan -13:05.898 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16290 network=udp -13:05.898 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16290 network=tcp -13:05.929 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.929 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16295 [Candidate]" term=2 -13:05.930 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.930 [DEBUG] TestAgent.server.raft: vote granted: from=f521a918-d068-d583-84a9-4f3af45258c2 term=2 tally=1 -13:05.930 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.930 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16295 [Leader]" -13:05.930 [INFO] TestAgent.server: cluster leadership acquired -13:05.930 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16295 -13:05.933 [INFO] TestAgent.server: New leader elected: payload=Node-f521a918-d068-d583-84a9-4f3af45258c2 -13:05.966 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.966 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16291 network=tcp -13:05.966 [INFO] TestAgent: started state syncer -13:05.967 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16296 network=tcp -13:05.976 [INFO] TestAgent: Requesting shutdown -13:05.976 [INFO] TestAgent.server: shutting down server -13:05.977 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:05.977 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:05.977Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:05.977Z [INFO] agent.router.manager: shutting down -13:05.982 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:05.998 [ERROR] TestAgent.server: failed to establish leadership: error="error generating CA root certificate: error computing next serial number: leadership lost while committing log" -13:06.006 [INFO] TestAgent: consul server down -13:06.006 [INFO] TestAgent: shutdown complete -13:06.006 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16290 network=tcp -13:06.006 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16290 network=udp -13:06.006 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16291 network=tcp -13:06.509 [INFO] TestAgent: Waiting for endpoints to shut down -13:06.509 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:19.649Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.659Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.660 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1c10b34c-1a8b-d7d8-7296-0709116216ff Address:127.0.0.1:24064}]" +49:19.660 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24064 [Follower]" leader= +49:19.660 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1c10b34c-1a8b-d7d8-7296-0709116216ff.dc1 127.0.0.1 +49:19.661 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1c10b34c-1a8b-d7d8-7296-0709116216ff 127.0.0.1 +2022-12-04T05:49:19.661Z [INFO] agent.router: Initializing LAN area manager +49:19.661 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24059 network=udp +49:19.661 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1c10b34c-1a8b-d7d8-7296-0709116216ff.dc1 area=wan +49:19.661 [INFO] TestAgent.server: Adding LAN server: server="Node-1c10b34c-1a8b-d7d8-7296-0709116216ff (Addr: tcp/127.0.0.1:24064) (DC: dc1)" +49:19.661 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24059 network=tcp +49:19.662 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24060 network=tcp +49:19.662 [INFO] TestAgent: started state syncer +49:19.662 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24051 network=tcp +49:19.708 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.708 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24064 [Candidate]" term=2 +49:19.708 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.708 [DEBUG] TestAgent.server.raft: vote granted: from=1c10b34c-1a8b-d7d8-7296-0709116216ff term=2 tally=1 +49:19.708 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.708 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24064 [Leader]" +49:19.708 [INFO] TestAgent.server: cluster leadership acquired +49:19.709 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24064 +49:19.709 [INFO] TestAgent.server: New leader elected: payload=Node-1c10b34c-1a8b-d7d8-7296-0709116216ff +49:19.710 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.711 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.711 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.711 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.711 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.711 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.711 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1c10b34c-1a8b-d7d8-7296-0709116216ff +49:19.711 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1c10b34c-1a8b-d7d8-7296-0709116216ff +49:19.712 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.857 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:19.858 [INFO] TestAgent: Synced node info +49:19.858 [DEBUG] TestAgent: Node info in sync +49:19.954 [INFO] TestAgent: Requesting shutdown +49:19.954 [INFO] TestAgent.server: shutting down server +49:19.954 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:19.954 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:19.954 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:19.954Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:19.954Z [INFO] agent.router.manager: shutting down +49:19.954 [INFO] TestAgent: consul server down +49:19.954 [INFO] TestAgent: shutdown complete +49:19.954 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24059 network=tcp +49:19.954 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24059 network=udp +49:19.954 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24060 network=tcp +49:20.454 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.454 [INFO] TestAgent: Endpoints down === RUN TestRPC_HTTPSMaxConnsPerClient/HTTPS -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:05.663Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:05.702Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:05.704 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:260a428b-fa2e-8313-6935-8275a9d9772a Address:127.0.0.1:16267}]" -13:05.705 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-260a428b-fa2e-8313-6935-8275a9d9772a.dc1 127.0.0.1 -13:05.705 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16267 [Follower]" leader= -13:05.705 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-260a428b-fa2e-8313-6935-8275a9d9772a 127.0.0.1 -2021-10-31T23:13:05.705Z [INFO] agent.router: Initializing LAN area manager -13:05.710 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16262 network=udp -13:05.710 [INFO] TestAgent.server: Adding LAN server: server="Node-260a428b-fa2e-8313-6935-8275a9d9772a (Addr: tcp/127.0.0.1:16267) (DC: dc1)" -13:05.710 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-260a428b-fa2e-8313-6935-8275a9d9772a.dc1 area=wan -13:05.711 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16262 network=tcp -13:05.711 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16263 network=tcp -13:05.711 [INFO] TestAgent: started state syncer -13:05.711 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16268 network=tcp -13:05.750 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.750 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16267 [Candidate]" term=2 -13:05.751 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.751 [DEBUG] TestAgent.server.raft: vote granted: from=260a428b-fa2e-8313-6935-8275a9d9772a term=2 tally=1 -13:05.751 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.751 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16267 [Leader]" -13:05.751 [INFO] TestAgent.server: cluster leadership acquired -13:05.751 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16267 -13:05.752 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.753 [INFO] TestAgent.server: New leader elected: payload=Node-260a428b-fa2e-8313-6935-8275a9d9772a -13:05.753 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.753 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.753 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.753 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.753 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.753 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-260a428b-fa2e-8313-6935-8275a9d9772a -13:05.753 [INFO] TestAgent.server: member joined, marking health alive: member=Node-260a428b-fa2e-8313-6935-8275a9d9772a -13:05.754 [INFO] TestAgent.server: federation state anti-entropy synced -13:06.031 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:06.032 [INFO] TestAgent: Synced node info -13:06.032 [DEBUG] TestAgent: Node info in sync -13:06.112 [INFO] TestAgent: Requesting shutdown -13:06.112 [INFO] TestAgent.server: shutting down server -13:06.112 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:06.112 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:06.112 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:06.112 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:06.112 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:06.113 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:06.113 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:06.113 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:06.113 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:06.113 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:06.113 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:06.113Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:06.117Z [INFO] agent.router.manager: shutting down -13:06.121 [INFO] TestAgent: consul server down -13:06.121 [INFO] TestAgent: shutdown complete -13:06.121 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16262 network=tcp -13:06.122 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16262 network=udp -13:06.122 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16263 network=tcp -13:06.625 [INFO] TestAgent: Waiting for endpoints to shut down -13:06.625 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsMatch_basic (1.04s) -=== CONT TestHTTPAPI_BlockEndpoints -2021-10-31T23:13:04.809Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:05.033Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:05.035 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9c6e33a6-a993-ffea-e5c9-25a2e6b1140f Address:127.0.0.1:16169}]" -13:05.036 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9c6e33a6-a993-ffea-e5c9-25a2e6b1140f.dc1 127.0.0.1 -13:05.038 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16169 [Follower]" leader= -13:05.086 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.086 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16169 [Candidate]" term=2 -13:05.086 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=9c6e33a6-a993-ffea-e5c9-25a2e6b1140f fallback=127.0.0.1:16169 error="Could not find address for server id 9c6e33a6-a993-ffea-e5c9-25a2e6b1140f" -13:05.086 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.086 [DEBUG] TestAgent.server.raft: vote granted: from=9c6e33a6-a993-ffea-e5c9-25a2e6b1140f term=2 tally=1 -13:05.086 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.086 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16169 [Leader]" -13:05.123 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9c6e33a6-a993-ffea-e5c9-25a2e6b1140f 127.0.0.1 -2021-10-31T23:13:05.123Z [INFO] agent.router: Initializing LAN area manager -13:05.123 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16157 network=udp -13:05.132 [INFO] TestAgent.server: Adding LAN server: server="Node-9c6e33a6-a993-ffea-e5c9-25a2e6b1140f (Addr: tcp/127.0.0.1:16169) (DC: dc1)" -13:05.132 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9c6e33a6-a993-ffea-e5c9-25a2e6b1140f.dc1 area=wan -13:05.132 [INFO] TestAgent.server: cluster leadership acquired -13:05.132 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16169 -13:05.134 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.136 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.136 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.136 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.136 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.136 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.136 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9c6e33a6-a993-ffea-e5c9-25a2e6b1140f -13:05.136 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9c6e33a6-a993-ffea-e5c9-25a2e6b1140f -13:05.137 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16157 network=tcp -13:05.137 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9c6e33a6-a993-ffea-e5c9-25a2e6b1140f -13:05.141 [INFO] TestAgent.server: New leader elected: payload=Node-9c6e33a6-a993-ffea-e5c9-25a2e6b1140f -13:05.142 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16158 network=tcp -13:05.154 [INFO] TestAgent.server: federation state anti-entropy synced -13:05.186 [INFO] TestAgent: started state syncer -13:05.189 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16170 network=tcp -13:05.190 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:05.190 [INFO] TestAgent: Synced node info -13:05.190 [DEBUG] TestAgent: Node info in sync -13:06.129 [INFO] TestAgent: Requesting shutdown -13:06.129 [INFO] TestAgent.server: shutting down server -13:06.129 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:06.129 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:06.129 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:06.129 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:06.129 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:06.129 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:06.129 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:06.129 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:06.129 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:06.129 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:06.130 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:06.130Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:06.130Z [INFO] agent.router.manager: shutting down -13:06.140 [INFO] TestAgent: consul server down -13:06.140 [INFO] TestAgent: shutdown complete -13:06.140 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16157 network=tcp -13:06.140 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16157 network=udp -13:06.140 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16158 network=tcp -13:06.641 [INFO] TestAgent: Waiting for endpoints to shut down -13:06.641 [INFO] TestAgent: Endpoints down -=== RUN TestTxnEndpoint_Bad_Size_Net/exceeds_configured_max_txn_len ---- PASS: TestHTTP_wrap_obfuscateLog (0.89s) +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:19.829Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.840Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.842 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6997522e-4f68-2604-3ba4-c9a057505aa0 Address:127.0.0.1:24057}]" +49:19.842 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24057 [Follower]" leader= +49:19.842 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6997522e-4f68-2604-3ba4-c9a057505aa0.dc1 127.0.0.1 +49:19.843 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6997522e-4f68-2604-3ba4-c9a057505aa0 127.0.0.1 +2022-12-04T05:49:19.843Z [INFO] agent.router: Initializing LAN area manager +49:19.843 [INFO] TestAgent.server: Adding LAN server: server="Node-6997522e-4f68-2604-3ba4-c9a057505aa0 (Addr: tcp/127.0.0.1:24057) (DC: dc1)" +49:19.843 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6997522e-4f68-2604-3ba4-c9a057505aa0.dc1 area=wan +49:19.843 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24052 network=udp +49:19.843 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24052 network=tcp +49:19.843 [INFO] TestAgent: Started HTTPS server: address=127.0.0.1:24054 network=tcp +49:19.844 [INFO] TestAgent: started state syncer +49:19.844 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24072 network=tcp +49:19.890 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.890 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24057 [Candidate]" term=2 +49:19.890 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.890 [DEBUG] TestAgent.server.raft: vote granted: from=6997522e-4f68-2604-3ba4-c9a057505aa0 term=2 tally=1 +49:19.890 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.890 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24057 [Leader]" +49:19.891 [INFO] TestAgent.server: cluster leadership acquired +49:19.891 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24057 +49:19.891 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.892 [INFO] TestAgent.server: New leader elected: payload=Node-6997522e-4f68-2604-3ba4-c9a057505aa0 +49:19.892 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.892 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.892 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.892 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.892 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.892 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6997522e-4f68-2604-3ba4-c9a057505aa0 +49:19.892 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6997522e-4f68-2604-3ba4-c9a057505aa0 +49:19.893 [INFO] TestAgent.server: federation state anti-entropy synced +49:19.956 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:19.956 [INFO] TestAgent: Synced node info +49:19.956 [DEBUG] TestAgent: Node info in sync +49:20.200 [INFO] TestAgent: Requesting shutdown +49:20.200 [INFO] TestAgent.server: shutting down server +49:20.200 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.201 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.201 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.201 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.201 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.201 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.201 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.201 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.201 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.201 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.201 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.201Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.201Z [INFO] agent.router.manager: shutting down +49:20.201 [INFO] TestAgent: consul server down +49:20.201 [INFO] TestAgent: shutdown complete +49:20.201 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24052 network=tcp +49:20.201 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24052 network=udp +49:20.201 [INFO] TestAgent: Stopping server: protocol=HTTPS address=127.0.0.1:24054 network=tcp +49:20.701 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.701 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPServer_HandshakeTimeout (0.89s) +=== CONT TestHealthConnectServiceNodes_Filter +--- PASS: TestHTTP_wrap_obfuscateLog (0.62s) --- PASS: TestHTTP_wrap_obfuscateLog//some/url?token=secret1&token=secret2 (0.00s) --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/clone/secret1 (0.00s) --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/clone/secret1?token=secret2 (0.00s) @@ -23127,8 +29304,188 @@ --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/destroy/secret1?token=secret2 (0.00s) --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/info/secret1 (0.00s) --- PASS: TestHTTP_wrap_obfuscateLog//v1/acl/info/secret1?token=secret2 (0.00s) +=== CONT TestHTTPAPI_BlockEndpoints +=== RUN TestHealthConnectServiceNodes_PassingFilter/bc_no_query_value +=== RUN TestHealthConnectServiceNodes_PassingFilter/passing_true +=== RUN TestHealthConnectServiceNodes_PassingFilter/passing_false +=== RUN TestHealthConnectServiceNodes_PassingFilter/passing_bad +2022-12-04T05:49:19.871Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.882Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.883 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:40b80471-ebd2-a3d0-ea1d-f0ea5c83bc0f Address:127.0.0.1:24082}]" +49:19.883 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24082 [Follower]" leader= +49:19.884 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-40b80471-ebd2-a3d0-ea1d-f0ea5c83bc0f.dc1 127.0.0.1 +49:19.884 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-40b80471-ebd2-a3d0-ea1d-f0ea5c83bc0f 127.0.0.1 +2022-12-04T05:49:19.884Z [INFO] agent.router: Initializing LAN area manager +49:19.885 [INFO] TestAgent.server: Adding LAN server: server="Node-40b80471-ebd2-a3d0-ea1d-f0ea5c83bc0f (Addr: tcp/127.0.0.1:24082) (DC: dc1)" +49:19.885 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-40b80471-ebd2-a3d0-ea1d-f0ea5c83bc0f.dc1 area=wan +49:19.885 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24090 network=udp +49:19.885 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24090 network=tcp +49:19.885 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24091 network=tcp +49:19.885 [INFO] TestAgent: started state syncer +49:19.885 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24083 network=tcp +49:19.940 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.940 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24082 [Candidate]" term=2 +49:19.940 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.940 [DEBUG] TestAgent.server.raft: vote granted: from=40b80471-ebd2-a3d0-ea1d-f0ea5c83bc0f term=2 tally=1 +49:19.940 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.940 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24082 [Leader]" +49:19.940 [INFO] TestAgent.server: cluster leadership acquired +49:19.940 [INFO] TestAgent.server: New leader elected: payload=Node-40b80471-ebd2-a3d0-ea1d-f0ea5c83bc0f +49:19.940 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24082 +49:19.941 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.942 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.942 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.942 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.942 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.942 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.942 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-40b80471-ebd2-a3d0-ea1d-f0ea5c83bc0f +49:19.942 [INFO] TestAgent.server: member joined, marking health alive: member=Node-40b80471-ebd2-a3d0-ea1d-f0ea5c83bc0f +49:19.943 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.199 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.199 [INFO] TestAgent: Synced node info +49:20.199 [DEBUG] TestAgent: Node info in sync +49:20.231 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/key?pretty from= latency=49.03µs +49:20.231 [INFO] TestAgent: Requesting shutdown +49:20.231 [INFO] TestAgent.server: shutting down server +49:20.232 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.232 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.232 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.232 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.232 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.232 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.232 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.232 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.232 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.232 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.232 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.232Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.232Z [INFO] agent.router.manager: shutting down +49:20.232 [INFO] TestAgent: consul server down +49:20.232 [INFO] TestAgent: shutdown complete +49:20.232 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24090 network=tcp +49:20.232 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24090 network=udp +49:20.232 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24091 network=tcp +49:20.732 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.732 [INFO] TestAgent: Endpoints down +--- PASS: TestPrettyPrintBare (0.87s) +=== CONT TestHealthIngressServiceNodes +2022-12-04T05:49:19.868Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.881Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.882 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8483c705-a687-83e6-fe08-ad9e94264bef Address:127.0.0.1:24071}]" +49:19.882 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24071 [Follower]" leader= +49:19.883 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8483c705-a687-83e6-fe08-ad9e94264bef.dc1 127.0.0.1 +49:19.883 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8483c705-a687-83e6-fe08-ad9e94264bef 127.0.0.1 +2022-12-04T05:49:19.883Z [INFO] agent.router: Initializing LAN area manager +49:19.883 [INFO] TestAgent.server: Adding LAN server: server="Node-8483c705-a687-83e6-fe08-ad9e94264bef (Addr: tcp/127.0.0.1:24071) (DC: dc1)" +49:19.883 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8483c705-a687-83e6-fe08-ad9e94264bef.dc1 area=wan +49:19.883 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24066 network=udp +49:19.883 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24066 network=tcp +49:19.884 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24067 network=tcp +49:19.884 [INFO] TestAgent: started state syncer +49:19.884 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24088 network=tcp +49:19.924 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.924 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24071 [Candidate]" term=2 +49:19.924 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.924 [DEBUG] TestAgent.server.raft: vote granted: from=8483c705-a687-83e6-fe08-ad9e94264bef term=2 tally=1 +49:19.924 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.924 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24071 [Leader]" +49:19.924 [INFO] TestAgent.server: cluster leadership acquired +49:19.925 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24071 +49:19.925 [INFO] TestAgent.server: New leader elected: payload=Node-8483c705-a687-83e6-fe08-ad9e94264bef +49:19.925 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.926 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.926 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.926 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.926 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.926 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.926 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8483c705-a687-83e6-fe08-ad9e94264bef +49:19.926 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8483c705-a687-83e6-fe08-ad9e94264bef +49:19.927 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.008 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.009 [INFO] TestAgent: Synced node info +49:20.238 [INFO] TestAgent: Requesting shutdown +49:20.238 [INFO] TestAgent.server: shutting down server +49:20.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.238 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.238 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.238 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.238 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.238 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.238 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.238 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.238Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.238Z [INFO] agent.router.manager: shutting down +49:20.238 [INFO] TestAgent: consul server down +49:20.238 [INFO] TestAgent: shutdown complete +49:20.238 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24066 network=tcp +49:20.238 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24066 network=udp +49:20.238 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24067 network=tcp +49:20.739 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.739 [INFO] TestAgent: Endpoints down +--- PASS: TestEnableWebUI (0.88s) === CONT TestSetMeta --- PASS: TestSetMeta (0.00s) +=== CONT TestOperator_Keyring_LocalOnly +2022-12-04T05:49:20.084Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.095Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.096 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e480db5c-88f0-b716-63a8-a46b41d2ba3e Address:127.0.0.1:24096}]" +49:20.096 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24096 [Follower]" leader= +49:20.097 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e480db5c-88f0-b716-63a8-a46b41d2ba3e.dc1 127.0.0.1 +49:20.097 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e480db5c-88f0-b716-63a8-a46b41d2ba3e 127.0.0.1 +2022-12-04T05:49:20.097Z [INFO] agent.router: Initializing LAN area manager +49:20.097 [INFO] TestAgent.server: Adding LAN server: server="Node-e480db5c-88f0-b716-63a8-a46b41d2ba3e (Addr: tcp/127.0.0.1:24096) (DC: dc1)" +49:20.098 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24105 network=udp +49:20.098 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e480db5c-88f0-b716-63a8-a46b41d2ba3e.dc1 area=wan +49:20.098 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24105 network=tcp +49:20.098 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24106 network=tcp +49:20.098 [INFO] TestAgent: started state syncer +49:20.098 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24097 network=tcp +49:20.159 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.159 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24096 [Candidate]" term=2 +49:20.159 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.159 [DEBUG] TestAgent.server.raft: vote granted: from=e480db5c-88f0-b716-63a8-a46b41d2ba3e term=2 tally=1 +49:20.159 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.159 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24096 [Leader]" +49:20.159 [INFO] TestAgent.server: cluster leadership acquired +49:20.159 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24096 +49:20.159 [INFO] TestAgent.server: New leader elected: payload=Node-e480db5c-88f0-b716-63a8-a46b41d2ba3e +49:20.160 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.161 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.161 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.161 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.161 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.161 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.161 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e480db5c-88f0-b716-63a8-a46b41d2ba3e +49:20.161 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e480db5c-88f0-b716-63a8-a46b41d2ba3e +49:20.162 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.253 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/key?pretty=1 from= latency=31.81µs +49:20.253 [INFO] TestAgent: Requesting shutdown +49:20.253 [INFO] TestAgent.server: shutting down server +49:20.253 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.253 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.253 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.253 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.253 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.253 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.253 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.253 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.253 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:20.253 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.253 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.253 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.254Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.254Z [INFO] agent.router.manager: shutting down +49:20.254 [INFO] TestAgent: consul server down +49:20.254 [INFO] TestAgent: shutdown complete +49:20.254 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24105 network=tcp +49:20.254 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24105 network=udp +49:20.254 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24106 network=tcp +49:20.754 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.754 [INFO] TestAgent: Endpoints down +--- PASS: TestPrettyPrint (0.68s) === CONT TestSetLastContact === RUN TestSetLastContact/neg === RUN TestSetLastContact/zero @@ -23139,4521 +29496,4328 @@ --- PASS: TestSetLastContact/zero (0.00s) --- PASS: TestSetLastContact/pos (0.00s) --- PASS: TestSetLastContact/pos_ms_only (0.00s) +=== CONT TestOperator_Keyring_InvalidRelayFactor +2022-12-04T05:49:20.034Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.043Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.045 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:86e7d0ef-0462-8b4d-67a7-08c0eff4e090 Address:127.0.0.1:24103}]" +49:20.045 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-86e7d0ef-0462-8b4d-67a7-08c0eff4e090.dc1 127.0.0.1 +49:20.046 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-86e7d0ef-0462-8b4d-67a7-08c0eff4e090 127.0.0.1 +2022-12-04T05:49:20.046Z [INFO] agent.router: Initializing LAN area manager +49:20.046 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24084 network=udp +49:20.046 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24103 [Follower]" leader= +49:20.046 [INFO] TestAgent.server: Adding LAN server: server="Node-86e7d0ef-0462-8b4d-67a7-08c0eff4e090 (Addr: tcp/127.0.0.1:24103) (DC: dc1)" +49:20.046 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-86e7d0ef-0462-8b4d-67a7-08c0eff4e090.dc1 area=wan +49:20.046 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24084 network=tcp +49:20.047 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24085 network=tcp +49:20.047 [INFO] TestAgent: started state syncer +49:20.047 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24104 network=tcp +49:20.087 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.087 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24103 [Candidate]" term=2 +49:20.088 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.088 [DEBUG] TestAgent.server.raft: vote granted: from=86e7d0ef-0462-8b4d-67a7-08c0eff4e090 term=2 tally=1 +49:20.088 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.088 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24103 [Leader]" +49:20.088 [INFO] TestAgent.server: cluster leadership acquired +49:20.088 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24103 +49:20.088 [INFO] TestAgent.server: New leader elected: payload=Node-86e7d0ef-0462-8b4d-67a7-08c0eff4e090 +49:20.089 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.090 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.090 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.090 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.090 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.090 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.090 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-86e7d0ef-0462-8b4d-67a7-08c0eff4e090 +49:20.090 [INFO] TestAgent.server: member joined, marking health alive: member=Node-86e7d0ef-0462-8b4d-67a7-08c0eff4e090 +49:20.091 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.222 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.222 [INFO] TestAgent: Synced node info +49:20.283 [INFO] TestAgent: Requesting shutdown +49:20.283 [INFO] TestAgent.server: shutting down server +49:20.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.283 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.283 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.283 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.283 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.283 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.283 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.283 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:20.284 [INFO] TestAgent: consul server down +49:20.284 [INFO] TestAgent: shutdown complete +49:20.284 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24084 network=tcp +49:20.284 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24084 network=udp +2022-12-04T05:49:20.283Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.284Z [INFO] agent.router.manager: shutting down +49:20.284 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24085 network=tcp +49:20.784 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.785 [INFO] TestAgent: Endpoints down +--- PASS: TestACLResolution (0.79s) === CONT TestSetKnownLeader --- PASS: TestSetKnownLeader (0.00s) +=== CONT TestHealthConnectServiceNodes +2022/12/04 05:49:20 http: TLS handshake error from 127.0.0.1:41008: read tcp 127.0.0.1:24177->127.0.0.1:41008: use of closed network connection +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022/12/04 05:49:20 http: TLS handshake error from 127.0.0.1:41004: EOF +2022/12/04 05:49:20 http: TLS handshake error from 127.0.0.1:41010: EOF +2022/12/04 05:49:20 http: TLS handshake error from 127.0.0.1:41006: EOF +2022-12-04T05:49:20.236Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.246Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.247 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9c523357-20f6-d172-63b0-59a3522b3154 Address:127.0.0.1:24131}]" +49:20.247 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24131 [Follower]" leader= +49:20.247 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9c523357-20f6-d172-63b0-59a3522b3154.dc1 127.0.0.1 +49:20.248 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9c523357-20f6-d172-63b0-59a3522b3154 127.0.0.1 +2022-12-04T05:49:20.248Z [INFO] agent.router: Initializing LAN area manager +49:20.248 [INFO] TestAgent.server: Adding LAN server: server="Node-9c523357-20f6-d172-63b0-59a3522b3154 (Addr: tcp/127.0.0.1:24131) (DC: dc1)" +49:20.248 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9c523357-20f6-d172-63b0-59a3522b3154.dc1 area=wan +49:20.248 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24126 network=udp +49:20.248 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24126 network=tcp +49:20.249 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24127 network=tcp +49:20.249 [INFO] TestAgent: started state syncer +49:20.249 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24132 network=tcp +49:20.312 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.312 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24131 [Candidate]" term=2 +49:20.312 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.312 [DEBUG] TestAgent.server.raft: vote granted: from=9c523357-20f6-d172-63b0-59a3522b3154 term=2 tally=1 +49:20.312 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.312 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24131 [Leader]" +49:20.312 [INFO] TestAgent.server: cluster leadership acquired +49:20.312 [INFO] TestAgent.server: New leader elected: payload=Node-9c523357-20f6-d172-63b0-59a3522b3154 +49:20.312 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24131 +49:20.313 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.314 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.314 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.314 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.314 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.314 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.314 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9c523357-20f6-d172-63b0-59a3522b3154 +49:20.314 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9c523357-20f6-d172-63b0-59a3522b3154 +49:20.315 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.495 [DEBUG] TestAgent.http: Request finished: method=GET url=/ui from= latency=2.135µs +49:20.495 [INFO] TestAgent: Requesting shutdown +49:20.495 [INFO] TestAgent.server: shutting down server +49:20.495 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.495 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.495 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.495 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.495 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.495 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.495 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.495 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:20.495 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.495 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.495 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.495 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.495Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.495Z [INFO] agent.router.manager: shutting down +49:20.495 [INFO] TestAgent: consul server down +49:20.495 [INFO] TestAgent: shutdown complete +49:20.495 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24126 network=tcp +49:20.495 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24126 network=udp +49:20.495 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24127 network=tcp +49:20.995 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.995 [INFO] TestAgent: Endpoints down +--- PASS: TestUIResponseHeaders (0.77s) +=== CONT TestOperator_KeyringUse +2022-12-04T05:49:20.147Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.156Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.157 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1868ebe8-ec58-2b13-7eeb-3ac91d217cf0 Address:127.0.0.1:24117}]" +49:20.158 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24117 [Follower]" leader= +49:20.158 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1868ebe8-ec58-2b13-7eeb-3ac91d217cf0.dc1 127.0.0.1 +49:20.158 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1868ebe8-ec58-2b13-7eeb-3ac91d217cf0 127.0.0.1 +2022-12-04T05:49:20.158Z [INFO] agent.router: Initializing LAN area manager +49:20.159 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24112 network=udp +49:20.159 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1868ebe8-ec58-2b13-7eeb-3ac91d217cf0.dc1 area=wan +49:20.159 [INFO] TestAgent.server: Adding LAN server: server="Node-1868ebe8-ec58-2b13-7eeb-3ac91d217cf0 (Addr: tcp/127.0.0.1:24117) (DC: dc1)" +49:20.159 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24112 network=tcp +49:20.159 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24113 network=tcp +49:20.159 [INFO] TestAgent: started state syncer +49:20.159 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24118 network=tcp +49:20.207 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.207 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24117 [Candidate]" term=2 +49:20.208 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.208 [DEBUG] TestAgent.server.raft: vote granted: from=1868ebe8-ec58-2b13-7eeb-3ac91d217cf0 term=2 tally=1 +49:20.208 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.208 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24117 [Leader]" +49:20.208 [INFO] TestAgent.server: cluster leadership acquired +49:20.208 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24117 +49:20.208 [INFO] TestAgent.server: New leader elected: payload=Node-1868ebe8-ec58-2b13-7eeb-3ac91d217cf0 +49:20.208 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.210 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.210 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.210 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.210 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.210 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.210 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1868ebe8-ec58-2b13-7eeb-3ac91d217cf0 +49:20.210 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1868ebe8-ec58-2b13-7eeb-3ac91d217cf0 +49:20.210 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.516 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/key from= latency=24.446µs +49:20.516 [INFO] TestAgent: Requesting shutdown +49:20.516 [INFO] TestAgent.server: shutting down server +49:20.516 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.516 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.516 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.516 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.516 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.516 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.516 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.516 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.516 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:20.516 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.516 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.516 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.517Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.517Z [INFO] agent.router.manager: shutting down +49:20.517 [INFO] TestAgent: consul server down +49:20.517 [INFO] TestAgent: shutdown complete +49:20.517 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24112 network=tcp +49:20.517 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24112 network=udp +49:20.517 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24113 network=tcp +49:21.018 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.018 [INFO] TestAgent: Endpoints down +--- PASS: TestContentTypeIsJSON (0.88s) +=== CONT TestHealthServiceNodes_WanTranslation +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:20.159Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.167Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.169 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3fd05bcb-409b-2237-e273-3b513814747a Address:127.0.0.1:24124}]" +49:20.169 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24124 [Follower]" leader= +49:20.169 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3fd05bcb-409b-2237-e273-3b513814747a.dc1 127.0.0.1 +49:20.170 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3fd05bcb-409b-2237-e273-3b513814747a 127.0.0.1 +2022-12-04T05:49:20.170Z [INFO] agent.router: Initializing LAN area manager +49:20.170 [INFO] TestAgent.server: Adding LAN server: server="Node-3fd05bcb-409b-2237-e273-3b513814747a (Addr: tcp/127.0.0.1:24124) (DC: dc1)" +49:20.170 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3fd05bcb-409b-2237-e273-3b513814747a.dc1 area=wan +49:20.170 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24119 network=tcp +49:20.170 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24119 network=udp +49:20.170 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24120 network=tcp +49:20.171 [INFO] TestAgent: started state syncer +49:20.171 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24125 network=tcp +49:20.212 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.212 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24124 [Candidate]" term=2 +49:20.212 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.212 [DEBUG] TestAgent.server.raft: vote granted: from=3fd05bcb-409b-2237-e273-3b513814747a term=2 tally=1 +49:20.212 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.212 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24124 [Leader]" +49:20.213 [INFO] TestAgent.server: cluster leadership acquired +49:20.213 [INFO] TestAgent.server: New leader elected: payload=Node-3fd05bcb-409b-2237-e273-3b513814747a +49:20.213 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24124 +49:20.213 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.215 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.215 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.215 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.215 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.215 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.215 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3fd05bcb-409b-2237-e273-3b513814747a +49:20.215 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3fd05bcb-409b-2237-e273-3b513814747a +49:20.215 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.575 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.575 [INFO] TestAgent: Synced node info +49:20.586 [INFO] TestAgent: Requesting shutdown +49:20.586 [INFO] TestAgent.server: shutting down server +49:20.586 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.586 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.586 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.586 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.586 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.586 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.586 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.586 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.586 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.586 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.586 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.586Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.586Z [INFO] agent.router.manager: shutting down +49:20.586 [INFO] TestAgent: consul server down +49:20.586 [INFO] TestAgent: shutdown complete +49:20.586 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24119 network=tcp +49:20.586 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24119 network=udp +49:20.586 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24120 network=tcp +49:21.086 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.087 [INFO] TestAgent: Endpoints down +--- PASS: TestPreparedQuery_Execute (6.07s) + --- PASS: TestPreparedQuery_Execute/#00 (0.71s) + --- PASS: TestPreparedQuery_Execute/#01 (0.68s) + --- PASS: TestPreparedQuery_Execute/#02 (0.93s) + --- PASS: TestPreparedQuery_Execute/#03 (0.76s) + --- PASS: TestPreparedQuery_Execute/#04 (0.72s) + --- PASS: TestPreparedQuery_Execute/#05 (0.62s) + --- PASS: TestPreparedQuery_Execute/#06 (0.72s) + --- PASS: TestPreparedQuery_Execute/#07 (0.94s) +=== CONT TestOperator_KeyringRemove +=== RUN TestHealthIngressServiceNodes/associated_service +=== RUN TestHealthIngressServiceNodes/non-associated_service +2022-12-04T05:49:20.257Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.270Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.276 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bc72a219-173d-4779-09b1-0ba6e2404b0f Address:127.0.0.1:24138}]" +49:20.277 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bc72a219-173d-4779-09b1-0ba6e2404b0f.dc1 127.0.0.1 +49:20.278 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bc72a219-173d-4779-09b1-0ba6e2404b0f 127.0.0.1 +2022-12-04T05:49:20.278Z [INFO] agent.router: Initializing LAN area manager +49:20.278 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24133 network=udp +49:20.278 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24138 [Follower]" leader= +49:20.278 [INFO] TestAgent.server: Adding LAN server: server="Node-bc72a219-173d-4779-09b1-0ba6e2404b0f (Addr: tcp/127.0.0.1:24138) (DC: dc1)" +49:20.278 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bc72a219-173d-4779-09b1-0ba6e2404b0f.dc1 area=wan +49:20.279 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24133 network=tcp +49:20.280 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24134 network=tcp +49:20.280 [INFO] TestAgent: started state syncer +49:20.280 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24139 network=tcp +49:20.340 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.340 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24138 [Candidate]" term=2 +49:20.340 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.340 [DEBUG] TestAgent.server.raft: vote granted: from=bc72a219-173d-4779-09b1-0ba6e2404b0f term=2 tally=1 +49:20.340 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.340 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24138 [Leader]" +49:20.340 [INFO] TestAgent.server: cluster leadership acquired +49:20.340 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24138 +49:20.341 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.341 [INFO] TestAgent.server: New leader elected: payload=Node-bc72a219-173d-4779-09b1-0ba6e2404b0f +49:20.342 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.342 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.342 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.342 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.342 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.342 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bc72a219-173d-4779-09b1-0ba6e2404b0f +49:20.342 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bc72a219-173d-4779-09b1-0ba6e2404b0f +49:20.343 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.668 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.669 [INFO] TestAgent: Synced node info +49:20.686 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from= latency=1.468µs +49:20.686 [INFO] TestAgent: Requesting shutdown +49:20.686 [INFO] TestAgent.server: shutting down server +49:20.686 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.686 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.686 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.686 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.686 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.686 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.686 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.686 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.686 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.686 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.686 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.686Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.686Z [INFO] agent.router.manager: shutting down +49:20.687 [INFO] TestAgent: consul server down +49:20.687 [INFO] TestAgent: shutdown complete +49:20.687 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24133 network=tcp +49:20.687 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24133 network=udp +49:20.687 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24134 network=tcp +49:21.187 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.187 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPAPIResponseHeaders (0.94s) +=== CONT TestHealthServiceNodes_CheckType +2022-12-04T05:49:20.327Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.338Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.343 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4b823455-9c9f-1b79-53b5-888f2b2bc3ef Address:127.0.0.1:24152}]" +49:20.343 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24152 [Follower]" leader= +49:20.344 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4b823455-9c9f-1b79-53b5-888f2b2bc3ef.dc1 127.0.0.1 +49:20.344 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4b823455-9c9f-1b79-53b5-888f2b2bc3ef 127.0.0.1 +2022-12-04T05:49:20.344Z [INFO] agent.router: Initializing LAN area manager +49:20.344 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24147 network=udp +49:20.344 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4b823455-9c9f-1b79-53b5-888f2b2bc3ef.dc1 area=wan +49:20.344 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24147 network=tcp +49:20.344 [INFO] TestAgent.server: Adding LAN server: server="Node-4b823455-9c9f-1b79-53b5-888f2b2bc3ef (Addr: tcp/127.0.0.1:24152) (DC: dc1)" +49:20.345 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24148 network=tcp +49:20.345 [INFO] TestAgent: started state syncer +49:20.345 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24153 network=tcp +49:20.382 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.382 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24152 [Candidate]" term=2 +49:20.382 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.382 [DEBUG] TestAgent.server.raft: vote granted: from=4b823455-9c9f-1b79-53b5-888f2b2bc3ef term=2 tally=1 +49:20.382 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.382 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24152 [Leader]" +49:20.383 [INFO] TestAgent.server: cluster leadership acquired +49:20.383 [INFO] TestAgent.server: New leader elected: payload=Node-4b823455-9c9f-1b79-53b5-888f2b2bc3ef +49:20.383 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24152 +49:20.384 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.385 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.385 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.385 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.385 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.385 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.385 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4b823455-9c9f-1b79-53b5-888f2b2bc3ef +49:20.385 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4b823455-9c9f-1b79-53b5-888f2b2bc3ef +49:20.385 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.508 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.508 [INFO] TestAgent: Synced node info +49:20.508 [DEBUG] TestAgent: Node info in sync +49:20.709 [INFO] TestAgent: Requesting shutdown +49:20.709 [INFO] TestAgent.server: shutting down server +49:20.709 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.709 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.709 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.709 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.709 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.709 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.709 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.709 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.709 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.709 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.709 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.709Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.709Z [INFO] agent.router.manager: shutting down +49:20.709 [INFO] TestAgent: consul server down +49:20.709 [INFO] TestAgent: shutdown complete +49:20.709 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24147 network=tcp +49:20.709 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24147 network=udp +49:20.710 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24148 network=tcp +49:21.210 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.210 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthConnectServiceNodes_PassingFilter (0.90s) + --- PASS: TestHealthConnectServiceNodes_PassingFilter/bc_no_query_value (0.00s) + --- PASS: TestHealthConnectServiceNodes_PassingFilter/passing_true (0.00s) + --- PASS: TestHealthConnectServiceNodes_PassingFilter/passing_false (0.00s) + --- PASS: TestHealthConnectServiceNodes_PassingFilter/passing_bad (0.00s) +=== CONT TestOperator_KeyringListLocalOnly +2022-12-04T05:49:20.437Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:20.437Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.449Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:20.449Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.451 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d46998d6-44f9-3d98-7579-b469f6c45a96 Address:127.0.0.1:24166}]" +49:20.451 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24166 [Follower]" leader= +49:20.451 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d46998d6-44f9-3d98-7579-b469f6c45a96.dc1 127.0.0.1 +49:20.452 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d46998d6-44f9-3d98-7579-b469f6c45a96 127.0.0.1 +2022-12-04T05:49:20.452Z [INFO] agent.router: Initializing LAN area manager +49:20.452 [INFO] TestAgent.server: Adding LAN server: server="Node-d46998d6-44f9-3d98-7579-b469f6c45a96 (Addr: tcp/127.0.0.1:24166) (DC: dc1)" +49:20.452 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24161 network=udp +49:20.452 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d46998d6-44f9-3d98-7579-b469f6c45a96.dc1 area=wan +49:20.452 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24161 network=tcp +49:20.453 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24162 network=tcp +49:20.453 [INFO] TestAgent: started state syncer +49:20.453 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24167 network=tcp +49:20.502 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:20.517 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.517 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24166 [Candidate]" term=2 +49:20.517 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.517 [DEBUG] TestAgent.server.raft: vote granted: from=d46998d6-44f9-3d98-7579-b469f6c45a96 term=2 tally=1 +49:20.517 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.517 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24166 [Leader]" +49:20.517 [INFO] TestAgent.server: cluster leadership acquired +49:20.517 [INFO] TestAgent.server: New leader elected: payload=Node-d46998d6-44f9-3d98-7579-b469f6c45a96 +49:20.517 [INFO] TestAgent.server: initializing acls +49:20.517 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:20.517 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:20.518 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:20.518 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:20.518 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:20.518 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:20.518 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d46998d6-44f9-3d98-7579-b469f6c45a96 +49:20.518 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d46998d6-44f9-3d98-7579-b469f6c45a96.dc1 +49:20.518 [INFO] TestAgent.server: Updating LAN server: server="Node-d46998d6-44f9-3d98-7579-b469f6c45a96 (Addr: tcp/127.0.0.1:24166) (DC: dc1)" +49:20.518 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d46998d6-44f9-3d98-7579-b469f6c45a96.dc1 area=wan +49:20.518 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.520 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.520 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.520 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.520 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.520 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.520 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d46998d6-44f9-3d98-7579-b469f6c45a96 +49:20.520 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d46998d6-44f9-3d98-7579-b469f6c45a96 +49:20.520 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d46998d6-44f9-3d98-7579-b469f6c45a96 +49:20.520 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.768 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d46998d6-44f9-3d98-7579-b469f6c45a96 +49:20.768 [INFO] TestAgent: Requesting shutdown +49:20.769 [INFO] TestAgent.server: shutting down server +49:20.769 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.769 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.769 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:20.769 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:20.769 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.769 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.769 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.769 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:20.769 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.769 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:20.769 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.769 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.769 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.769 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.769 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:20.769 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +2022-12-04T05:49:20.769Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.769Z [INFO] agent.router.manager: shutting down +49:20.769 [INFO] TestAgent: consul server down +49:20.769 [INFO] TestAgent: shutdown complete +49:20.769 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24161 network=tcp +49:20.769 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24161 network=udp +49:20.769 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24162 network=tcp +49:21.269 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.269 [INFO] TestAgent: Endpoints down +=== RUN TestSnapshot_Options/PUT#02 +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:20.467Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.479Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.480 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ca925870-f93e-ab28-b564-325599a02987 Address:127.0.0.1:24180}]" +49:20.480 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24180 [Follower]" leader= +49:20.481 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ca925870-f93e-ab28-b564-325599a02987.dc1 127.0.0.1 +49:20.481 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ca925870-f93e-ab28-b564-325599a02987 127.0.0.1 +2022-12-04T05:49:20.481Z [INFO] agent.router: Initializing LAN area manager +49:20.481 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ca925870-f93e-ab28-b564-325599a02987.dc1 area=wan +49:20.481 [INFO] TestAgent.server: Adding LAN server: server="Node-ca925870-f93e-ab28-b564-325599a02987 (Addr: tcp/127.0.0.1:24180) (DC: dc1)" +49:20.481 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24175 network=tcp +49:20.481 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24175 network=udp +49:20.482 [INFO] TestAgent: Started HTTPS server: address=127.0.0.1:24177 network=tcp +49:20.482 [INFO] TestAgent: started state syncer +49:20.482 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24181 network=tcp +49:20.538 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.538 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24180 [Candidate]" term=2 +49:20.538 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.538 [DEBUG] TestAgent.server.raft: vote granted: from=ca925870-f93e-ab28-b564-325599a02987 term=2 tally=1 +49:20.538 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.538 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24180 [Leader]" +49:20.538 [INFO] TestAgent.server: cluster leadership acquired +49:20.538 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24180 +49:20.539 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.540 [INFO] TestAgent.server: New leader elected: payload=Node-ca925870-f93e-ab28-b564-325599a02987 +49:20.541 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.541 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.541 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.541 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.541 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.542 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ca925870-f93e-ab28-b564-325599a02987 +49:20.542 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ca925870-f93e-ab28-b564-325599a02987 +49:20.544 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.723 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.723 [INFO] TestAgent: Synced node info +49:20.723 [DEBUG] TestAgent: Node info in sync +49:20.779 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.779 [DEBUG] TestAgent: Node info in sync +49:20.804 [INFO] TestAgent: Requesting shutdown +49:20.804 [INFO] TestAgent.server: shutting down server +49:20.804 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.804 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.804 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.804Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.804Z [INFO] agent.router.manager: shutting down +49:20.804 [INFO] TestAgent: consul server down +49:20.804 [INFO] TestAgent: shutdown complete +49:20.804 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24175 network=tcp +49:20.804 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24175 network=udp +49:20.804 [INFO] TestAgent: Stopping server: protocol=HTTPS address=127.0.0.1:24177 network=tcp +49:21.305 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.305 [INFO] TestAgent: Endpoints down +--- PASS: TestRPC_HTTPSMaxConnsPerClient (1.67s) + --- PASS: TestRPC_HTTPSMaxConnsPerClient/HTTP (0.82s) + --- PASS: TestRPC_HTTPSMaxConnsPerClient/HTTPS (0.85s) +=== CONT TestHealthServiceNodes_PassingFilter +2022-12-04T05:49:20.712Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.722Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.723 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0162183c-908d-7b54-eba6-5f64449891ec Address:127.0.0.1:24187}]" +49:20.723 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24187 [Follower]" leader= +49:20.724 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0162183c-908d-7b54-eba6-5f64449891ec.dc1 127.0.0.1 +49:20.724 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0162183c-908d-7b54-eba6-5f64449891ec 127.0.0.1 +2022-12-04T05:49:20.724Z [INFO] agent.router: Initializing LAN area manager +49:20.724 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0162183c-908d-7b54-eba6-5f64449891ec.dc1 area=wan +49:20.724 [INFO] TestAgent.server: Adding LAN server: server="Node-0162183c-908d-7b54-eba6-5f64449891ec (Addr: tcp/127.0.0.1:24187) (DC: dc1)" +49:20.724 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24182 network=udp +49:20.725 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24182 network=tcp +49:20.725 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24183 network=tcp +49:20.725 [INFO] TestAgent: started state syncer +49:20.725 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24188 network=tcp +49:20.791 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.791 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24187 [Candidate]" term=2 +49:20.791 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.791 [DEBUG] TestAgent.server.raft: vote granted: from=0162183c-908d-7b54-eba6-5f64449891ec term=2 tally=1 +49:20.791 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.791 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24187 [Leader]" +49:20.791 [INFO] TestAgent.server: cluster leadership acquired +49:20.791 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24187 +49:20.792 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.792 [INFO] TestAgent.server: New leader elected: payload=Node-0162183c-908d-7b54-eba6-5f64449891ec +49:20.793 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.793 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.793 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.793 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.793 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.793 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0162183c-908d-7b54-eba6-5f64449891ec +49:20.793 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0162183c-908d-7b54-eba6-5f64449891ec +49:20.794 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.840 [INFO] TestAgent: Requesting shutdown +49:20.840 [INFO] TestAgent.server: shutting down server +49:20.840 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.840 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.840 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.840 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.840 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.840 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.840 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.840 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.840 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:20.840 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.840 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.840 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.841Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.841Z [INFO] agent.router.manager: shutting down +49:20.841 [INFO] TestAgent: consul server down +49:20.841 [INFO] TestAgent: shutdown complete +49:20.841 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24182 network=tcp +49:20.841 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24182 network=udp +49:20.841 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24183 network=tcp +49:21.341 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.341 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthConnectServiceNodes_Filter (0.64s) +=== CONT TestOperator_KeyringList +2022-12-04T05:49:20.796Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.806Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.808 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8534c7b8-e910-363d-dbd7-0af3601e67f7 Address:127.0.0.1:24222}]" +49:20.808 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24222 [Follower]" leader= +49:20.808 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8534c7b8-e910-363d-dbd7-0af3601e67f7.dc1 127.0.0.1 +49:20.809 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8534c7b8-e910-363d-dbd7-0af3601e67f7 127.0.0.1 +2022-12-04T05:49:20.809Z [INFO] agent.router: Initializing LAN area manager +49:20.809 [INFO] TestAgent.server: Adding LAN server: server="Node-8534c7b8-e910-363d-dbd7-0af3601e67f7 (Addr: tcp/127.0.0.1:24222) (DC: dc1)" +49:20.809 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24217 network=udp +49:20.809 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8534c7b8-e910-363d-dbd7-0af3601e67f7.dc1 area=wan +49:20.809 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24217 network=tcp +49:20.809 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24218 network=tcp +49:20.809 [INFO] TestAgent: started state syncer +49:20.809 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24223 network=tcp +49:20.856 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.856 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24222 [Candidate]" term=2 +49:20.856 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.856 [DEBUG] TestAgent.server.raft: vote granted: from=8534c7b8-e910-363d-dbd7-0af3601e67f7 term=2 tally=1 +49:20.856 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.856 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24222 [Leader]" +49:20.857 [INFO] TestAgent.server: cluster leadership acquired +49:20.857 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24222 +49:20.857 [INFO] TestAgent.server: New leader elected: payload=Node-8534c7b8-e910-363d-dbd7-0af3601e67f7 +49:20.857 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.858 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.858 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.858 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.858 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.858 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.858 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8534c7b8-e910-363d-dbd7-0af3601e67f7 +49:20.858 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8534c7b8-e910-363d-dbd7-0af3601e67f7 +49:20.859 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.894 [INFO] TestAgent: Requesting shutdown +49:20.894 [INFO] TestAgent.server: shutting down server +49:20.894 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.894 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.894 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.894 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.894 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.894 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.894 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.894 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:20.894 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.894 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.894 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.894 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.894Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.894Z [INFO] agent.router.manager: shutting down +49:20.895 [INFO] TestAgent: consul server down +49:20.895 [INFO] TestAgent: shutdown complete +49:20.895 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24217 network=tcp +49:20.895 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24217 network=udp +49:20.895 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24218 network=tcp +49:21.395 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.395 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthConnectServiceNodes (0.61s) +=== CONT TestOperator_KeyringInstall +2022-12-04T05:49:20.751Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.763Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.764 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f1ee359b-373a-2bf1-a12f-c91ccdc5f2f3 Address:127.0.0.1:24208}]" +49:20.765 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24208 [Follower]" leader= +49:20.765 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f1ee359b-373a-2bf1-a12f-c91ccdc5f2f3.dc1 127.0.0.1 +49:20.765 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f1ee359b-373a-2bf1-a12f-c91ccdc5f2f3 127.0.0.1 +2022-12-04T05:49:20.765Z [INFO] agent.router: Initializing LAN area manager +49:20.765 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24203 network=udp +49:20.765 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f1ee359b-373a-2bf1-a12f-c91ccdc5f2f3.dc1 area=wan +49:20.765 [INFO] TestAgent.server: Adding LAN server: server="Node-f1ee359b-373a-2bf1-a12f-c91ccdc5f2f3 (Addr: tcp/127.0.0.1:24208) (DC: dc1)" +49:20.766 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24203 network=tcp +49:20.766 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24204 network=tcp +49:20.766 [INFO] TestAgent: started state syncer +49:20.766 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24209 network=tcp +49:20.801 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.801 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24208 [Candidate]" term=2 +49:20.801 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.801 [DEBUG] TestAgent.server.raft: vote granted: from=f1ee359b-373a-2bf1-a12f-c91ccdc5f2f3 term=2 tally=1 +49:20.801 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.801 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24208 [Leader]" +49:20.802 [INFO] TestAgent.server: cluster leadership acquired +49:20.802 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24208 +49:20.802 [INFO] TestAgent.server: New leader elected: payload=Node-f1ee359b-373a-2bf1-a12f-c91ccdc5f2f3 +49:20.802 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.803 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.803 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.803 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.803 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.803 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.804 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f1ee359b-373a-2bf1-a12f-c91ccdc5f2f3 +49:20.804 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f1ee359b-373a-2bf1-a12f-c91ccdc5f2f3 +49:20.804 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.876 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.876 [INFO] TestAgent: Synced node info +49:20.876 [DEBUG] TestAgent: Node info in sync +49:20.921 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +49:20.921 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-f1ee359b-373a-2bf1-a12f-c91ccdc5f2f3 +49:20.921 [INFO] TestAgent: Requesting shutdown +49:20.921 [INFO] TestAgent.server: shutting down server +49:20.921 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.921 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.921 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.921 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.921 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.921 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.921 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.922 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.922 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.922 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.922 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.922Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.922Z [INFO] agent.router.manager: shutting down +49:20.922 [INFO] TestAgent: consul server down +49:20.922 [INFO] TestAgent: shutdown complete +49:20.922 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24203 network=tcp +49:20.922 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24203 network=udp +49:20.922 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24204 network=tcp +49:21.422 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.422 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_Keyring_LocalOnly (0.68s) === CONT TestSetIndex --- PASS: TestSetIndex (0.00s) -=== CONT TestHealthServiceNodes_PassingFilter -2021-10-31T23:13:05.747Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:05.877Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:05.934 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2849af11-d205-1880-3433-ddf1f0886dc4 Address:127.0.0.1:16288}]" -13:05.935 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2849af11-d205-1880-3433-ddf1f0886dc4.dc1 127.0.0.1 -13:05.935 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2849af11-d205-1880-3433-ddf1f0886dc4 127.0.0.1 -2021-10-31T23:13:05.935Z [INFO] agent.router: Initializing LAN area manager -13:05.935 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16288 [Follower]" leader= -13:05.936 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16283 network=udp -13:05.936 [INFO] TestAgent.server: Adding LAN server: server="Node-2849af11-d205-1880-3433-ddf1f0886dc4 (Addr: tcp/127.0.0.1:16288) (DC: dc1)" -13:05.936 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2849af11-d205-1880-3433-ddf1f0886dc4.dc1 area=wan -13:05.936 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16283 network=tcp -13:05.936 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16284 network=tcp -13:05.936 [INFO] TestAgent: started state syncer -13:05.937 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16289 network=tcp -13:05.982 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.982 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16288 [Candidate]" term=2 -13:05.982 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.982 [DEBUG] TestAgent.server.raft: vote granted: from=2849af11-d205-1880-3433-ddf1f0886dc4 term=2 tally=1 -13:05.982 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.982 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16288 [Leader]" -13:05.982 [INFO] TestAgent.server: cluster leadership acquired -13:05.982 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16288 -13:05.982 [INFO] TestAgent.server: New leader elected: payload=Node-2849af11-d205-1880-3433-ddf1f0886dc4 -13:05.992 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.998 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.998 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.998 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.998 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.998 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.006 [INFO] TestAgent.server: federation state anti-entropy synced -13:06.013 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2849af11-d205-1880-3433-ddf1f0886dc4 -13:06.013 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2849af11-d205-1880-3433-ddf1f0886dc4 -13:06.128 [INFO] TestAgent: Requesting shutdown -13:06.128 [INFO] TestAgent.server: shutting down server -13:06.128 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:06.128 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:06.128 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:06.128 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:06.128 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:06.128 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:06.128 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:06.128 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:06.128 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:06.128 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:06.128 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:06.128 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:06.128Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:06.129Z [INFO] agent.router.manager: shutting down -13:06.139 [INFO] TestAgent: consul server down -13:06.139 [INFO] TestAgent: shutdown complete -13:06.139 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16283 network=tcp -13:06.139 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16283 network=udp -13:06.139 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16284 network=tcp -13:06.647 [INFO] TestAgent: Waiting for endpoints to shut down -13:06.647 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsList_empty (0.93s) +=== CONT TestOperator_RaftPeer +=== RUN TestOperator_RaftPeer/#00 +2022-12-04T05:49:19.600Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:19.613Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:19.614 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fe462b8b-2435-f648-05e7-c39f6c4a94d7 Address:127.0.0.1:24043}]" +49:19.615 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24043 [Follower]" leader= +49:19.615 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fe462b8b-2435-f648-05e7-c39f6c4a94d7.dc1 127.0.0.1 +49:19.615 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fe462b8b-2435-f648-05e7-c39f6c4a94d7 127.0.0.1 +2022-12-04T05:49:19.615Z [INFO] agent.router: Initializing LAN area manager +49:19.616 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24038 network=udp +49:19.616 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fe462b8b-2435-f648-05e7-c39f6c4a94d7.dc1 area=wan +49:19.616 [INFO] TestAgent.server: Adding LAN server: server="Node-fe462b8b-2435-f648-05e7-c39f6c4a94d7 (Addr: tcp/127.0.0.1:24043) (DC: dc1)" +49:19.616 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24038 network=tcp +49:19.616 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24039 network=tcp +49:19.616 [INFO] TestAgent: started state syncer +49:19.616 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24058 network=tcp +49:19.652 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:19.652 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24043 [Candidate]" term=2 +49:19.652 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:19.652 [DEBUG] TestAgent.server.raft: vote granted: from=fe462b8b-2435-f648-05e7-c39f6c4a94d7 term=2 tally=1 +49:19.652 [INFO] TestAgent.server.raft: election won: tally=1 +49:19.652 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24043 [Leader]" +49:19.653 [INFO] TestAgent.server: cluster leadership acquired +49:19.653 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24043 +49:19.653 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:19.655 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:19.655 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:19.655 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:19.655 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:19.655 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:19.655 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe462b8b-2435-f648-05e7-c39f6c4a94d7 +49:19.655 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fe462b8b-2435-f648-05e7-c39f6c4a94d7 +49:19.655 [INFO] TestAgent.server: New leader elected: payload=Node-fe462b8b-2435-f648-05e7-c39f6c4a94d7 +49:19.655 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.017 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.022 [INFO] TestAgent: Synced node info +49:20.022 [DEBUG] TestAgent: Node info in sync +49:20.536 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.541 [DEBUG] TestAgent: Node info in sync +49:21.015 [INFO] TestAgent: Requesting shutdown +49:21.015 [INFO] TestAgent.server: shutting down server +49:21.015 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.015 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.015 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.015 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.015 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.015 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.015 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.015 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.015 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.015 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.015 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:21.015Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.015Z [INFO] agent.router.manager: shutting down +49:21.016 [INFO] TestAgent: consul server down +49:21.016 [INFO] TestAgent: shutdown complete +49:21.016 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24038 network=tcp +49:21.016 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24038 network=udp +49:21.016 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24039 network=tcp +49:21.517 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.517 [INFO] TestAgent: Endpoints down +--- PASS: TestPProfHandlers_EnableDebug (1.93s) +=== CONT TestOperator_RaftConfiguration +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:20.764Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.786Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.787 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a886508e-8196-3e87-c6cb-8d644611ef38 Address:127.0.0.1:24215}]" +49:20.788 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a886508e-8196-3e87-c6cb-8d644611ef38.dc1 127.0.0.1 +49:20.788 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a886508e-8196-3e87-c6cb-8d644611ef38 127.0.0.1 +2022-12-04T05:49:20.788Z [INFO] agent.router: Initializing LAN area manager +49:20.789 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24210 network=udp +49:20.789 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24215 [Follower]" leader= +49:20.789 [INFO] TestAgent.server: Adding LAN server: server="Node-a886508e-8196-3e87-c6cb-8d644611ef38 (Addr: tcp/127.0.0.1:24215) (DC: dc1)" +49:20.789 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a886508e-8196-3e87-c6cb-8d644611ef38.dc1 area=wan +49:20.789 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24210 network=tcp +49:20.789 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24211 network=tcp +49:20.789 [INFO] TestAgent: started state syncer +49:20.789 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24216 network=tcp +49:20.827 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.827 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24215 [Candidate]" term=2 +49:20.828 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.828 [DEBUG] TestAgent.server.raft: vote granted: from=a886508e-8196-3e87-c6cb-8d644611ef38 term=2 tally=1 +49:20.828 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.828 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24215 [Leader]" +49:20.828 [INFO] TestAgent.server: cluster leadership acquired +49:20.828 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24215 +49:20.828 [INFO] TestAgent.server: New leader elected: payload=Node-a886508e-8196-3e87-c6cb-8d644611ef38 +49:20.828 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.829 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.829 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.829 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.829 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.829 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.830 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a886508e-8196-3e87-c6cb-8d644611ef38 +49:20.830 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a886508e-8196-3e87-c6cb-8d644611ef38 +49:20.830 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.867 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.867 [INFO] TestAgent: Synced node info +49:21.057 [INFO] TestAgent: Requesting shutdown +49:21.057 [INFO] TestAgent.server: shutting down server +49:21.057 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.057 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.057 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.057 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.057 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.058 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.058 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.058 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.058 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.058 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.058 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:21.058Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.058Z [INFO] agent.router.manager: shutting down +49:21.058 [INFO] TestAgent: consul server down +49:21.058 [INFO] TestAgent: shutdown complete +49:21.058 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24210 network=tcp +49:21.058 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24210 network=udp +49:21.058 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24211 network=tcp +49:21.558 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.558 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_Keyring_InvalidRelayFactor (0.80s) +=== CONT TestKVSEndpoint_DELETE_ConflictingFlags +2022-12-04T05:49:20.336Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.346Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.348 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3123150c-dd36-5574-5010-dae0fa854537 Address:127.0.0.1:24159}]" +49:20.348 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24159 [Follower]" leader= +49:20.348 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3123150c-dd36-5574-5010-dae0fa854537.dc1 127.0.0.1 +49:20.349 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3123150c-dd36-5574-5010-dae0fa854537 127.0.0.1 +2022-12-04T05:49:20.349Z [INFO] agent.router: Initializing LAN area manager +49:20.349 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24154 network=udp +49:20.349 [INFO] TestAgent.server: Adding LAN server: server="Node-3123150c-dd36-5574-5010-dae0fa854537 (Addr: tcp/127.0.0.1:24159) (DC: dc1)" +49:20.349 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3123150c-dd36-5574-5010-dae0fa854537.dc1 area=wan +49:20.349 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24154 network=tcp +49:20.349 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24155 network=tcp +49:20.349 [INFO] TestAgent: started state syncer +49:20.349 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24160 network=tcp +49:20.404 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.404 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24159 [Candidate]" term=2 +49:20.404 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.404 [DEBUG] TestAgent.server.raft: vote granted: from=3123150c-dd36-5574-5010-dae0fa854537 term=2 tally=1 +49:20.404 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.404 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24159 [Leader]" +49:20.405 [INFO] TestAgent.server: cluster leadership acquired +49:20.405 [INFO] TestAgent.server: New leader elected: payload=Node-3123150c-dd36-5574-5010-dae0fa854537 +49:20.405 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24159 +49:20.405 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.407 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.407 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.407 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.407 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.407 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.407 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3123150c-dd36-5574-5010-dae0fa854537 +49:20.407 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3123150c-dd36-5574-5010-dae0fa854537 +49:20.407 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.778 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.779 [INFO] TestAgent: Synced node info +49:21.020 [WARN] TestAgent.server.rpc: Attempting to apply large raft entry: size_in_bytes=4719032 +49:21.062 [INFO] TestAgent: Requesting shutdown +49:21.062 [INFO] TestAgent.server: shutting down server +49:21.062 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.062 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.062 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.062 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.062 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.062 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.062 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.062 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.062 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.062 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.062 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:21.062Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.062Z [INFO] agent.router.manager: shutting down +49:21.063 [INFO] TestAgent: consul server down +49:21.063 [INFO] TestAgent: shutdown complete +49:21.063 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24154 network=tcp +49:21.063 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24154 network=udp +49:21.063 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24155 network=tcp +49:21.563 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.563 [INFO] TestAgent: Endpoints down +=== RUN TestTxnEndpoint_Bad_Size_Net/allowed_kv_max_backward_compatible +2022-12-04T05:49:20.743Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.754Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.756 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6a61e056-1f60-5ef2-6cd1-999c7f43e027 Address:127.0.0.1:24201}]" +49:20.756 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24201 [Follower]" leader= +49:20.757 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6a61e056-1f60-5ef2-6cd1-999c7f43e027.dc1 127.0.0.1 +49:20.757 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6a61e056-1f60-5ef2-6cd1-999c7f43e027 127.0.0.1 +2022-12-04T05:49:20.757Z [INFO] agent.router: Initializing LAN area manager +49:20.757 [INFO] TestAgent.server: Adding LAN server: server="Node-6a61e056-1f60-5ef2-6cd1-999c7f43e027 (Addr: tcp/127.0.0.1:24201) (DC: dc1)" +49:20.757 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6a61e056-1f60-5ef2-6cd1-999c7f43e027.dc1 area=wan +49:20.757 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24196 network=udp +49:20.757 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24196 network=tcp +49:20.758 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24197 network=tcp +49:20.758 [INFO] TestAgent: started state syncer +49:20.758 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24202 network=tcp +49:20.821 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.821 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24201 [Candidate]" term=2 +49:20.821 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.821 [DEBUG] TestAgent.server.raft: vote granted: from=6a61e056-1f60-5ef2-6cd1-999c7f43e027 term=2 tally=1 +49:20.821 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.821 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24201 [Leader]" +49:20.821 [INFO] TestAgent.server: cluster leadership acquired +49:20.821 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24201 +49:20.821 [INFO] TestAgent.server: New leader elected: payload=Node-6a61e056-1f60-5ef2-6cd1-999c7f43e027 +49:20.822 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.823 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.823 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.823 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.823 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.823 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.823 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6a61e056-1f60-5ef2-6cd1-999c7f43e027 +49:20.823 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6a61e056-1f60-5ef2-6cd1-999c7f43e027 +49:20.824 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.916 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.916 [INFO] TestAgent: Synced node info +49:20.916 [DEBUG] TestAgent: Node info in sync +49:21.114 [INFO] TestAgent: Requesting shutdown +49:21.114 [INFO] TestAgent.server: shutting down server +49:21.114 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.114 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.114 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.114 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.114 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.114 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.114 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.114 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.114 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.114 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.115 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:21.115Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.115Z [INFO] agent.router.manager: shutting down +49:21.115 [INFO] TestAgent: consul server down +49:21.115 [INFO] TestAgent: shutdown complete +49:21.115 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24196 network=tcp +49:21.115 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24196 network=udp +49:21.115 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24197 network=tcp +49:21.615 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.615 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthIngressServiceNodes (0.88s) + --- PASS: TestHealthIngressServiceNodes/associated_service (0.00s) + --- PASS: TestHealthIngressServiceNodes/non-associated_service (0.00s) === CONT TestHealthServiceNodes_DistanceSort -2021-10-31T23:13:05.998Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.052Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.055 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8c3a7a73-7237-fcad-6a48-b024c3121813 Address:127.0.0.1:16330}]" -13:06.056 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8c3a7a73-7237-fcad-6a48-b024c3121813.dc1 127.0.0.1 -13:06.056 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8c3a7a73-7237-fcad-6a48-b024c3121813 127.0.0.1 -2021-10-31T23:13:06.056Z [INFO] agent.router: Initializing LAN area manager -13:06.057 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16325 network=udp -13:06.057 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16330 [Follower]" leader= -13:06.059 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8c3a7a73-7237-fcad-6a48-b024c3121813.dc1 area=wan -13:06.113 [INFO] TestAgent.server: Adding LAN server: server="Node-8c3a7a73-7237-fcad-6a48-b024c3121813 (Addr: tcp/127.0.0.1:16330) (DC: dc1)" -13:06.118 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16325 network=tcp -13:06.125 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.125 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16330 [Candidate]" term=2 -13:06.126 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.126 [DEBUG] TestAgent.server.raft: vote granted: from=8c3a7a73-7237-fcad-6a48-b024c3121813 term=2 tally=1 -13:06.126 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.126 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16330 [Leader]" -13:06.126 [INFO] TestAgent.server: cluster leadership acquired -13:06.126 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16330 -13:06.127 [INFO] TestAgent.server: New leader elected: payload=Node-8c3a7a73-7237-fcad-6a48-b024c3121813 -13:06.129 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16326 network=tcp -13:06.130 [INFO] TestAgent: started state syncer -13:06.130 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16331 network=tcp -13:06.130 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/key from= latency=112.292µs -13:06.130 [INFO] TestAgent: Requesting shutdown -13:06.130 [INFO] TestAgent.server: shutting down server -13:06.130 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:06.130 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:06.130Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:06.130Z [INFO] agent.router.manager: shutting down -13:06.130 [WARN] TestAgent: Syncing node info failed.: error="raft is already shutdown" -13:06.130 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="raft is already shutdown" -13:06.138 [INFO] TestAgent: consul server down -13:06.138 [INFO] TestAgent: shutdown complete -13:06.138 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16325 network=tcp -13:06.138 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16325 network=udp -13:06.138 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16326 network=tcp -13:06.139 [ERROR] TestAgent.server: failed to establish leadership: error="error configuring provider: leadership lost while committing log" -13:06.140 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=0 retry_limit=3 error="raft is already shutdown" -13:06.140 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=1 retry_limit=3 error="raft is already shutdown" -13:06.140 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=2 retry_limit=3 error="raft is already shutdown" -13:06.140 [ERROR] TestAgent.server: failed to transfer leadership: error="failed to transfer leadership in 3 attempts" -13:06.647 [INFO] TestAgent: Waiting for endpoints to shut down -13:06.647 [INFO] TestAgent: Endpoints down ---- PASS: TestContentTypeIsJSON (0.76s) -=== CONT TestHealthServiceNodes_Filter -2021-10-31T23:13:05.823Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:05.903Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:05.922 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8c053565-f270-f537-c8bd-fa217be8f06e Address:127.0.0.1:16316}]" -13:05.923 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8c053565-f270-f537-c8bd-fa217be8f06e.dc1 127.0.0.1 -13:05.923 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16316 [Follower]" leader= -13:05.923 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8c053565-f270-f537-c8bd-fa217be8f06e 127.0.0.1 -2021-10-31T23:13:05.923Z [INFO] agent.router: Initializing LAN area manager -13:05.923 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8c053565-f270-f537-c8bd-fa217be8f06e.dc1 area=wan -13:05.923 [INFO] TestAgent.server: Adding LAN server: server="Node-8c053565-f270-f537-c8bd-fa217be8f06e (Addr: tcp/127.0.0.1:16316) (DC: dc1)" -13:05.923 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16311 network=udp -13:05.924 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16311 network=tcp -13:05.924 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16312 network=tcp -13:05.924 [INFO] TestAgent: started state syncer -13:05.924 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16317 network=tcp -13:05.969 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:05.969 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16316 [Candidate]" term=2 -13:05.977 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:05.977 [DEBUG] TestAgent.server.raft: vote granted: from=8c053565-f270-f537-c8bd-fa217be8f06e term=2 tally=1 -13:05.977 [INFO] TestAgent.server.raft: election won: tally=1 -13:05.977 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16316 [Leader]" -13:05.978 [INFO] TestAgent.server: cluster leadership acquired -13:05.978 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16316 -13:05.979 [INFO] TestAgent.server: New leader elected: payload=Node-8c053565-f270-f537-c8bd-fa217be8f06e -13:05.990 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.991 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.991 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.991 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.991 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.991 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.991 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8c053565-f270-f537-c8bd-fa217be8f06e -13:05.991 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8c053565-f270-f537-c8bd-fa217be8f06e -13:05.993 [INFO] TestAgent.server: federation state anti-entropy synced -13:06.040 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:06.041 [INFO] TestAgent: Synced node info -13:06.041 [DEBUG] TestAgent: Node info in sync -13:06.250 [INFO] TestAgent: Requesting shutdown -13:06.250 [INFO] TestAgent.server: shutting down server -13:06.250 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:06.250 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:06.250 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:06.250 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:06.250 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:06.250 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:06.250 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:06.250 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:06.250 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:06.250 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:06.250 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:06.251Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:06.251Z [INFO] agent.router.manager: shutting down -13:06.251 [INFO] TestAgent: consul server down -13:06.251 [INFO] TestAgent: shutdown complete -13:06.251 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16311 network=tcp -13:06.251 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16311 network=udp -13:06.251 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16312 network=tcp -13:06.760 [INFO] TestAgent: Waiting for endpoints to shut down -13:06.778 [INFO] TestAgent: Endpoints down ---- PASS: TestACLResolution (1.03s) -=== CONT TestHealthServiceNodes_NodeMetaFilter -2021-10-31T23:13:06.044Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.137Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.139 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:46183fc3-562c-2417-44fc-f7b037103dab Address:127.0.0.1:16344}]" -13:06.140 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-46183fc3-562c-2417-44fc-f7b037103dab.dc1 127.0.0.1 -13:06.140 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16344 [Follower]" leader= -13:06.140 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-46183fc3-562c-2417-44fc-f7b037103dab 127.0.0.1 -2021-10-31T23:13:06.140Z [INFO] agent.router: Initializing LAN area manager -13:06.140 [INFO] TestAgent.server: Adding LAN server: server="Node-46183fc3-562c-2417-44fc-f7b037103dab (Addr: tcp/127.0.0.1:16344) (DC: dc1)" -13:06.140 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-46183fc3-562c-2417-44fc-f7b037103dab.dc1 area=wan -13:06.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16339 network=udp -13:06.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16339 network=tcp -13:06.141 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16340 network=tcp -13:06.141 [INFO] TestAgent: started state syncer -13:06.141 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16345 network=tcp -13:06.177 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.177 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16344 [Candidate]" term=2 -13:06.178 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.178 [DEBUG] TestAgent.server.raft: vote granted: from=46183fc3-562c-2417-44fc-f7b037103dab term=2 tally=1 -13:06.178 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.178 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16344 [Leader]" -13:06.178 [INFO] TestAgent.server: cluster leadership acquired -13:06.178 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16344 -13:06.179 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:06.180 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:06.180 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:06.180 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:06.180 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:06.180 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.180 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-46183fc3-562c-2417-44fc-f7b037103dab -13:06.180 [INFO] TestAgent.server: member joined, marking health alive: member=Node-46183fc3-562c-2417-44fc-f7b037103dab -13:06.181 [INFO] TestAgent.server: New leader elected: payload=Node-46183fc3-562c-2417-44fc-f7b037103dab -13:06.181 [INFO] TestAgent.server: federation state anti-entropy synced -13:06.302 [DEBUG] TestAgent.http: Request finished: method=GET url=/ui from= latency=2.58µs -13:06.302 [INFO] TestAgent: Requesting shutdown -13:06.302 [INFO] TestAgent.server: shutting down server -13:06.302 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:06.302 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:06.302 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:06.302 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:06.302 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:06.302 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:06.302 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:06.302 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:06.302 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:06.302 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:06.302 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:06.302 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:06.302Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:06.302Z [INFO] agent.router.manager: shutting down -13:06.302 [INFO] TestAgent: consul server down -13:06.302 [INFO] TestAgent: shutdown complete -13:06.302 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16339 network=tcp -13:06.302 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16339 network=udp -13:06.302 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16340 network=tcp -13:06.805 [INFO] TestAgent: Waiting for endpoints to shut down -13:06.805 [INFO] TestAgent: Endpoints down ---- PASS: TestUIResponseHeaders (0.82s) -=== CONT TestHealthServiceNodes -2021-10-31T23:13:05.845Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.043Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.109 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3476202d-b0fb-f84c-d0ab-ee6e371d7fd3 Address:127.0.0.1:16309}]" -13:06.110 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3476202d-b0fb-f84c-d0ab-ee6e371d7fd3.dc1 127.0.0.1 -13:06.111 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3476202d-b0fb-f84c-d0ab-ee6e371d7fd3 127.0.0.1 -2021-10-31T23:13:06.111Z [INFO] agent.router: Initializing LAN area manager -13:06.111 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16304 network=udp -13:06.111 [INFO] TestAgent.server: Adding LAN server: server="Node-3476202d-b0fb-f84c-d0ab-ee6e371d7fd3 (Addr: tcp/127.0.0.1:16309) (DC: dc1)" -13:06.111 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16304 network=tcp -13:06.112 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16305 network=tcp -13:06.112 [INFO] TestAgent: started state syncer -13:06.112 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16310 network=tcp -13:06.112 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16309 [Follower]" leader= -13:06.112 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3476202d-b0fb-f84c-d0ab-ee6e371d7fd3.dc1 area=wan -13:06.177 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.177 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16309 [Candidate]" term=2 -13:06.178 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.178 [DEBUG] TestAgent.server.raft: vote granted: from=3476202d-b0fb-f84c-d0ab-ee6e371d7fd3 term=2 tally=1 -13:06.178 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.178 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16309 [Leader]" -13:06.178 [INFO] TestAgent.server: cluster leadership acquired -13:06.178 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16309 -13:06.179 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:06.180 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:06.180 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:06.180 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:06.180 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:06.180 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.180 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3476202d-b0fb-f84c-d0ab-ee6e371d7fd3 -13:06.180 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3476202d-b0fb-f84c-d0ab-ee6e371d7fd3 -13:06.180 [INFO] TestAgent.server: New leader elected: payload=Node-3476202d-b0fb-f84c-d0ab-ee6e371d7fd3 -13:06.180 [INFO] TestAgent.server: federation state anti-entropy synced -13:06.338 [INFO] TestAgent: Requesting shutdown -13:06.338 [INFO] TestAgent.server: shutting down server -13:06.338 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:06.339 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:06.339 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:06.339 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:06.339 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:06.339 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:06.339 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:06.339 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:06.339 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:06.339 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:06.339 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:06.339 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:06.339Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:06.339Z [INFO] agent.router.manager: shutting down -13:06.339 [INFO] TestAgent: consul server down -13:06.339 [INFO] TestAgent: shutdown complete -13:06.339 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16304 network=tcp -13:06.339 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16304 network=udp -13:06.339 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16305 network=tcp -13:06.845 [INFO] TestAgent: Waiting for endpoints to shut down -13:06.845 [INFO] TestAgent: Endpoints down ---- PASS: TestEnableWebUI (1.10s) -=== CONT TestHTTPServer_UnixSocket_FileExists -[WARN] freeport: 10 out of 66 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:05.946Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:05.969Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.010 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e65ca369-ca94-6c5f-be74-ab2c36611c03 Address:127.0.0.1:16337}]" -13:06.011 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16337 [Follower]" leader= -13:06.011 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e65ca369-ca94-6c5f-be74-ab2c36611c03.dc1 127.0.0.1 -13:06.012 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e65ca369-ca94-6c5f-be74-ab2c36611c03 127.0.0.1 -2021-10-31T23:13:06.012Z [INFO] agent.router: Initializing LAN area manager -13:06.012 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16332 network=udp -13:06.012 [INFO] TestAgent.server: Adding LAN server: server="Node-e65ca369-ca94-6c5f-be74-ab2c36611c03 (Addr: tcp/127.0.0.1:16337) (DC: dc1)" -13:06.012 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e65ca369-ca94-6c5f-be74-ab2c36611c03.dc1 area=wan -13:06.013 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16332 network=tcp -13:06.013 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16333 network=tcp -13:06.013 [INFO] TestAgent: started state syncer -13:06.013 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16338 network=tcp -13:06.113 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.113 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16337 [Candidate]" term=2 -13:06.113 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.113 [DEBUG] TestAgent.server.raft: vote granted: from=e65ca369-ca94-6c5f-be74-ab2c36611c03 term=2 tally=1 -13:06.113 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.113 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16337 [Leader]" -13:06.113 [INFO] TestAgent.server: cluster leadership acquired -13:06.117 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16337 -13:06.117 [INFO] TestAgent.server: New leader elected: payload=Node-e65ca369-ca94-6c5f-be74-ab2c36611c03 -13:06.138 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:06.139 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:06.139 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:06.139 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:06.139 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:06.139 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.139 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e65ca369-ca94-6c5f-be74-ab2c36611c03 -13:06.139 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e65ca369-ca94-6c5f-be74-ab2c36611c03 -13:06.140 [INFO] TestAgent.server: federation state anti-entropy synced -13:06.302 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:06.302 [INFO] TestAgent: Synced node info -13:06.303 [DEBUG] TestAgent: Node info in sync -13:06.362 [INFO] TestAgent: Requesting shutdown -13:06.362 [INFO] TestAgent.server: shutting down server -13:06.362 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:06.362 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:06.362 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:06.362 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:06.362 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:06.362 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:06.362 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:06.362 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:06.362 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:06.362 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:06.362 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:06.362Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:06.362Z [INFO] agent.router.manager: shutting down -13:06.363 [INFO] TestAgent: consul server down -13:06.363 [INFO] TestAgent: shutdown complete -13:06.363 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16332 network=tcp -13:06.363 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16332 network=udp -13:06.363 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16333 network=tcp -13:06.865 [INFO] TestAgent: Waiting for endpoints to shut down -13:06.865 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_Get (2.18s) - --- PASS: TestPreparedQuery_Get/#00 (1.19s) - --- PASS: TestPreparedQuery_Get/#01 (0.98s) -=== CONT TestHealthServiceChecks_DistanceSort -2021-10-31T23:13:05.791Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.013Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.016 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f3a9eb75-9992-c70f-7203-4b7817a969b4 Address:127.0.0.1:16302}]" -13:06.033 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16302 [Follower]" leader= -13:06.038 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f3a9eb75-9992-c70f-7203-4b7817a969b4.dc1 127.0.0.1 -13:06.039 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f3a9eb75-9992-c70f-7203-4b7817a969b4 127.0.0.1 -2021-10-31T23:13:06.039Z [INFO] agent.router: Initializing LAN area manager -13:06.039 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16297 network=udp -13:06.039 [INFO] TestAgent.server: Adding LAN server: server="Node-f3a9eb75-9992-c70f-7203-4b7817a969b4 (Addr: tcp/127.0.0.1:16302) (DC: dc1)" -13:06.040 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16297 network=tcp -13:06.040 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f3a9eb75-9992-c70f-7203-4b7817a969b4.dc1 area=wan -13:06.041 [INFO] TestAgent: Started HTTPS server: address=127.0.0.1:16299 network=tcp -13:06.041 [INFO] TestAgent: started state syncer -13:06.041 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16303 network=tcp -13:06.081 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.082 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16302 [Candidate]" term=2 -13:06.113 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.113 [DEBUG] TestAgent.server.raft: vote granted: from=f3a9eb75-9992-c70f-7203-4b7817a969b4 term=2 tally=1 -13:06.113 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.113 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16302 [Leader]" -13:06.118 [INFO] TestAgent.server: cluster leadership acquired -13:06.118 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16302 -13:06.118 [INFO] TestAgent.server: New leader elected: payload=Node-f3a9eb75-9992-c70f-7203-4b7817a969b4 -13:06.138 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:06.141 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:06.141 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:06.141 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:06.141 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:06.141 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.141 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f3a9eb75-9992-c70f-7203-4b7817a969b4 -13:06.141 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f3a9eb75-9992-c70f-7203-4b7817a969b4 -13:06.141 [INFO] TestAgent.server: federation state anti-entropy synced -13:06.406 [INFO] TestAgent: Requesting shutdown -13:06.406 [INFO] TestAgent.server: shutting down server -13:06.406 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:06.406 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:06.406 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:06.406 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:06.406 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:06.406 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:06.406 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:06.406 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:06.406 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:06.406 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:06.406 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:06.406 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:06.406Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:06.406Z [INFO] agent.router.manager: shutting down -13:06.406 [INFO] TestAgent: consul server down -13:06.406 [INFO] TestAgent: shutdown complete -13:06.406 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16297 network=tcp -13:06.407 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16297 network=udp -13:06.407 [INFO] TestAgent: Stopping server: protocol=HTTPS address=127.0.0.1:16299 network=tcp -13:06.925 [INFO] TestAgent: Waiting for endpoints to shut down -13:06.925 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPServer_HandshakeTimeout (1.18s) -=== CONT TestAgentKeyring_ACL -2021/10/31 23:13:06 http: TLS handshake error from 127.0.0.1:43580: read tcp 127.0.0.1:16369->127.0.0.1:43580: use of closed network connection -2021/10/31 23:13:06 http: TLS handshake error from 127.0.0.1:43576: EOF -2021/10/31 23:13:06 http: TLS handshake error from 127.0.0.1:43582: EOF -2021/10/31 23:13:06 http: TLS handshake error from 127.0.0.1:43578: EOF -2021-10-31T23:13:06.426Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.457Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.459 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:60a1e194-fdf4-5d23-6326-733bb4b1d6e5 Address:127.0.0.1:16351}]" -13:06.460 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-60a1e194-fdf4-5d23-6326-733bb4b1d6e5.dc1 127.0.0.1 -13:06.460 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16351 [Follower]" leader= -13:06.460 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-60a1e194-fdf4-5d23-6326-733bb4b1d6e5 127.0.0.1 -2021-10-31T23:13:06.460Z [INFO] agent.router: Initializing LAN area manager -13:06.460 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16346 network=udp -13:06.461 [INFO] TestAgent.server: Adding LAN server: server="Node-60a1e194-fdf4-5d23-6326-733bb4b1d6e5 (Addr: tcp/127.0.0.1:16351) (DC: dc1)" -13:06.461 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-60a1e194-fdf4-5d23-6326-733bb4b1d6e5.dc1 area=wan -13:06.461 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16346 network=tcp -13:06.461 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16347 network=tcp -13:06.461 [INFO] TestAgent: started state syncer -13:06.462 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16352 network=tcp -13:06.497 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.497 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16351 [Candidate]" term=2 -13:06.498 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.498 [DEBUG] TestAgent.server.raft: vote granted: from=60a1e194-fdf4-5d23-6326-733bb4b1d6e5 term=2 tally=1 -13:06.498 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.498 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16351 [Leader]" -13:06.498 [INFO] TestAgent.server: cluster leadership acquired -13:06.498 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16351 -13:06.498 [INFO] TestAgent.server: New leader elected: payload=Node-60a1e194-fdf4-5d23-6326-733bb4b1d6e5 -13:06.499 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:06.500 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:06.500 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:06.500 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:06.500 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:06.500 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.500 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-60a1e194-fdf4-5d23-6326-733bb4b1d6e5 -13:06.500 [INFO] TestAgent.server: member joined, marking health alive: member=Node-60a1e194-fdf4-5d23-6326-733bb4b1d6e5 -13:06.501 [INFO] TestAgent.server: federation state anti-entropy synced -13:06.530 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:06.530 [INFO] TestAgent: Synced node info -13:06.545 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:06.546 [INFO] TestAgent: Requesting shutdown -13:06.546 [INFO] TestAgent.server: shutting down server -13:06.546 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:06.546 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:06.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:06.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:06.546 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:06.546 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:06.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:06.546 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:06.546 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:06.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:06.546 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:06.546Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:06.546Z [INFO] agent.router.manager: shutting down -13:06.546 [INFO] TestAgent: consul server down -13:06.546 [INFO] TestAgent: shutdown complete -13:06.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16346 network=tcp -13:06.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16346 network=udp -13:06.546 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16347 network=tcp -13:07.049 [INFO] TestAgent: Waiting for endpoints to shut down -13:07.049 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_Explain (2.36s) - --- PASS: TestPreparedQuery_Explain/#00 (0.94s) - --- PASS: TestPreparedQuery_Explain/#01 (0.77s) - --- PASS: TestPreparedQuery_Explain/#02 (0.65s) -=== CONT TestHTTPServer_UnixSocket -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:20.715Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.726Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.727 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8db42b76-aec2-cc3c-d5e1-f0b15ce275ef Address:127.0.0.1:24194}]" +49:20.727 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24194 [Follower]" leader= +49:20.728 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8db42b76-aec2-cc3c-d5e1-f0b15ce275ef.dc1 127.0.0.1 +49:20.728 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8db42b76-aec2-cc3c-d5e1-f0b15ce275ef 127.0.0.1 +2022-12-04T05:49:20.728Z [INFO] agent.router: Initializing LAN area manager +49:20.729 [INFO] TestAgent.server: Adding LAN server: server="Node-8db42b76-aec2-cc3c-d5e1-f0b15ce275ef (Addr: tcp/127.0.0.1:24194) (DC: dc1)" +49:20.729 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8db42b76-aec2-cc3c-d5e1-f0b15ce275ef.dc1 area=wan +49:20.729 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24189 network=udp +49:20.729 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24189 network=tcp +49:20.729 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24190 network=tcp +49:20.729 [INFO] TestAgent: started state syncer +49:20.729 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24195 network=tcp +49:20.789 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.789 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24194 [Candidate]" term=2 +49:20.790 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.790 [DEBUG] TestAgent.server.raft: vote granted: from=8db42b76-aec2-cc3c-d5e1-f0b15ce275ef term=2 tally=1 +49:20.790 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.790 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24194 [Leader]" +49:20.790 [INFO] TestAgent.server: cluster leadership acquired +49:20.790 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24194 +49:20.790 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.791 [INFO] TestAgent.server: New leader elected: payload=Node-8db42b76-aec2-cc3c-d5e1-f0b15ce275ef +49:20.792 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.792 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.792 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.792 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.792 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.792 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8db42b76-aec2-cc3c-d5e1-f0b15ce275ef +49:20.792 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8db42b76-aec2-cc3c-d5e1-f0b15ce275ef +49:20.792 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.999 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.999 [INFO] TestAgent: Synced node info +49:20.999 [DEBUG] TestAgent: Node info in sync +49:21.138 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/self from= error="Endpoint is blocked by agent configuration" +49:21.138 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from= latency=1.955µs +49:21.138 [INFO] TestAgent: Requesting shutdown +49:21.138 [INFO] TestAgent.server: shutting down server +49:21.138 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.138 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.138 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.138 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.138 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.138 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.138 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.138 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.138 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.138 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.139 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:21.139Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.139Z [INFO] agent.router.manager: shutting down +49:21.139 [INFO] TestAgent: consul server down +49:21.139 [INFO] TestAgent: shutdown complete +49:21.139 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24189 network=tcp +49:21.139 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24189 network=udp +49:21.139 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24190 network=tcp +49:21.640 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.640 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPAPI_BlockEndpoints (0.93s) +=== CONT TestKVSEndpoint_PUT_ConflictingFlags === RUN TestHealthServiceNodes_PassingFilter/bc_no_query_value === RUN TestHealthServiceNodes_PassingFilter/passing_true === RUN TestHealthServiceNodes_PassingFilter/passing_false === RUN TestHealthServiceNodes_PassingFilter/passing_bad -2021-10-31T23:13:06.452Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.497Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.511 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f3a24c58-439f-453f-a120-a72d80614f8f Address:127.0.0.1:16358}]" -13:06.511 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f3a24c58-439f-453f-a120-a72d80614f8f.dc1 127.0.0.1 -13:06.512 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f3a24c58-439f-453f-a120-a72d80614f8f 127.0.0.1 -2021-10-31T23:13:06.512Z [INFO] agent.router: Initializing LAN area manager -13:06.512 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16353 network=udp -13:06.512 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16358 [Follower]" leader= -13:06.512 [INFO] TestAgent.server: Adding LAN server: server="Node-f3a24c58-439f-453f-a120-a72d80614f8f (Addr: tcp/127.0.0.1:16358) (DC: dc1)" -13:06.513 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f3a24c58-439f-453f-a120-a72d80614f8f.dc1 area=wan -13:06.513 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16353 network=tcp -13:06.513 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16354 network=tcp -13:06.513 [INFO] TestAgent: started state syncer -13:06.513 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16359 network=tcp -13:06.577 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.577 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16358 [Candidate]" term=2 -13:06.578 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.578 [DEBUG] TestAgent.server.raft: vote granted: from=f3a24c58-439f-453f-a120-a72d80614f8f term=2 tally=1 -13:06.578 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.578 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16358 [Leader]" -13:06.578 [INFO] TestAgent.server: cluster leadership acquired -13:06.578 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16358 -13:06.579 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:06.580 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:06.580 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:06.580 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:06.580 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:06.580 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.580 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f3a24c58-439f-453f-a120-a72d80614f8f -13:06.580 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f3a24c58-439f-453f-a120-a72d80614f8f -13:06.581 [INFO] TestAgent.server: New leader elected: payload=Node-f3a24c58-439f-453f-a120-a72d80614f8f -13:06.581 [INFO] TestAgent.server: federation state anti-entropy synced -13:06.674 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from= latency=2.523µs -13:06.674 [INFO] TestAgent: Requesting shutdown -13:06.674 [INFO] TestAgent.server: shutting down server -13:06.674 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:06.674 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:06.674 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:06.674 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:06.674 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:06.674 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:06.674 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:06.674 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:06.674 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:06.674 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:06.674 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2021-10-31T23:13:06.674Z [INFO] agent.router.manager: shutting down -13:06.674 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:06.674Z [INFO] agent.router.manager: shutting down -13:06.675 [INFO] TestAgent: consul server down -13:06.675 [INFO] TestAgent: shutdown complete -13:06.675 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16353 network=tcp -13:06.675 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16353 network=udp -13:06.675 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16354 network=tcp -13:07.177 [INFO] TestAgent: Waiting for endpoints to shut down -13:07.177 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPAPIResponseHeaders (0.76s) -=== CONT TestAgent_InitKeyring ---- PASS: TestAgent_InitKeyring (0.00s) -=== CONT TestHealthServiceChecks_Filtering -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:06.720Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.792Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.802 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8b700d1f-4a0d-da31-fd41-7ea6aa3855f2 Address:127.0.0.1:16400}]" -13:06.803 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8b700d1f-4a0d-da31-fd41-7ea6aa3855f2.dc1 127.0.0.1 -13:06.803 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8b700d1f-4a0d-da31-fd41-7ea6aa3855f2 127.0.0.1 -2021-10-31T23:13:06.803Z [INFO] agent.router: Initializing LAN area manager -13:06.803 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16395 network=udp -13:06.803 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16400 [Follower]" leader= -13:06.804 [INFO] TestAgent.server: Adding LAN server: server="Node-8b700d1f-4a0d-da31-fd41-7ea6aa3855f2 (Addr: tcp/127.0.0.1:16400) (DC: dc1)" -13:06.804 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8b700d1f-4a0d-da31-fd41-7ea6aa3855f2.dc1 area=wan -13:06.804 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16395 network=tcp -13:06.804 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16396 network=tcp -13:06.805 [INFO] TestAgent: started state syncer -13:06.805 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16401 network=tcp -13:06.845 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.846 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16400 [Candidate]" term=2 -13:06.846 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.846 [DEBUG] TestAgent.server.raft: vote granted: from=8b700d1f-4a0d-da31-fd41-7ea6aa3855f2 term=2 tally=1 -13:06.846 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.846 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16400 [Leader]" -13:06.846 [INFO] TestAgent.server: cluster leadership acquired -13:06.846 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16400 -13:06.847 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:06.848 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:06.848 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:06.848 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:06.848 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:06.848 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.848 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8b700d1f-4a0d-da31-fd41-7ea6aa3855f2 -13:06.848 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8b700d1f-4a0d-da31-fd41-7ea6aa3855f2 -13:06.849 [INFO] TestAgent.server: New leader elected: payload=Node-8b700d1f-4a0d-da31-fd41-7ea6aa3855f2 -13:06.849 [INFO] TestAgent.server: federation state anti-entropy synced -13:06.898 [INFO] TestAgent: Requesting shutdown -13:06.898 [INFO] TestAgent.server: shutting down server -13:06.898 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:06.898 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:06.898 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:06.898 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:06.898 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:06.898 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:06.898 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:06.898 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:06.898 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:06.898 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:06.898 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:06.898 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:06.898Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:06.898Z [INFO] agent.router.manager: shutting down -13:06.898 [INFO] TestAgent: consul server down -13:06.898 [INFO] TestAgent: shutdown complete -13:06.898 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16395 network=tcp -13:06.898 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16395 network=udp -13:06.898 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16396 network=tcp -13:07.402 [INFO] TestAgent: Waiting for endpoints to shut down -13:07.402 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceNodes_DistanceSort (0.76s) -=== CONT TestFilterNonPassing ---- PASS: TestFilterNonPassing (0.00s) -=== CONT TestHealthServiceChecks_NodeMetaFilter -2021-10-31T23:13:06.717Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.800Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.801 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3fbdac91-3ca6-b781-aa86-2adef536c2da Address:127.0.0.1:16407}]" -13:06.801 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16407 [Follower]" leader= -13:06.807 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3fbdac91-3ca6-b781-aa86-2adef536c2da.dc1 127.0.0.1 -13:06.808 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3fbdac91-3ca6-b781-aa86-2adef536c2da 127.0.0.1 -2021-10-31T23:13:06.808Z [INFO] agent.router: Initializing LAN area manager -13:06.808 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16402 network=udp -13:06.809 [INFO] TestAgent.server: Adding LAN server: server="Node-3fbdac91-3ca6-b781-aa86-2adef536c2da (Addr: tcp/127.0.0.1:16407) (DC: dc1)" -13:06.809 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3fbdac91-3ca6-b781-aa86-2adef536c2da.dc1 area=wan -13:06.809 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16402 network=tcp -13:06.809 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16403 network=tcp -13:06.809 [INFO] TestAgent: started state syncer -13:06.809 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16408 network=tcp -13:06.869 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.870 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16407 [Candidate]" term=2 -13:06.870 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.870 [DEBUG] TestAgent.server.raft: vote granted: from=3fbdac91-3ca6-b781-aa86-2adef536c2da term=2 tally=1 -13:06.870 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.870 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16407 [Leader]" -13:06.870 [INFO] TestAgent.server: cluster leadership acquired -13:06.870 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16407 -13:06.871 [INFO] TestAgent.server: New leader elected: payload=Node-3fbdac91-3ca6-b781-aa86-2adef536c2da -13:06.871 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:06.872 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:06.872 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:06.872 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:06.872 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:06.877 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.877 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3fbdac91-3ca6-b781-aa86-2adef536c2da -13:06.877 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3fbdac91-3ca6-b781-aa86-2adef536c2da -13:06.936 [INFO] TestAgent: Requesting shutdown -13:06.936 [INFO] TestAgent.server: shutting down server -13:06.936 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:06.936 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:06.936 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:06.936 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:06.936 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:06.937 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:06.940 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:06.940 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:06.940 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" -13:06.940 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:06.940 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:06.940 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2021-10-31T23:13:06.941Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:06.941Z [INFO] agent.router.manager: shutting down -13:06.941 [INFO] TestAgent: consul server down -13:06.941 [INFO] TestAgent: shutdown complete -13:06.941 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16402 network=tcp -13:06.941 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16402 network=udp -13:06.941 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16403 network=tcp -13:07.441 [INFO] TestAgent: Waiting for endpoints to shut down -13:07.441 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceNodes_Filter (0.81s) -=== CONT TestHealthConnectServiceNodes_PassingFilter -2021-10-31T23:13:06.540Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.583Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.585 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:da0e77a8-4c25-cd69-f94e-8217c7f5cd86 Address:127.0.0.1:16372}]" -13:06.585 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16372 [Follower]" leader= -13:06.594 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-da0e77a8-4c25-cd69-f94e-8217c7f5cd86.dc1 127.0.0.1 -13:06.594 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-da0e77a8-4c25-cd69-f94e-8217c7f5cd86 127.0.0.1 -2021-10-31T23:13:06.594Z [INFO] agent.router: Initializing LAN area manager -13:06.595 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16367 network=udp -13:06.595 [INFO] TestAgent.server: Adding LAN server: server="Node-da0e77a8-4c25-cd69-f94e-8217c7f5cd86 (Addr: tcp/127.0.0.1:16372) (DC: dc1)" -13:06.595 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-da0e77a8-4c25-cd69-f94e-8217c7f5cd86.dc1 area=wan -13:06.595 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16367 network=tcp -13:06.596 [INFO] TestAgent: Started HTTPS server: address=127.0.0.1:16369 network=tcp -13:06.596 [INFO] TestAgent: started state syncer -13:06.596 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16373 network=tcp -13:06.646 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.646 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16372 [Candidate]" term=2 -13:06.646 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.646 [DEBUG] TestAgent.server.raft: vote granted: from=da0e77a8-4c25-cd69-f94e-8217c7f5cd86 term=2 tally=1 -13:06.646 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.646 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16372 [Leader]" -13:06.646 [INFO] TestAgent.server: cluster leadership acquired -13:06.647 [INFO] TestAgent.server: New leader elected: payload=Node-da0e77a8-4c25-cd69-f94e-8217c7f5cd86 -13:06.647 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16372 -13:06.648 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:06.649 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:06.649 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:06.649 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:06.649 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:06.649 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.649 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-da0e77a8-4c25-cd69-f94e-8217c7f5cd86 -13:06.649 [INFO] TestAgent.server: member joined, marking health alive: member=Node-da0e77a8-4c25-cd69-f94e-8217c7f5cd86 -13:06.650 [INFO] TestAgent.server: federation state anti-entropy synced -13:06.674 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:06.679 [INFO] TestAgent: Synced node info -13:06.679 [DEBUG] TestAgent: Node info in sync -13:06.984 [INFO] TestAgent: Requesting shutdown -13:06.984 [INFO] TestAgent.server: shutting down server -13:06.984 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:06.984 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:06.984 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:06.984Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:06.984Z [INFO] agent.router.manager: shutting down -13:06.985 [INFO] TestAgent: consul server down -13:06.985 [INFO] TestAgent: shutdown complete -13:06.985 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16367 network=tcp -13:06.985 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16367 network=udp -13:06.985 [INFO] TestAgent: Stopping server: protocol=HTTPS address=127.0.0.1:16369 network=tcp -13:07.485 [INFO] TestAgent: Waiting for endpoints to shut down -13:07.485 [INFO] TestAgent: Endpoints down ---- PASS: TestRPC_HTTPSMaxConnsPerClient (1.76s) - --- PASS: TestRPC_HTTPSMaxConnsPerClient/HTTP (0.78s) - --- PASS: TestRPC_HTTPSMaxConnsPerClient/HTTPS (0.98s) -=== CONT TestAgent_InmemKeyrings -=== RUN TestAgent_InmemKeyrings/no_keys -2021-10-31T23:13:06.824Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.856Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.862 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:afbd487f-334e-ab0b-c48e-3e23fac30c91 Address:127.0.0.1:16428}]" -13:06.862 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16428 [Follower]" leader= -13:06.863 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-afbd487f-334e-ab0b-c48e-3e23fac30c91.dc1 127.0.0.1 -13:06.863 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-afbd487f-334e-ab0b-c48e-3e23fac30c91 127.0.0.1 -2021-10-31T23:13:06.863Z [INFO] agent.router: Initializing LAN area manager -13:06.864 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16423 network=udp -13:06.864 [INFO] TestAgent.server: Adding LAN server: server="Node-afbd487f-334e-ab0b-c48e-3e23fac30c91 (Addr: tcp/127.0.0.1:16428) (DC: dc1)" -13:06.864 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-afbd487f-334e-ab0b-c48e-3e23fac30c91.dc1 area=wan -13:06.864 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16423 network=tcp -13:06.864 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16424 network=tcp -13:06.865 [INFO] TestAgent: started state syncer -13:06.865 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16429 network=tcp -13:06.927 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.927 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16428 [Candidate]" term=2 -13:06.927 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.927 [DEBUG] TestAgent.server.raft: vote granted: from=afbd487f-334e-ab0b-c48e-3e23fac30c91 term=2 tally=1 -13:06.927 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.927 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16428 [Leader]" -13:06.927 [INFO] TestAgent.server: cluster leadership acquired -13:06.928 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16428 -13:06.929 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:06.930 [INFO] TestAgent.server: New leader elected: payload=Node-afbd487f-334e-ab0b-c48e-3e23fac30c91 -13:06.931 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:06.931 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:06.931 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:06.931 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:06.931 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.931 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-afbd487f-334e-ab0b-c48e-3e23fac30c91 -13:06.932 [INFO] TestAgent.server: member joined, marking health alive: member=Node-afbd487f-334e-ab0b-c48e-3e23fac30c91 -13:06.933 [INFO] TestAgent.server: federation state anti-entropy synced -13:07.014 [INFO] TestAgent: Requesting shutdown -13:07.014 [INFO] TestAgent.server: shutting down server -13:07.014 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:07.014 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:07.014 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:07.014 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:07.014 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:07.016 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:07.016 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:07.016 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:07.016 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:07.016 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:07.021 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:07.026 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:07.026Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:07.026Z [INFO] agent.router.manager: shutting down -13:07.026 [INFO] TestAgent: consul server down -13:07.026 [INFO] TestAgent: shutdown complete -13:07.026 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16423 network=tcp -13:07.026 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16423 network=udp -13:07.046 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16424 network=tcp -13:07.547 [INFO] TestAgent: Waiting for endpoints to shut down -13:07.547 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceNodes (0.74s) -=== CONT TestHealthConnectServiceNodes_Filter +2022-12-04T05:49:21.009Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.019Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.021 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8717d6b3-1bf2-c786-c94a-a40a94330500 Address:127.0.0.1:24236}]" +49:21.021 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24236 [Follower]" leader= +49:21.021 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8717d6b3-1bf2-c786-c94a-a40a94330500.dc1 127.0.0.1 +49:21.021 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8717d6b3-1bf2-c786-c94a-a40a94330500 127.0.0.1 +2022-12-04T05:49:21.022Z [INFO] agent.router: Initializing LAN area manager +49:21.022 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24231 network=udp +49:21.022 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8717d6b3-1bf2-c786-c94a-a40a94330500.dc1 area=wan +49:21.022 [INFO] TestAgent.server: Adding LAN server: server="Node-8717d6b3-1bf2-c786-c94a-a40a94330500 (Addr: tcp/127.0.0.1:24236) (DC: dc1)" +49:21.022 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24231 network=tcp +49:21.022 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24232 network=tcp +49:21.023 [INFO] TestAgent: started state syncer +49:21.023 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24237 network=tcp +49:21.079 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.079 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24236 [Candidate]" term=2 +49:21.079 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.079 [DEBUG] TestAgent.server.raft: vote granted: from=8717d6b3-1bf2-c786-c94a-a40a94330500 term=2 tally=1 +49:21.079 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.079 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24236 [Leader]" +49:21.079 [INFO] TestAgent.server: cluster leadership acquired +49:21.079 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24236 +49:21.080 [INFO] TestAgent.server: New leader elected: payload=Node-8717d6b3-1bf2-c786-c94a-a40a94330500 +49:21.080 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.082 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.082 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.082 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.082 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.082 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.082 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8717d6b3-1bf2-c786-c94a-a40a94330500 +49:21.082 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8717d6b3-1bf2-c786-c94a-a40a94330500 +49:21.082 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.119 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.120 [INFO] TestAgent: Synced node info +49:21.120 [DEBUG] TestAgent: Node info in sync +49:21.246 [INFO] TestAgent.server.serf.wan: serf: Received install-key query +49:21.246 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-8717d6b3-1bf2-c786-c94a-a40a94330500.dc1 +49:21.246 [INFO] TestAgent.server.serf.lan: serf: Received install-key query +49:21.247 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-8717d6b3-1bf2-c786-c94a-a40a94330500 +49:21.247 [INFO] TestAgent.server.serf.wan: serf: Received use-key query +49:21.247 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-8717d6b3-1bf2-c786-c94a-a40a94330500.dc1 +49:21.247 [INFO] TestAgent.server.serf.lan: serf: Received use-key query +49:21.247 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-8717d6b3-1bf2-c786-c94a-a40a94330500 +49:21.248 [INFO] TestAgent.server.serf.wan: serf: Received remove-key query +49:21.248 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-8717d6b3-1bf2-c786-c94a-a40a94330500.dc1 +49:21.248 [INFO] TestAgent.server.serf.lan: serf: Received remove-key query +49:21.248 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-8717d6b3-1bf2-c786-c94a-a40a94330500 +49:21.248 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +49:21.248 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-8717d6b3-1bf2-c786-c94a-a40a94330500.dc1 +49:21.248 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +49:21.249 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-8717d6b3-1bf2-c786-c94a-a40a94330500 +49:21.249 [INFO] TestAgent: Requesting shutdown +49:21.249 [INFO] TestAgent.server: shutting down server +49:21.249 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.249 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.249 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.249 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.249 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.249 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.249 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.249 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.249 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.249 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:21.249 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:21.249Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.249Z [INFO] agent.router.manager: shutting down +49:21.249 [INFO] TestAgent: consul server down +49:21.249 [INFO] TestAgent: shutdown complete +49:21.249 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24231 network=tcp +49:21.249 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24231 network=udp +49:21.249 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24232 network=tcp +49:21.752 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.752 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_KeyringUse (0.76s) +=== CONT TestKVSEndpoint_GET_Raw +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:21.197Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.206Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.208 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0a239c5c-cf43-90e8-212b-42c9fc4a7284 Address:127.0.0.1:24257}]" +49:21.208 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24257 [Follower]" leader= +49:21.208 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0a239c5c-cf43-90e8-212b-42c9fc4a7284.dc1 127.0.0.1 +49:21.208 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0a239c5c-cf43-90e8-212b-42c9fc4a7284 127.0.0.1 +2022-12-04T05:49:21.208Z [INFO] agent.router: Initializing LAN area manager +49:21.209 [INFO] TestAgent.server: Adding LAN server: server="Node-0a239c5c-cf43-90e8-212b-42c9fc4a7284 (Addr: tcp/127.0.0.1:24257) (DC: dc1)" +49:21.209 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0a239c5c-cf43-90e8-212b-42c9fc4a7284.dc1 area=wan +49:21.209 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24252 network=udp +49:21.209 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24252 network=tcp +49:21.209 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24253 network=tcp +49:21.209 [INFO] TestAgent: started state syncer +49:21.209 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24258 network=tcp +49:21.256 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.256 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24257 [Candidate]" term=2 +49:21.256 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.256 [DEBUG] TestAgent.server.raft: vote granted: from=0a239c5c-cf43-90e8-212b-42c9fc4a7284 term=2 tally=1 +49:21.256 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.256 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24257 [Leader]" +49:21.256 [INFO] TestAgent.server: cluster leadership acquired +49:21.256 [INFO] TestAgent.server: New leader elected: payload=Node-0a239c5c-cf43-90e8-212b-42c9fc4a7284 +49:21.256 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24257 +49:21.257 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.258 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.258 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.258 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.258 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.258 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.258 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a239c5c-cf43-90e8-212b-42c9fc4a7284 +49:21.258 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0a239c5c-cf43-90e8-212b-42c9fc4a7284 +49:21.258 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.322 [INFO] TestAgent: Requesting shutdown +49:21.322 [INFO] TestAgent.server: shutting down server +49:21.322 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.322 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.322 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.322 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.322 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.322 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.322 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.322 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:21.322 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.322 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.322 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.323 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:21.323Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.323Z [INFO] agent.router.manager: shutting down +49:21.323 [INFO] TestAgent: consul server down +49:21.323 [INFO] TestAgent: shutdown complete +49:21.323 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24252 network=tcp +49:21.323 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24252 network=udp +49:21.323 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24253 network=tcp +49:21.824 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.824 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceNodes_CheckType (0.64s) +=== CONT TestKVSEndpoint_AcquireRelease +2022-12-04T05:49:21.279Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:21.279Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.290Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:21.291Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.292 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5a32dece-c804-0afe-5680-8438c52256da Address:127.0.0.1:24271}]" +49:21.292 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24271 [Follower]" leader= +49:21.293 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5a32dece-c804-0afe-5680-8438c52256da.dc1 127.0.0.1 +49:21.293 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5a32dece-c804-0afe-5680-8438c52256da 127.0.0.1 +2022-12-04T05:49:21.293Z [INFO] agent.router: Initializing LAN area manager +49:21.294 [INFO] TestAgent.server: Adding LAN server: server="Node-5a32dece-c804-0afe-5680-8438c52256da (Addr: tcp/127.0.0.1:24271) (DC: dc1)" +49:21.294 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24266 network=udp +49:21.294 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5a32dece-c804-0afe-5680-8438c52256da.dc1 area=wan +49:21.294 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24266 network=tcp +49:21.294 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24267 network=tcp +49:21.295 [INFO] TestAgent: started state syncer +49:21.295 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24272 network=tcp +49:21.344 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:21.358 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.358 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24271 [Candidate]" term=2 +49:21.358 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.358 [DEBUG] TestAgent.server.raft: vote granted: from=5a32dece-c804-0afe-5680-8438c52256da term=2 tally=1 +49:21.358 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.358 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24271 [Leader]" +49:21.358 [INFO] TestAgent.server: cluster leadership acquired +49:21.358 [INFO] TestAgent.server: initializing acls +49:21.359 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:21.359 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:21.359 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:21.359 [INFO] TestAgent.server: New leader elected: payload=Node-5a32dece-c804-0afe-5680-8438c52256da +49:21.359 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:21.359 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:21.359 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:21.360 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5a32dece-c804-0afe-5680-8438c52256da +49:21.360 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5a32dece-c804-0afe-5680-8438c52256da.dc1 +49:21.360 [INFO] TestAgent.server: Updating LAN server: server="Node-5a32dece-c804-0afe-5680-8438c52256da (Addr: tcp/127.0.0.1:24271) (DC: dc1)" +49:21.360 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5a32dece-c804-0afe-5680-8438c52256da.dc1 area=wan +49:21.360 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.361 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.361 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.361 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.361 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.361 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.361 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a32dece-c804-0afe-5680-8438c52256da +49:21.362 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5a32dece-c804-0afe-5680-8438c52256da +49:21.362 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a32dece-c804-0afe-5680-8438c52256da +49:21.362 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.420 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.421 [INFO] TestAgent: Synced node info +49:21.421 [DEBUG] TestAgent: Node info in sync +49:21.482 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5a32dece-c804-0afe-5680-8438c52256da +49:21.482 [INFO] TestAgent: Requesting shutdown +49:21.482 [INFO] TestAgent.server: shutting down server +49:21.482 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.482 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:21.482 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:21.482 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.482 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.482 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.482 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.482 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.482 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.482 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.482 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:21.482 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:21.482 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.482 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.482 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:21.482Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.482Z [INFO] agent.router.manager: shutting down +49:21.482 [INFO] TestAgent: consul server down +49:21.482 [INFO] TestAgent: shutdown complete +49:21.482 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24266 network=tcp +49:21.482 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24266 network=udp +49:21.482 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24267 network=tcp +49:21.984 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.985 [INFO] TestAgent: Endpoints down +--- PASS: TestSnapshot_Options (4.25s) + --- PASS: TestSnapshot_Options/GET (0.91s) + --- PASS: TestSnapshot_Options/GET#01 (0.79s) + --- PASS: TestSnapshot_Options/GET#02 (0.87s) + --- PASS: TestSnapshot_Options/PUT (0.12s) + --- PASS: TestSnapshot_Options/PUT#01 (0.85s) + --- PASS: TestSnapshot_Options/PUT#02 (0.72s) +=== CONT TestHealthServiceNodes_Filter +2022-12-04T05:49:21.100Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.113Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.115 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8927a012-4873-ab90-271f-484f44a44502 Address:127.0.0.1:24250}]" +49:21.115 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24250 [Follower]" leader= +49:21.116 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8927a012-4873-ab90-271f-484f44a44502.dc1 127.0.0.1 +49:21.117 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8927a012-4873-ab90-271f-484f44a44502 127.0.0.1 +2022-12-04T05:49:21.117Z [INFO] agent.router: Initializing LAN area manager +49:21.117 [INFO] TestAgent.server: Adding LAN server: server="Node-8927a012-4873-ab90-271f-484f44a44502 (Addr: tcp/127.0.0.1:24250) (DC: dc1)" +49:21.117 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8927a012-4873-ab90-271f-484f44a44502.dc1 area=wan +49:21.117 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24245 network=udp +49:21.117 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24245 network=tcp +49:21.117 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24246 network=tcp +49:21.117 [INFO] TestAgent: started state syncer +49:21.118 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24251 network=tcp +49:21.155 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.155 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24250 [Candidate]" term=2 +49:21.155 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.155 [DEBUG] TestAgent.server.raft: vote granted: from=8927a012-4873-ab90-271f-484f44a44502 term=2 tally=1 +49:21.155 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.155 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24250 [Leader]" +49:21.155 [INFO] TestAgent.server: cluster leadership acquired +49:21.155 [INFO] TestAgent.server: New leader elected: payload=Node-8927a012-4873-ab90-271f-484f44a44502 +49:21.155 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24250 +49:21.156 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.157 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.157 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.157 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.157 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.157 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.157 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8927a012-4873-ab90-271f-484f44a44502 +49:21.157 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8927a012-4873-ab90-271f-484f44a44502 +49:21.158 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.471 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.472 [INFO] TestAgent: Synced node info +49:21.499 [INFO] TestAgent.server.serf.wan: serf: Received install-key query +49:21.500 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-8927a012-4873-ab90-271f-484f44a44502.dc1 +49:21.500 [INFO] TestAgent.server.serf.lan: serf: Received install-key query +49:21.501 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-8927a012-4873-ab90-271f-484f44a44502 +49:21.501 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +49:21.501 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-8927a012-4873-ab90-271f-484f44a44502.dc1 +49:21.501 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +49:21.501 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-8927a012-4873-ab90-271f-484f44a44502 +49:21.501 [INFO] TestAgent.server.serf.wan: serf: Received remove-key query +49:21.502 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-8927a012-4873-ab90-271f-484f44a44502.dc1 +49:21.502 [INFO] TestAgent.server.serf.lan: serf: Received remove-key query +49:21.502 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-8927a012-4873-ab90-271f-484f44a44502 +49:21.502 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +49:21.503 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-8927a012-4873-ab90-271f-484f44a44502.dc1 +49:21.503 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +49:21.503 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-8927a012-4873-ab90-271f-484f44a44502 +49:21.503 [INFO] TestAgent: Requesting shutdown +49:21.503 [INFO] TestAgent.server: shutting down server +49:21.503 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.503 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.503 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.503 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.503 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.503 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.503 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.503 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.503 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.503 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.503 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:21.503Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.503Z [INFO] agent.router.manager: shutting down +49:21.503 [INFO] TestAgent: consul server down +49:21.503 [INFO] TestAgent: shutdown complete +49:21.503 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24245 network=tcp +49:21.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24245 network=udp +49:21.504 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24246 network=tcp +49:22.004 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.004 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_KeyringRemove (0.92s) +=== CONT TestHealthServiceChecks_DistanceSort +2022-12-04T05:49:21.410Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.422Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.424 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3c136283-706b-37c1-b721-ee88665fca99 Address:127.0.0.1:24292}]" +49:21.425 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3c136283-706b-37c1-b721-ee88665fca99.dc1 127.0.0.1 +49:21.425 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3c136283-706b-37c1-b721-ee88665fca99 127.0.0.1 +2022-12-04T05:49:21.425Z [INFO] agent.router: Initializing LAN area manager +49:21.425 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24287 network=udp +49:21.425 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24292 [Follower]" leader= +49:21.426 [INFO] TestAgent.server: Adding LAN server: server="Node-3c136283-706b-37c1-b721-ee88665fca99 (Addr: tcp/127.0.0.1:24292) (DC: dc1)" +49:21.426 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3c136283-706b-37c1-b721-ee88665fca99.dc1 area=wan +49:21.426 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24287 network=tcp +49:21.426 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24288 network=tcp +49:21.426 [INFO] TestAgent: started state syncer +49:21.426 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24293 network=tcp +49:21.485 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.485 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24292 [Candidate]" term=2 +49:21.485 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.485 [DEBUG] TestAgent.server.raft: vote granted: from=3c136283-706b-37c1-b721-ee88665fca99 term=2 tally=1 +49:21.485 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.485 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24292 [Leader]" +49:21.485 [INFO] TestAgent.server: cluster leadership acquired +49:21.485 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24292 +49:21.486 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.486 [INFO] TestAgent.server: New leader elected: payload=Node-3c136283-706b-37c1-b721-ee88665fca99 +49:21.487 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.487 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.487 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.487 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.487 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.487 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3c136283-706b-37c1-b721-ee88665fca99 +49:21.487 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3c136283-706b-37c1-b721-ee88665fca99 +49:21.488 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.528 [INFO] TestAgent.server.serf.wan: serf: Received install-key query +49:21.528 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-3c136283-706b-37c1-b721-ee88665fca99.dc1 +49:21.528 [INFO] TestAgent.server.serf.lan: serf: Received install-key query +49:21.529 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-3c136283-706b-37c1-b721-ee88665fca99 +49:21.529 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +49:21.529 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-3c136283-706b-37c1-b721-ee88665fca99.dc1 +49:21.529 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +49:21.529 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-3c136283-706b-37c1-b721-ee88665fca99 +49:21.529 [INFO] TestAgent: Requesting shutdown +49:21.529 [INFO] TestAgent.server: shutting down server +49:21.529 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.529 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.529 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.529 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.529 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.530 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:21.530 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.530 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.530 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:21.530 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.530 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.530 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:49:21.530Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.530Z [INFO] agent.router.manager: shutting down +49:21.530 [INFO] TestAgent: consul server down +49:21.530 [INFO] TestAgent: shutdown complete +49:21.530 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24287 network=tcp +49:21.530 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24287 network=udp +49:21.530 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24288 network=tcp +49:22.030 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.030 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_KeyringInstall (0.63s) +=== CONT TestHealthServiceNodes_NodeMetaFilter [WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:06.695Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.776Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.795 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f39cab4d-e045-ef33-1232-da799e4145ec Address:127.0.0.1:16379}]" -13:06.796 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f39cab4d-e045-ef33-1232-da799e4145ec.dc1 127.0.0.1 -13:06.797 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f39cab4d-e045-ef33-1232-da799e4145ec 127.0.0.1 -2021-10-31T23:13:06.797Z [INFO] agent.router: Initializing LAN area manager -13:06.797 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16379 [Follower]" leader= -13:06.798 [INFO] TestAgent.server: Adding LAN server: server="Node-f39cab4d-e045-ef33-1232-da799e4145ec (Addr: tcp/127.0.0.1:16379) (DC: dc1)" -13:06.798 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f39cab4d-e045-ef33-1232-da799e4145ec.dc1 area=wan -13:06.798 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16374 network=udp -13:06.798 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16374 network=tcp -13:06.799 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16375 network=tcp -13:06.799 [INFO] TestAgent: started state syncer -13:06.799 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16380 network=tcp -13:06.850 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.850 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16379 [Candidate]" term=2 -13:06.854 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.854 [DEBUG] TestAgent.server.raft: vote granted: from=f39cab4d-e045-ef33-1232-da799e4145ec term=2 tally=1 -13:06.854 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.854 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16379 [Leader]" -13:06.854 [INFO] TestAgent.server: cluster leadership acquired -13:06.854 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16379 -13:06.855 [INFO] TestAgent.server: New leader elected: payload=Node-f39cab4d-e045-ef33-1232-da799e4145ec -13:06.855 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:06.857 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:06.857 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:06.857 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:06.857 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:06.857 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.857 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f39cab4d-e045-ef33-1232-da799e4145ec -13:06.857 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f39cab4d-e045-ef33-1232-da799e4145ec -13:06.858 [INFO] TestAgent.server: federation state anti-entropy synced -13:06.923 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:06.926 [INFO] TestAgent: Synced node info -13:07.134 [ERROR] TestAgent.http: Request error: method=GET url=/v1/agent/self from= error="Endpoint is blocked by agent configuration" -13:07.134 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from= latency=3.496µs -13:07.134 [INFO] TestAgent: Requesting shutdown -13:07.134 [INFO] TestAgent.server: shutting down server -13:07.134 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:07.134 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:07.134 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:07.134 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:07.134 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:07.134 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:07.134 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:07.134 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:07.134 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:07.134 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:07.134 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:07.134Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:07.134Z [INFO] agent.router.manager: shutting down -13:07.134 [INFO] TestAgent: consul server down -13:07.134 [INFO] TestAgent: shutdown complete -13:07.134 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16374 network=tcp -13:07.135 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16374 network=udp -13:07.135 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16375 network=tcp -13:07.637 [INFO] TestAgent: Waiting for endpoints to shut down -13:07.637 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPAPI_BlockEndpoints (1.01s) -=== CONT TestHealthServiceChecks -2021-10-31T23:13:06.721Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.788Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.795 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef6a52fc-2877-791f-783a-017235d606d6 Address:127.0.0.1:16393}]" -13:06.796 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef6a52fc-2877-791f-783a-017235d606d6.dc1 127.0.0.1 -13:06.797 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef6a52fc-2877-791f-783a-017235d606d6 127.0.0.1 -2021-10-31T23:13:06.797Z [INFO] agent.router: Initializing LAN area manager -13:06.797 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16388 network=udp -13:06.797 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16393 [Follower]" leader= -13:06.797 [INFO] TestAgent.server: Adding LAN server: server="Node-ef6a52fc-2877-791f-783a-017235d606d6 (Addr: tcp/127.0.0.1:16393) (DC: dc1)" -13:06.797 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef6a52fc-2877-791f-783a-017235d606d6.dc1 area=wan -13:06.798 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16388 network=tcp -13:06.798 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16389 network=tcp -13:06.798 [INFO] TestAgent: started state syncer -13:06.798 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16394 network=tcp -13:06.861 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.861 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16393 [Candidate]" term=2 -13:06.862 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.862 [DEBUG] TestAgent.server.raft: vote granted: from=ef6a52fc-2877-791f-783a-017235d606d6 term=2 tally=1 -13:06.862 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.862 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16393 [Leader]" -13:06.862 [INFO] TestAgent.server: cluster leadership acquired -13:06.862 [INFO] TestAgent.server: New leader elected: payload=Node-ef6a52fc-2877-791f-783a-017235d606d6 -13:06.862 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16393 -13:06.863 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:06.864 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:06.864 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:06.864 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:06.864 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:06.864 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.864 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef6a52fc-2877-791f-783a-017235d606d6 -13:06.864 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef6a52fc-2877-791f-783a-017235d606d6 -13:06.865 [INFO] TestAgent.server: federation state anti-entropy synced -13:06.877 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:06.878 [INFO] TestAgent: Synced node info -13:07.171 [INFO] TestAgent: Requesting shutdown -13:07.171 [INFO] TestAgent.server: shutting down server -13:07.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:07.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:07.171 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:07.171 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:07.171 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:07.171 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:07.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:07.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:07.171 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:07.171 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:07.171 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:07.171Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:07.171Z [INFO] agent.router.manager: shutting down -13:07.171 [INFO] TestAgent: consul server down -13:07.171 [INFO] TestAgent: shutdown complete -13:07.171 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16388 network=tcp -13:07.171 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16388 network=udp -13:07.171 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16389 network=tcp -13:07.673 [INFO] TestAgent: Waiting for endpoints to shut down -13:07.673 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceNodes_PassingFilter (1.03s) +2022-12-04T05:49:21.220Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.232Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.234 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f6fbd029-a8cf-34dd-abc8-d1b4b970b451 Address:127.0.0.1:24264}]" +49:21.234 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24264 [Follower]" leader= +49:21.235 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f6fbd029-a8cf-34dd-abc8-d1b4b970b451.dc1 127.0.0.1 +49:21.235 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f6fbd029-a8cf-34dd-abc8-d1b4b970b451 127.0.0.1 +2022-12-04T05:49:21.235Z [INFO] agent.router: Initializing LAN area manager +49:21.236 [INFO] TestAgent.server: Adding LAN server: server="Node-f6fbd029-a8cf-34dd-abc8-d1b4b970b451 (Addr: tcp/127.0.0.1:24264) (DC: dc1)" +49:21.236 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f6fbd029-a8cf-34dd-abc8-d1b4b970b451.dc1 area=wan +49:21.236 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24259 network=udp +49:21.236 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24259 network=tcp +49:21.236 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24260 network=tcp +49:21.236 [INFO] TestAgent: started state syncer +49:21.236 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24265 network=tcp +49:21.302 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.302 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24264 [Candidate]" term=2 +49:21.302 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.302 [DEBUG] TestAgent.server.raft: vote granted: from=f6fbd029-a8cf-34dd-abc8-d1b4b970b451 term=2 tally=1 +49:21.302 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.302 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24264 [Leader]" +49:21.302 [INFO] TestAgent.server: cluster leadership acquired +49:21.302 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24264 +49:21.303 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.304 [INFO] TestAgent.server: New leader elected: payload=Node-f6fbd029-a8cf-34dd-abc8-d1b4b970b451 +49:21.304 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.305 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.305 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.305 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.305 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.305 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f6fbd029-a8cf-34dd-abc8-d1b4b970b451 +49:21.305 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f6fbd029-a8cf-34dd-abc8-d1b4b970b451 +49:21.305 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.420 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.420 [INFO] TestAgent: Synced node info +49:21.420 [DEBUG] TestAgent: Node info in sync +49:21.577 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +49:21.578 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-f6fbd029-a8cf-34dd-abc8-d1b4b970b451 +49:21.578 [INFO] TestAgent: Requesting shutdown +49:21.578 [INFO] TestAgent.server: shutting down server +49:21.578 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.578 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.578 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.578 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.578 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.578 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.578 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.578 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.578 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.578 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.578 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:21.578Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.578Z [INFO] agent.router.manager: shutting down +49:21.578 [INFO] TestAgent: consul server down +49:21.578 [INFO] TestAgent: shutdown complete +49:21.578 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24259 network=tcp +49:21.578 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24259 network=udp +49:21.578 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24260 network=tcp +49:22.078 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.078 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_KeyringListLocalOnly (0.87s) +=== CONT TestHealthServiceNodes +2022-12-04T05:49:20.840Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.850Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.851 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:91674e0a-d3e0-d3a4-4a8a-3e929cff6a81 Address:127.0.0.1:24229}]" +49:20.851 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24229 [Follower]" leader= +49:20.851 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-91674e0a-d3e0-d3a4-4a8a-3e929cff6a81.dc1 127.0.0.1 +49:20.852 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-91674e0a-d3e0-d3a4-4a8a-3e929cff6a81 127.0.0.1 +2022-12-04T05:49:20.852Z [INFO] agent.router: Initializing LAN area manager +49:20.852 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-91674e0a-d3e0-d3a4-4a8a-3e929cff6a81.dc1 area=wan +49:20.852 [INFO] TestAgent.server: Adding LAN server: server="Node-91674e0a-d3e0-d3a4-4a8a-3e929cff6a81 (Addr: tcp/127.0.0.1:24229) (DC: dc1)" +49:20.852 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24224 network=udp +49:20.852 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24224 network=tcp +49:20.853 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24225 network=tcp +49:20.853 [INFO] TestAgent: started state syncer +49:20.853 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24230 network=tcp +49:20.888 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.888 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24229 [Candidate]" term=2 +49:20.889 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.889 [DEBUG] TestAgent.server.raft: vote granted: from=91674e0a-d3e0-d3a4-4a8a-3e929cff6a81 term=2 tally=1 +49:20.889 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.889 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24229 [Leader]" +49:20.889 [INFO] TestAgent.server: cluster leadership acquired +49:20.889 [INFO] TestAgent.server: New leader elected: payload=Node-91674e0a-d3e0-d3a4-4a8a-3e929cff6a81 +49:20.889 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24229 +49:20.890 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.891 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.891 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.891 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.891 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.891 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.891 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-91674e0a-d3e0-d3a4-4a8a-3e929cff6a81 +49:20.891 [INFO] TestAgent.server: member joined, marking health alive: member=Node-91674e0a-d3e0-d3a4-4a8a-3e929cff6a81 +49:20.891 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.109 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.110 [INFO] TestAgent: Synced node info +49:21.110 [DEBUG] TestAgent: Node info in sync +49:21.155 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from= latency=1.606µs +49:21.155 [INFO] TestAgent: Requesting shutdown +49:21.155 [INFO] TestAgent.server: shutting down server +49:21.155 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.155 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.155 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.155 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.155 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.155 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.155 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.155 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.155 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.155 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.155 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:21.155Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.155Z [INFO] agent.router.manager: shutting down +49:21.155 [INFO] TestAgent: consul server down +49:21.155 [INFO] TestAgent: shutdown complete +49:21.155 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24224 network=tcp +49:21.155 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24224 network=udp +49:21.155 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24225 network=tcp +49:21.655 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.655 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:20.449Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.474Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.476 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:88ba92e2-3a0b-4851-bd63-5391b30a4915 Address:127.0.0.1:24173}]" +49:20.476 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24173 [Follower]" leader= +49:20.476 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-88ba92e2-3a0b-4851-bd63-5391b30a4915.dc1 127.0.0.1 +49:20.477 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-88ba92e2-3a0b-4851-bd63-5391b30a4915 127.0.0.1 +2022-12-04T05:49:20.477Z [INFO] agent.router: Initializing LAN area manager +49:20.477 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-88ba92e2-3a0b-4851-bd63-5391b30a4915.dc1 area=wan +49:20.477 [INFO] TestAgent.server: Adding LAN server: server="Node-88ba92e2-3a0b-4851-bd63-5391b30a4915 (Addr: tcp/127.0.0.1:24173) (DC: dc1)" +49:20.477 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24168 network=udp +49:20.477 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24168 network=tcp +49:20.477 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24169 network=tcp +49:20.477 [INFO] TestAgent: started state syncer +49:20.477 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24174 network=tcp +49:20.531 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.531 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24173 [Candidate]" term=2 +49:20.531 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.531 [DEBUG] TestAgent.server.raft: vote granted: from=88ba92e2-3a0b-4851-bd63-5391b30a4915 term=2 tally=1 +49:20.531 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.531 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24173 [Leader]" +49:20.531 [INFO] TestAgent.server: cluster leadership acquired +49:20.531 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24173 +49:20.531 [INFO] TestAgent.server: New leader elected: payload=Node-88ba92e2-3a0b-4851-bd63-5391b30a4915 +49:20.534 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.536 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.536 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.536 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.536 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.536 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.536 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-88ba92e2-3a0b-4851-bd63-5391b30a4915 +49:20.536 [INFO] TestAgent.server: member joined, marking health alive: member=Node-88ba92e2-3a0b-4851-bd63-5391b30a4915 +49:20.537 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.585 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.586 [INFO] TestAgent: Synced node info +49:20.586 [DEBUG] TestAgent: Node info in sync +49:20.828 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from= latency=2.152µs +49:21.655 [INFO] TestAgent: Requesting shutdown +49:21.655 [INFO] TestAgent.server: shutting down server +49:21.655 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.655 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.655 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.655 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.655 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.655 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.656 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.656 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.656 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.656 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.656 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:21.656Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.656Z [INFO] agent.router.manager: shutting down +49:21.656 [INFO] TestAgent: consul server down +49:21.656 [INFO] TestAgent: shutdown complete +49:21.656 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24168 network=tcp +49:21.656 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24168 network=udp +49:21.656 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24169 network=tcp +49:22.156 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.156 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPAPI_TranslateAddrHeader (1.72s) +=== CONT TestHealthServiceChecks_Filtering +2022-12-04T05:49:21.435Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.447Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.449 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0959a049-f9a7-791a-c93c-baed02b15dea Address:127.0.0.1:23293}]" +49:21.449 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23293 [Follower]" leader= +49:21.449 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0959a049-f9a7-791a-c93c-baed02b15dea.dc1 127.0.0.1 +49:21.450 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0959a049-f9a7-791a-c93c-baed02b15dea 127.0.0.1 +2022-12-04T05:49:21.450Z [INFO] agent.router: Initializing LAN area manager +49:21.450 [INFO] TestAgent.server: Adding LAN server: server="Node-0959a049-f9a7-791a-c93c-baed02b15dea (Addr: tcp/127.0.0.1:23293) (DC: dc1)" +49:21.450 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0959a049-f9a7-791a-c93c-baed02b15dea.dc1 area=wan +49:21.450 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23288 network=udp +49:21.450 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23288 network=tcp +49:21.450 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23289 network=tcp +49:21.451 [INFO] TestAgent: started state syncer +49:21.451 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23294 network=tcp +49:21.490 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.490 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23293 [Candidate]" term=2 +49:21.490 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.490 [DEBUG] TestAgent.server.raft: vote granted: from=0959a049-f9a7-791a-c93c-baed02b15dea term=2 tally=1 +49:21.490 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.490 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23293 [Leader]" +49:21.491 [INFO] TestAgent.server: cluster leadership acquired +49:21.491 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23293 +49:21.491 [INFO] TestAgent.server: New leader elected: payload=Node-0959a049-f9a7-791a-c93c-baed02b15dea +49:21.491 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.498 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.498 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.498 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.498 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.498 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.498 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0959a049-f9a7-791a-c93c-baed02b15dea +49:21.498 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0959a049-f9a7-791a-c93c-baed02b15dea +49:21.499 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.624 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.624 [INFO] TestAgent: Synced node info +49:21.624 [DEBUG] TestAgent: Node info in sync +49:21.669 [INFO] TestAgent: Requesting shutdown +49:21.669 [INFO] TestAgent.server: shutting down server +49:21.669 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.669 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.669 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.669 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.669 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.669 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.669 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.669 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.669 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.669 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.669 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:21.669Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.669Z [INFO] agent.router.manager: shutting down +49:21.669 [INFO] TestAgent: consul server down +49:21.669 [INFO] TestAgent: shutdown complete +49:21.669 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23288 network=tcp +49:21.669 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23288 network=udp +49:21.670 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23289 network=tcp +49:22.170 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.170 [INFO] TestAgent: Endpoints down +=== RUN TestOperator_RaftPeer/#01 +2022-12-04T05:49:21.318Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.330Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.332 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4372bd06-2404-f01b-40b4-cf73d961c6b7 Address:127.0.0.1:24278}]" +49:21.332 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24278 [Follower]" leader= +49:21.333 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4372bd06-2404-f01b-40b4-cf73d961c6b7.dc1 127.0.0.1 +49:21.333 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4372bd06-2404-f01b-40b4-cf73d961c6b7 127.0.0.1 +2022-12-04T05:49:21.333Z [INFO] agent.router: Initializing LAN area manager +49:21.333 [INFO] TestAgent.server: Adding LAN server: server="Node-4372bd06-2404-f01b-40b4-cf73d961c6b7 (Addr: tcp/127.0.0.1:24278) (DC: dc1)" +49:21.334 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4372bd06-2404-f01b-40b4-cf73d961c6b7.dc1 area=wan +49:21.334 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24273 network=tcp +49:21.334 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24273 network=udp +49:21.334 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24274 network=tcp +49:21.334 [INFO] TestAgent: started state syncer +49:21.334 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24279 network=tcp +49:21.396 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.396 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24278 [Candidate]" term=2 +49:21.397 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.397 [DEBUG] TestAgent.server.raft: vote granted: from=4372bd06-2404-f01b-40b4-cf73d961c6b7 term=2 tally=1 +49:21.397 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.397 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24278 [Leader]" +49:21.397 [INFO] TestAgent.server: cluster leadership acquired +49:21.397 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24278 +49:21.397 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.399 [INFO] TestAgent.server: New leader elected: payload=Node-4372bd06-2404-f01b-40b4-cf73d961c6b7 +49:21.399 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.399 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.399 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.399 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.399 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.399 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4372bd06-2404-f01b-40b4-cf73d961c6b7 +49:21.399 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4372bd06-2404-f01b-40b4-cf73d961c6b7 +49:21.400 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.509 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.510 [INFO] TestAgent: Synced node info +49:21.510 [DEBUG] TestAgent: Node info in sync +49:21.678 [INFO] TestAgent: Requesting shutdown +49:21.678 [INFO] TestAgent.server: shutting down server +49:21.678 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.678 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.678 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.678 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.678 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.678 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.678 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.678 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.678 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:21.678 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.678 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:49:21.679Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.679Z [INFO] agent.router.manager: shutting down +49:21.679 [INFO] TestAgent: consul server down +49:21.679 [INFO] TestAgent: shutdown complete +49:21.679 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24273 network=tcp +49:21.679 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24273 network=udp +49:21.679 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24274 network=tcp +49:22.179 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.179 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceNodes_PassingFilter (0.87s) --- PASS: TestHealthServiceNodes_PassingFilter/bc_no_query_value (0.00s) --- PASS: TestHealthServiceNodes_PassingFilter/passing_true (0.00s) --- PASS: TestHealthServiceNodes_PassingFilter/passing_false (0.00s) --- PASS: TestHealthServiceNodes_PassingFilter/passing_bad (0.00s) -=== CONT TestConfig_Apply_IngressGateway -2021-10-31T23:13:06.805Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.846Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.848 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a0ff7d33-3117-1dc4-dae8-a9f4dad38d98 Address:127.0.0.1:16421}]" -13:06.849 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a0ff7d33-3117-1dc4-dae8-a9f4dad38d98.dc1 127.0.0.1 -13:06.849 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16421 [Follower]" leader= -13:06.854 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a0ff7d33-3117-1dc4-dae8-a9f4dad38d98 127.0.0.1 -2021-10-31T23:13:06.854Z [INFO] agent.router: Initializing LAN area manager -13:06.854 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16416 network=udp -13:06.854 [INFO] TestAgent.server: Adding LAN server: server="Node-a0ff7d33-3117-1dc4-dae8-a9f4dad38d98 (Addr: tcp/127.0.0.1:16421) (DC: dc1)" -13:06.854 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a0ff7d33-3117-1dc4-dae8-a9f4dad38d98.dc1 area=wan -13:06.854 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16416 network=tcp -13:06.855 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16417 network=tcp -13:06.855 [INFO] TestAgent: started state syncer -13:06.855 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16422 network=tcp -13:06.901 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.902 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16421 [Candidate]" term=2 -13:06.902 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.902 [DEBUG] TestAgent.server.raft: vote granted: from=a0ff7d33-3117-1dc4-dae8-a9f4dad38d98 term=2 tally=1 -13:06.902 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.902 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16421 [Leader]" -13:06.902 [INFO] TestAgent.server: cluster leadership acquired -13:06.902 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16421 -13:06.903 [INFO] TestAgent.server: New leader elected: payload=Node-a0ff7d33-3117-1dc4-dae8-a9f4dad38d98 -13:06.925 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:06.938 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:06.938 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:06.938 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:06.938 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:06.938 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.938 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a0ff7d33-3117-1dc4-dae8-a9f4dad38d98 -13:06.938 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a0ff7d33-3117-1dc4-dae8-a9f4dad38d98 -13:06.939 [INFO] TestAgent.server: federation state anti-entropy synced -13:06.960 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:06.960 [INFO] TestAgent: Synced node info -13:06.960 [DEBUG] TestAgent: Node info in sync -13:07.230 [INFO] TestAgent: Requesting shutdown -13:07.231 [INFO] TestAgent.server: shutting down server -13:07.231 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:07.231 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:07.231 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:07.231 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:07.231 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:07.231 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:07.231 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:07.231 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:07.231 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:07.231 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:07.231 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:07.232Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:07.232Z [INFO] agent.router.manager: shutting down -13:07.232 [INFO] TestAgent: consul server down -13:07.232 [INFO] TestAgent: shutdown complete -13:07.232 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16416 network=tcp -13:07.232 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16416 network=udp -13:07.232 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16417 network=tcp -13:07.734 [INFO] TestAgent: Waiting for endpoints to shut down -13:07.734 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceNodes_NodeMetaFilter (0.96s) -=== CONT TestHealthIngressServiceNodes -=== RUN TestHealthConnectServiceNodes_PassingFilter/bc_no_query_value -=== RUN TestHealthConnectServiceNodes_PassingFilter/passing_true -=== RUN TestHealthConnectServiceNodes_PassingFilter/passing_false -=== RUN TestHealthConnectServiceNodes_PassingFilter/passing_bad -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:06.925Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:07.019Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:07.020 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:38b7667f-894e-21ab-acf1-bf7aa1aa9f7d Address:127.0.0.1:16442}]" -13:07.021 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-38b7667f-894e-21ab-acf1-bf7aa1aa9f7d.dc1 127.0.0.1 -13:07.021 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16442 [Follower]" leader= -13:07.026 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-38b7667f-894e-21ab-acf1-bf7aa1aa9f7d 127.0.0.1 -2021-10-31T23:13:07.026Z [INFO] agent.router: Initializing LAN area manager -13:07.026 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16437 network=udp -13:07.027 [INFO] TestAgent.server: Adding LAN server: server="Node-38b7667f-894e-21ab-acf1-bf7aa1aa9f7d (Addr: tcp/127.0.0.1:16442) (DC: dc1)" -13:07.027 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-38b7667f-894e-21ab-acf1-bf7aa1aa9f7d.dc1 area=wan -13:07.027 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16437 network=tcp -13:07.027 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16438 network=tcp -13:07.028 [INFO] TestAgent: started state syncer -13:07.028 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16443 network=tcp -13:07.086 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:07.086 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16442 [Candidate]" term=2 -13:07.086 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:07.086 [DEBUG] TestAgent.server.raft: vote granted: from=38b7667f-894e-21ab-acf1-bf7aa1aa9f7d term=2 tally=1 -13:07.086 [INFO] TestAgent.server.raft: election won: tally=1 -13:07.086 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16442 [Leader]" -13:07.086 [INFO] TestAgent.server: cluster leadership acquired -13:07.086 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16442 -13:07.087 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:07.088 [INFO] TestAgent.server: New leader elected: payload=Node-38b7667f-894e-21ab-acf1-bf7aa1aa9f7d -13:07.089 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:07.089 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:07.089 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:07.089 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:07.089 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:07.089 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-38b7667f-894e-21ab-acf1-bf7aa1aa9f7d -13:07.089 [INFO] TestAgent.server: member joined, marking health alive: member=Node-38b7667f-894e-21ab-acf1-bf7aa1aa9f7d -13:07.093 [INFO] TestAgent.server: federation state anti-entropy synced -13:07.190 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:07.190 [INFO] TestAgent: Synced node info -13:07.359 [INFO] TestAgent: Requesting shutdown -13:07.359 [INFO] TestAgent.server: shutting down server -13:07.359 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:07.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:07.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:07.359 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:07.359 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:07.359 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:07.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:07.359 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:07.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:07.359 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:07.360 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:07.360Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:07.360Z [INFO] agent.router.manager: shutting down -13:07.360 [INFO] TestAgent: consul server down -13:07.360 [INFO] TestAgent: shutdown complete -13:07.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16437 network=tcp -13:07.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16437 network=udp -13:07.360 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16438 network=tcp -13:07.861 [INFO] TestAgent: Waiting for endpoints to shut down -13:08.169 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceChecks_DistanceSort (1.30s) -=== CONT TestAgent_LoadKeyrings -=== RUN TestAgent_LoadKeyrings/no_keys -2021-10-31T23:13:06.987Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:06.987Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:07.050Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:07.050Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:07.052 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:95a3a75b-8576-33fb-4495-45238982e532 Address:127.0.0.1:16449}]" -13:07.053 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-95a3a75b-8576-33fb-4495-45238982e532.dc1 127.0.0.1 -13:07.053 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16449 [Follower]" leader= -13:07.066 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-95a3a75b-8576-33fb-4495-45238982e532 127.0.0.1 -2021-10-31T23:13:07.066Z [INFO] agent.router: Initializing LAN area manager -13:07.066 [INFO] TestAgent.server: Adding LAN server: server="Node-95a3a75b-8576-33fb-4495-45238982e532 (Addr: tcp/127.0.0.1:16449) (DC: dc1)" -13:07.066 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16444 network=udp -13:07.066 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-95a3a75b-8576-33fb-4495-45238982e532.dc1 area=wan -13:07.066 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16444 network=tcp -13:07.067 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16445 network=tcp -13:07.067 [INFO] TestAgent: started state syncer -13:07.067 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16450 network=tcp -13:07.116 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:07.121 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:07.121 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16449 [Candidate]" term=2 -13:07.122 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:07.122 [DEBUG] TestAgent.server.raft: vote granted: from=95a3a75b-8576-33fb-4495-45238982e532 term=2 tally=1 -13:07.122 [INFO] TestAgent.server.raft: election won: tally=1 -13:07.122 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16449 [Leader]" -13:07.122 [INFO] TestAgent.server: cluster leadership acquired -13:07.122 [INFO] TestAgent.server: initializing acls -13:07.122 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:07.122 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:07.123 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:07.123 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:07.123 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:07.123 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:07.123 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-95a3a75b-8576-33fb-4495-45238982e532 -13:07.123 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-95a3a75b-8576-33fb-4495-45238982e532.dc1 -13:07.124 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:07.125 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:07.125 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:07.125 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:07.125 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:07.125 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:07.126 [INFO] TestAgent.server: New leader elected: payload=Node-95a3a75b-8576-33fb-4495-45238982e532 -13:07.126 [INFO] TestAgent.server: Updating LAN server: server="Node-95a3a75b-8576-33fb-4495-45238982e532 (Addr: tcp/127.0.0.1:16449) (DC: dc1)" -13:07.126 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-95a3a75b-8576-33fb-4495-45238982e532.dc1 area=wan -13:07.127 [INFO] TestAgent.server: federation state anti-entropy synced -13:07.133 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-95a3a75b-8576-33fb-4495-45238982e532 -13:07.134 [INFO] TestAgent.server: member joined, marking health alive: member=Node-95a3a75b-8576-33fb-4495-45238982e532 -13:07.134 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-95a3a75b-8576-33fb-4495-45238982e532 -13:07.182 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:07.182 [INFO] TestAgent: Synced node info -13:07.182 [DEBUG] TestAgent: Node info in sync -13:07.350 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-95a3a75b-8576-33fb-4495-45238982e532 -13:07.351 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -13:07.357 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-95a3a75b-8576-33fb-4495-45238982e532.dc1 -13:07.357 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -13:07.357 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-95a3a75b-8576-33fb-4495-45238982e532 -13:07.357 [INFO] TestAgent.server.serf.wan: serf: Received install-key query -13:07.358 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-95a3a75b-8576-33fb-4495-45238982e532.dc1 -13:07.358 [INFO] TestAgent.server.serf.lan: serf: Received install-key query -13:07.359 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-95a3a75b-8576-33fb-4495-45238982e532 -13:07.359 [INFO] TestAgent.server.serf.wan: serf: Received use-key query -13:07.366 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-95a3a75b-8576-33fb-4495-45238982e532.dc1 -13:07.366 [INFO] TestAgent.server.serf.lan: serf: Received use-key query -13:07.370 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-95a3a75b-8576-33fb-4495-45238982e532 -13:07.370 [INFO] TestAgent.server.serf.wan: serf: Received remove-key query -13:07.370 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-95a3a75b-8576-33fb-4495-45238982e532.dc1 -13:07.370 [INFO] TestAgent.server.serf.lan: serf: Received remove-key query -13:07.371 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-95a3a75b-8576-33fb-4495-45238982e532 -13:07.371 [INFO] TestAgent: Requesting shutdown -13:07.371 [INFO] TestAgent.server: shutting down server -13:07.371 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:07.371 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:07.371 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:07.371 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:07.371 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:07.371 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:07.371 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:07.371 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:07.371 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:07.371 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:07.371 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:07.371 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:07.371 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:07.371 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:07.371 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:07.372Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:07.372Z [INFO] agent.router.manager: shutting down -13:07.372 [INFO] TestAgent: consul server down -13:07.372 [INFO] TestAgent: shutdown complete -13:07.372 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16444 network=tcp -13:07.372 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16444 network=udp -13:07.372 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16445 network=tcp -13:07.873 [INFO] TestAgent: Waiting for endpoints to shut down -13:08.169 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:06.879Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.925Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.938 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:223096a8-67ac-7b11-faeb-74e14d9ab73e Address:127.0.0.1:16435}]" -13:06.939 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-223096a8-67ac-7b11-faeb-74e14d9ab73e.dc1 127.0.0.1 -13:06.940 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-223096a8-67ac-7b11-faeb-74e14d9ab73e 127.0.0.1 -2021-10-31T23:13:06.940Z [INFO] agent.router: Initializing LAN area manager -13:06.940 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16435 [Follower]" leader= -13:06.941 [INFO] TestAgent.server: Adding LAN server: server="Node-223096a8-67ac-7b11-faeb-74e14d9ab73e (Addr: tcp/127.0.0.1:16435) (DC: dc1)" -13:06.941 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-223096a8-67ac-7b11-faeb-74e14d9ab73e.dc1 area=wan -13:06.941 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16430 network=udp -13:06.941 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16430 network=tcp -13:06.941 [WARN] TestAgent: Replacing socket: path=/tmp/consul-test/TestHTTPServer_UnixSocket_FileExists-consul233081807/test.sock -13:06.954 [INFO] TestAgent: Started HTTP server: address=/tmp/consul-test/TestHTTPServer_UnixSocket_FileExists-consul233081807/test.sock network=unix -13:06.954 [INFO] TestAgent: started state syncer -13:06.955 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16436 network=tcp -13:07.014 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:07.014 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16435 [Candidate]" term=2 -13:07.014 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:07.014 [DEBUG] TestAgent.server.raft: vote granted: from=223096a8-67ac-7b11-faeb-74e14d9ab73e term=2 tally=1 -13:07.014 [INFO] TestAgent.server.raft: election won: tally=1 -13:07.014 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16435 [Leader]" -13:07.014 [INFO] TestAgent.server: cluster leadership acquired -13:07.014 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16435 -13:07.015 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:07.016 [INFO] TestAgent.server: New leader elected: payload=Node-223096a8-67ac-7b11-faeb-74e14d9ab73e -13:07.022 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:07.022 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:07.022 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:07.022 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:07.022 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:07.022 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-223096a8-67ac-7b11-faeb-74e14d9ab73e -13:07.022 [INFO] TestAgent.server: member joined, marking health alive: member=Node-223096a8-67ac-7b11-faeb-74e14d9ab73e -13:07.023 [INFO] TestAgent.server: federation state anti-entropy synced -13:07.316 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:07.317 [INFO] TestAgent: Synced node info -13:07.334 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:07.334 [DEBUG] TestAgent: Node info in sync -13:07.334 [DEBUG] TestAgent: Node info in sync -13:07.379 [INFO] TestAgent: Requesting shutdown -13:07.379 [INFO] TestAgent.server: shutting down server -13:07.379 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:07.379 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:07.379 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:07.379 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:07.379 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:07.379 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:07.379 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:07.379 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:07.379 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:07.379 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:07.379 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:07.379Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:07.379Z [INFO] agent.router.manager: shutting down -13:07.379 [INFO] TestAgent: consul server down -13:07.379 [INFO] TestAgent: shutdown complete -13:07.379 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16430 network=tcp -13:07.379 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16430 network=udp -13:07.379 [INFO] TestAgent: Stopping server: protocol=HTTP address=/tmp/consul-test/TestHTTPServer_UnixSocket_FileExists-consul233081807/test.sock network=unix -13:07.882 [INFO] TestAgent: Waiting for endpoints to shut down -13:08.169 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentKeyring_ACL (1.25s) -=== CONT TestHealthConnectServiceNodes ---- PASS: TestHTTPServer_UnixSocket_FileExists (1.33s) -=== CONT TestIntentionsSpecificDelete_good -2021-10-31T23:13:07.092Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:07.131Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:07.149 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1807019e-648f-ccff-dfc7-fe7187040f0f Address:127.0.0.1:16456}]" -13:07.150 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1807019e-648f-ccff-dfc7-fe7187040f0f.dc1 127.0.0.1 -13:07.151 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16456 [Follower]" leader= -13:07.151 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1807019e-648f-ccff-dfc7-fe7187040f0f 127.0.0.1 -2021-10-31T23:13:07.151Z [INFO] agent.router: Initializing LAN area manager -13:07.151 [INFO] TestAgent.server: Adding LAN server: server="Node-1807019e-648f-ccff-dfc7-fe7187040f0f (Addr: tcp/127.0.0.1:16456) (DC: dc1)" -13:07.151 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1807019e-648f-ccff-dfc7-fe7187040f0f.dc1 area=wan -13:07.151 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16451 network=tcp -13:07.151 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16451 network=udp -13:07.152 [INFO] TestAgent: Started HTTP server: address=/tmp/consul-test/TestHTTPServer_UnixSocket-consul140359283/test.sock network=unix -13:07.152 [INFO] TestAgent: started state syncer -13:07.152 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16457 network=tcp -13:07.208 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:07.208 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16456 [Candidate]" term=2 -13:07.208 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:07.208 [DEBUG] TestAgent.server.raft: vote granted: from=1807019e-648f-ccff-dfc7-fe7187040f0f term=2 tally=1 -13:07.208 [INFO] TestAgent.server.raft: election won: tally=1 -13:07.208 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16456 [Leader]" -13:07.208 [INFO] TestAgent.server: cluster leadership acquired -13:07.208 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16456 -13:07.209 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:07.209 [INFO] TestAgent.server: New leader elected: payload=Node-1807019e-648f-ccff-dfc7-fe7187040f0f -13:07.211 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:07.211 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:07.211 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:07.211 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:07.211 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:07.211 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1807019e-648f-ccff-dfc7-fe7187040f0f -13:07.211 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1807019e-648f-ccff-dfc7-fe7187040f0f -13:07.211 [INFO] TestAgent.server: federation state anti-entropy synced -13:07.399 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=@ latency=1.72568ms -13:07.400 [INFO] TestAgent: Requesting shutdown -13:07.400 [INFO] TestAgent.server: shutting down server -13:07.400 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:07.400 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:07.400 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:07.400 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:07.400 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:07.400 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:07.400 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:07.400 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:07.400 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:07.400 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:07.400 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:07.400 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:07.400Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:07.400Z [INFO] agent.router.manager: shutting down -13:07.400 [INFO] TestAgent: consul server down -13:07.400 [INFO] TestAgent: shutdown complete -13:07.400 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16451 network=tcp -13:07.400 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16451 network=udp -13:07.400 [INFO] TestAgent: Stopping server: protocol=HTTP address=/tmp/consul-test/TestHTTPServer_UnixSocket-consul140359283/test.sock network=unix -13:07.901 [INFO] TestAgent: Waiting for endpoints to shut down -13:08.170 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPServer_UnixSocket (1.13s) -=== CONT TestHealthServiceNodes_WanTranslation -2021-10-31T23:13:07.204Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:07.270Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:07.272 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:78f94c82-1f4f-4805-d6fd-fc984b2dae0c Address:127.0.0.1:16463}]" -13:07.273 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-78f94c82-1f4f-4805-d6fd-fc984b2dae0c.dc1 127.0.0.1 -13:07.273 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16463 [Follower]" leader= -13:07.289 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-78f94c82-1f4f-4805-d6fd-fc984b2dae0c 127.0.0.1 -2021-10-31T23:13:07.289Z [INFO] agent.router: Initializing LAN area manager -13:07.290 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-78f94c82-1f4f-4805-d6fd-fc984b2dae0c.dc1 area=wan -13:07.290 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16458 network=udp -13:07.290 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16458 network=tcp -13:07.291 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16459 network=tcp -13:07.291 [INFO] TestAgent: started state syncer -13:07.291 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16464 network=tcp -13:07.294 [INFO] TestAgent.server: Adding LAN server: server="Node-78f94c82-1f4f-4805-d6fd-fc984b2dae0c (Addr: tcp/127.0.0.1:16463) (DC: dc1)" -13:07.341 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:07.341 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16463 [Candidate]" term=2 -13:07.342 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:07.342 [DEBUG] TestAgent.server.raft: vote granted: from=78f94c82-1f4f-4805-d6fd-fc984b2dae0c term=2 tally=1 -13:07.342 [INFO] TestAgent.server.raft: election won: tally=1 -13:07.342 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16463 [Leader]" -13:07.342 [INFO] TestAgent.server: cluster leadership acquired -13:07.342 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16463 -13:07.343 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:07.344 [INFO] TestAgent.server: New leader elected: payload=Node-78f94c82-1f4f-4805-d6fd-fc984b2dae0c -13:07.344 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:07.344 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:07.344 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:07.344 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:07.344 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:07.344 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-78f94c82-1f4f-4805-d6fd-fc984b2dae0c -13:07.344 [INFO] TestAgent.server: member joined, marking health alive: member=Node-78f94c82-1f4f-4805-d6fd-fc984b2dae0c -13:07.345 [INFO] TestAgent.server: federation state anti-entropy synced -13:07.409 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:07.410 [INFO] TestAgent: Synced node info -13:07.439 [INFO] TestAgent: Requesting shutdown -13:07.439 [INFO] TestAgent.server: shutting down server -13:07.439 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:07.439 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:07.439 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:07.439 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:07.439 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:07.439 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:07.439 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:07.439 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:07.439 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:07.439 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:07.439 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:07.439Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:07.439Z [INFO] agent.router.manager: shutting down -13:07.440 [INFO] TestAgent: consul server down -13:07.440 [INFO] TestAgent: shutdown complete -13:07.440 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16458 network=tcp -13:07.440 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16458 network=udp -13:07.440 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16459 network=tcp -13:07.941 [INFO] TestAgent: Waiting for endpoints to shut down -13:08.170 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceChecks_Filtering (1.01s) +=== CONT TestHealthServiceChecks_NodeMetaFilter +2022-12-04T05:49:21.527Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.536Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.537 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b5a54af3-b655-d83e-24b0-69675b8887c1 Address:127.0.0.1:23314}]" +49:21.538 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23314 [Follower]" leader= +49:21.538 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b5a54af3-b655-d83e-24b0-69675b8887c1.dc1 127.0.0.1 +49:21.538 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b5a54af3-b655-d83e-24b0-69675b8887c1 127.0.0.1 +2022-12-04T05:49:21.538Z [INFO] agent.router: Initializing LAN area manager +49:21.539 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23281 network=udp +49:21.539 [INFO] TestAgent.server: Adding LAN server: server="Node-b5a54af3-b655-d83e-24b0-69675b8887c1 (Addr: tcp/127.0.0.1:23314) (DC: dc1)" +49:21.539 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b5a54af3-b655-d83e-24b0-69675b8887c1.dc1 area=wan +49:21.539 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23281 network=tcp +49:21.539 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23282 network=tcp +49:21.539 [INFO] TestAgent: started state syncer +49:21.539 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23315 network=tcp +49:21.594 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.594 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23314 [Candidate]" term=2 +49:21.594 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.594 [DEBUG] TestAgent.server.raft: vote granted: from=b5a54af3-b655-d83e-24b0-69675b8887c1 term=2 tally=1 +49:21.594 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.594 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23314 [Leader]" +49:21.594 [INFO] TestAgent.server: cluster leadership acquired +49:21.594 [INFO] TestAgent.server: New leader elected: payload=Node-b5a54af3-b655-d83e-24b0-69675b8887c1 +49:21.594 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23314 +49:21.595 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.596 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.596 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.596 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.596 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.596 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.596 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b5a54af3-b655-d83e-24b0-69675b8887c1 +49:21.596 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b5a54af3-b655-d83e-24b0-69675b8887c1 +49:21.597 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.700 [INFO] TestAgent: Requesting shutdown +49:21.700 [INFO] TestAgent.server: shutting down server +49:21.700 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.700 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.700 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.700 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.700 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.700 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.700 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.700 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:21.700 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:21.700 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.700 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.700 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +2022-12-04T05:49:21.700Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.701Z [INFO] agent.router.manager: shutting down +49:21.701 [INFO] TestAgent: consul server down +49:21.701 [INFO] TestAgent: shutdown complete +49:21.701 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23281 network=tcp +49:21.701 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23281 network=udp +49:21.701 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23282 network=tcp +49:22.201 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.201 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_RaftConfiguration (0.68s) +=== CONT TestHealthServiceChecks +2022-12-04T05:49:21.650Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.660Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.662 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3ae61029-4930-c097-4792-2b87f34d8a5b Address:127.0.0.1:23363}]" +49:21.662 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23363 [Follower]" leader= +49:21.662 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3ae61029-4930-c097-4792-2b87f34d8a5b.dc1 127.0.0.1 +49:21.663 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3ae61029-4930-c097-4792-2b87f34d8a5b 127.0.0.1 +2022-12-04T05:49:21.663Z [INFO] agent.router: Initializing LAN area manager +49:21.663 [INFO] TestAgent.server: Adding LAN server: server="Node-3ae61029-4930-c097-4792-2b87f34d8a5b (Addr: tcp/127.0.0.1:23363) (DC: dc1)" +49:21.663 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3ae61029-4930-c097-4792-2b87f34d8a5b.dc1 area=wan +49:21.663 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23386 network=udp +49:21.663 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23386 network=tcp +49:21.664 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23387 network=tcp +49:21.664 [INFO] TestAgent: started state syncer +49:21.664 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23364 network=tcp +49:21.706 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.706 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23363 [Candidate]" term=2 +49:21.706 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.706 [DEBUG] TestAgent.server.raft: vote granted: from=3ae61029-4930-c097-4792-2b87f34d8a5b term=2 tally=1 +49:21.706 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.706 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23363 [Leader]" +49:21.707 [INFO] TestAgent.server: cluster leadership acquired +49:21.707 [INFO] TestAgent.server: New leader elected: payload=Node-3ae61029-4930-c097-4792-2b87f34d8a5b +49:21.707 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23363 +49:21.707 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.709 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.709 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.709 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.709 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.709 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.709 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3ae61029-4930-c097-4792-2b87f34d8a5b +49:21.709 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3ae61029-4930-c097-4792-2b87f34d8a5b +49:21.710 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.714 [INFO] TestAgent: Requesting shutdown +49:21.714 [INFO] TestAgent.server: shutting down server +49:21.714 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.714 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.714 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.714 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.714 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.714 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.715 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.715 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:21.715 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.715 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.715 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.715 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:21.715Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.715Z [INFO] agent.router.manager: shutting down +49:21.715 [INFO] TestAgent: consul server down +49:21.715 [INFO] TestAgent: shutdown complete +49:21.715 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23386 network=tcp +49:21.715 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23386 network=udp +49:21.715 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23387 network=tcp +49:22.215 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.215 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_PUT_ConflictingFlags (0.58s) +=== CONT TestHTTPServer_UnixSocket_FileExists +2022-12-04T05:49:21.351Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.363Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.364 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9e6d683c-8fb0-0e9e-255e-0ee3e37e83c4 Address:127.0.0.1:24285}]" +49:21.364 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24285 [Follower]" leader= +49:21.365 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9e6d683c-8fb0-0e9e-255e-0ee3e37e83c4.dc1 127.0.0.1 +49:21.365 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9e6d683c-8fb0-0e9e-255e-0ee3e37e83c4 127.0.0.1 +2022-12-04T05:49:21.365Z [INFO] agent.router: Initializing LAN area manager +49:21.365 [INFO] TestAgent.server: Adding LAN server: server="Node-9e6d683c-8fb0-0e9e-255e-0ee3e37e83c4 (Addr: tcp/127.0.0.1:24285) (DC: dc1)" +49:21.365 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9e6d683c-8fb0-0e9e-255e-0ee3e37e83c4.dc1 area=wan +49:21.365 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24280 network=udp +49:21.365 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24280 network=tcp +49:21.366 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24281 network=tcp +49:21.366 [INFO] TestAgent: started state syncer +49:21.366 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24286 network=tcp +49:21.428 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.428 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24285 [Candidate]" term=2 +49:21.428 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.428 [DEBUG] TestAgent.server.raft: vote granted: from=9e6d683c-8fb0-0e9e-255e-0ee3e37e83c4 term=2 tally=1 +49:21.428 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.428 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24285 [Leader]" +49:21.428 [INFO] TestAgent.server: cluster leadership acquired +49:21.428 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24285 +49:21.429 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.430 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.430 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.430 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.430 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.430 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.430 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9e6d683c-8fb0-0e9e-255e-0ee3e37e83c4 +49:21.430 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9e6d683c-8fb0-0e9e-255e-0ee3e37e83c4 +49:21.430 [INFO] TestAgent.server: New leader elected: payload=Node-9e6d683c-8fb0-0e9e-255e-0ee3e37e83c4 +49:21.431 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.442 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.443 [INFO] TestAgent: Synced node info +49:21.775 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +49:21.775 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-9e6d683c-8fb0-0e9e-255e-0ee3e37e83c4.dc1 +49:21.775 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +49:21.776 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-9e6d683c-8fb0-0e9e-255e-0ee3e37e83c4 +49:21.776 [INFO] TestAgent: Requesting shutdown +49:21.776 [INFO] TestAgent.server: shutting down server +49:21.776 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.776 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.776 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.776 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.776 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.776 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.776 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.776 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.776 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.776 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:21.776 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:49:21.776Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.776Z [INFO] agent.router.manager: shutting down +49:21.776 [INFO] TestAgent: consul server down +49:21.776 [INFO] TestAgent: shutdown complete +49:21.776 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24280 network=tcp +49:21.776 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24280 network=udp +49:21.776 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24281 network=tcp +49:22.276 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.276 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_KeyringList (0.94s) +=== CONT TestHealthNodeChecks_Filtering +[WARN] freeport: 10 out of 66 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:21.568Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.578Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.580 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f6bed0a3-8e5f-3d63-970c-52cf74fec35f Address:127.0.0.1:23335}]" +49:21.580 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23335 [Follower]" leader= +49:21.580 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f6bed0a3-8e5f-3d63-970c-52cf74fec35f.dc1 127.0.0.1 +49:21.580 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f6bed0a3-8e5f-3d63-970c-52cf74fec35f 127.0.0.1 +2022-12-04T05:49:21.580Z [INFO] agent.router: Initializing LAN area manager +49:21.581 [INFO] TestAgent.server: Adding LAN server: server="Node-f6bed0a3-8e5f-3d63-970c-52cf74fec35f (Addr: tcp/127.0.0.1:23335) (DC: dc1)" +49:21.581 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f6bed0a3-8e5f-3d63-970c-52cf74fec35f.dc1 area=wan +49:21.581 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23323 network=udp +49:21.581 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23323 network=tcp +49:21.581 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23324 network=tcp +49:21.581 [INFO] TestAgent: started state syncer +49:21.581 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23336 network=tcp +49:21.623 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.623 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23335 [Candidate]" term=2 +49:21.623 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.623 [DEBUG] TestAgent.server.raft: vote granted: from=f6bed0a3-8e5f-3d63-970c-52cf74fec35f term=2 tally=1 +49:21.623 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.623 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23335 [Leader]" +49:21.623 [INFO] TestAgent.server: cluster leadership acquired +49:21.623 [INFO] TestAgent.server: New leader elected: payload=Node-f6bed0a3-8e5f-3d63-970c-52cf74fec35f +49:21.623 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23335 +49:21.624 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.625 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.625 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.625 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.625 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.625 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.625 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f6bed0a3-8e5f-3d63-970c-52cf74fec35f +49:21.625 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f6bed0a3-8e5f-3d63-970c-52cf74fec35f +49:21.626 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.804 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.805 [INFO] TestAgent: Synced node info +49:21.910 [INFO] TestAgent: Requesting shutdown +49:21.910 [INFO] TestAgent.server: shutting down server +49:21.910 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.910 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.910 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.910 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.910 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.910 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.910 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.910 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.910 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.910 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.910 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:21.910Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.910Z [INFO] agent.router.manager: shutting down +49:21.910 [INFO] TestAgent: consul server down +49:21.910 [INFO] TestAgent: shutdown complete +49:21.910 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23323 network=tcp +49:21.910 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23323 network=udp +49:21.910 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23324 network=tcp +49:22.410 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.410 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_DELETE_ConflictingFlags (0.85s) +=== CONT TestHTTPServer_UnixSocket +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:21.770Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.794Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.796 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:926e3277-eb29-30ef-6b0c-83f9c9ec051e Address:127.0.0.1:23377}]" +49:21.796 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23377 [Follower]" leader= +49:21.796 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-926e3277-eb29-30ef-6b0c-83f9c9ec051e.dc1 127.0.0.1 +49:21.797 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-926e3277-eb29-30ef-6b0c-83f9c9ec051e 127.0.0.1 +2022-12-04T05:49:21.797Z [INFO] agent.router: Initializing LAN area manager +49:21.797 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23351 network=udp +49:21.797 [INFO] TestAgent.server: Adding LAN server: server="Node-926e3277-eb29-30ef-6b0c-83f9c9ec051e (Addr: tcp/127.0.0.1:23377) (DC: dc1)" +49:21.798 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-926e3277-eb29-30ef-6b0c-83f9c9ec051e.dc1 area=wan +49:21.798 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23351 network=tcp +49:21.799 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23352 network=tcp +49:21.799 [INFO] TestAgent: started state syncer +49:21.799 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23378 network=tcp +49:21.860 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.860 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23377 [Candidate]" term=2 +49:21.861 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.861 [DEBUG] TestAgent.server.raft: vote granted: from=926e3277-eb29-30ef-6b0c-83f9c9ec051e term=2 tally=1 +49:21.861 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.861 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23377 [Leader]" +49:21.861 [INFO] TestAgent.server: cluster leadership acquired +49:21.861 [INFO] TestAgent.server: New leader elected: payload=Node-926e3277-eb29-30ef-6b0c-83f9c9ec051e +49:21.861 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23377 +49:21.862 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.863 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.863 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.863 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.863 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.863 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.863 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-926e3277-eb29-30ef-6b0c-83f9c9ec051e +49:21.863 [INFO] TestAgent.server: member joined, marking health alive: member=Node-926e3277-eb29-30ef-6b0c-83f9c9ec051e +49:21.864 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.952 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.952 [INFO] TestAgent: Synced node info +49:21.952 [DEBUG] TestAgent: Node info in sync +49:22.078 [INFO] TestAgent: Requesting shutdown +49:22.078 [INFO] TestAgent.server: shutting down server +49:22.078 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.078 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.078 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.078 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.078 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.079 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.079 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.079 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.079 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.079 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.079 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.079Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.079Z [INFO] agent.router.manager: shutting down +49:22.079 [INFO] TestAgent: consul server down +49:22.079 [INFO] TestAgent: shutdown complete +49:22.079 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23351 network=tcp +49:22.079 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23351 network=udp +49:22.079 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23352 network=tcp +49:22.579 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.579 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_GET_Raw (0.83s) === CONT TestHealthNodeChecks -2021-10-31T23:13:06.818Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.861Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.874 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8aa0cc63-1b03-fedf-4f62-76a9f3ea0058 Address:127.0.0.1:16414}]" -13:06.875 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8aa0cc63-1b03-fedf-4f62-76a9f3ea0058.dc1 127.0.0.1 -13:06.876 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8aa0cc63-1b03-fedf-4f62-76a9f3ea0058 127.0.0.1 -2021-10-31T23:13:06.876Z [INFO] agent.router: Initializing LAN area manager -13:06.876 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16409 network=udp -13:06.876 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16414 [Follower]" leader= -13:06.876 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16409 network=tcp -13:06.876 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8aa0cc63-1b03-fedf-4f62-76a9f3ea0058.dc1 area=wan -13:06.876 [INFO] TestAgent.server: Adding LAN server: server="Node-8aa0cc63-1b03-fedf-4f62-76a9f3ea0058 (Addr: tcp/127.0.0.1:16414) (DC: dc1)" -13:06.877 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16410 network=tcp -13:06.877 [INFO] TestAgent: started state syncer -13:06.877 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16415 network=tcp -13:06.931 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.931 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16414 [Candidate]" term=2 -13:06.931 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.931 [DEBUG] TestAgent.server.raft: vote granted: from=8aa0cc63-1b03-fedf-4f62-76a9f3ea0058 term=2 tally=1 -13:06.931 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.931 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16414 [Leader]" -13:06.931 [INFO] TestAgent.server: cluster leadership acquired -13:06.931 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16414 -13:06.932 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:06.933 [INFO] TestAgent.server: New leader elected: payload=Node-8aa0cc63-1b03-fedf-4f62-76a9f3ea0058 -13:06.934 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:06.934 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:06.934 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:06.934 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:06.934 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.934 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8aa0cc63-1b03-fedf-4f62-76a9f3ea0058 -13:06.934 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8aa0cc63-1b03-fedf-4f62-76a9f3ea0058 -13:06.950 [INFO] TestAgent.server: federation state anti-entropy synced -13:07.014 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:07.014 [INFO] TestAgent: Synced node info -13:07.018 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from= latency=3.597µs -13:07.018 [INFO] TestAgent: Requesting shutdown -13:07.018 [INFO] TestAgent.server: shutting down server -13:07.018 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:07.018 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:07.018 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:07.018 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:07.018 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:07.018 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:07.018 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:07.018 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:07.018 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:07.018 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:07.018 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:07.018Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:07.018Z [INFO] agent.router.manager: shutting down -13:07.018 [INFO] TestAgent: consul server down -13:07.018 [INFO] TestAgent: shutdown complete -13:07.018 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16409 network=tcp -13:07.018 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16409 network=udp -13:07.018 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16410 network=tcp -13:07.521 [INFO] TestAgent: Waiting for endpoints to shut down -13:07.521 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:06.496Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.538Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.539 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b22f73cc-66ad-8071-8023-c4d7d4379cd9 Address:127.0.0.1:16365}]" -13:06.540 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b22f73cc-66ad-8071-8023-c4d7d4379cd9.dc1 127.0.0.1 -13:06.540 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b22f73cc-66ad-8071-8023-c4d7d4379cd9 127.0.0.1 -2021-10-31T23:13:06.540Z [INFO] agent.router: Initializing LAN area manager -13:06.540 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16360 network=udp -13:06.540 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16365 [Follower]" leader= -13:06.541 [INFO] TestAgent.server: Adding LAN server: server="Node-b22f73cc-66ad-8071-8023-c4d7d4379cd9 (Addr: tcp/127.0.0.1:16365) (DC: dc1)" -13:06.541 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b22f73cc-66ad-8071-8023-c4d7d4379cd9.dc1 area=wan -13:06.541 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16360 network=tcp -13:06.541 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16361 network=tcp -13:06.541 [INFO] TestAgent: started state syncer -13:06.541 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16366 network=tcp -13:06.597 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.597 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16365 [Candidate]" term=2 -13:06.598 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.598 [DEBUG] TestAgent.server.raft: vote granted: from=b22f73cc-66ad-8071-8023-c4d7d4379cd9 term=2 tally=1 -13:06.598 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.598 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16365 [Leader]" -13:06.598 [INFO] TestAgent.server: cluster leadership acquired -13:06.598 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16365 -13:06.599 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:06.600 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:06.600 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:06.600 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:06.600 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:06.600 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.600 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b22f73cc-66ad-8071-8023-c4d7d4379cd9 -13:06.600 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b22f73cc-66ad-8071-8023-c4d7d4379cd9 -13:06.601 [INFO] TestAgent.server: New leader elected: payload=Node-b22f73cc-66ad-8071-8023-c4d7d4379cd9 -13:06.601 [INFO] TestAgent.server: federation state anti-entropy synced -13:06.719 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from= latency=2.262µs -13:06.767 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:06.767 [INFO] TestAgent: Synced node info -13:06.767 [DEBUG] TestAgent: Node info in sync -13:07.255 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:07.255 [DEBUG] TestAgent: Node info in sync -13:07.522 [INFO] TestAgent: Requesting shutdown -13:07.522 [INFO] TestAgent.server: shutting down server -13:07.522 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:07.522 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:07.522 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:07.522 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:07.522 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:07.522 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:07.522 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:07.522 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:07.522 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:07.522 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:07.522 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:07.522Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:07.522Z [INFO] agent.router.manager: shutting down -13:07.522 [INFO] TestAgent: consul server down -13:07.522 [INFO] TestAgent: shutdown complete -13:07.522 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16360 network=tcp -13:07.522 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16360 network=udp -13:07.522 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16361 network=tcp -13:08.024 [INFO] TestAgent: Waiting for endpoints to shut down -13:08.170 [INFO] TestAgent: Endpoints down ---- PASS: TestHTTPAPI_TranslateAddrHeader (1.72s) -=== CONT TestHealthServiceNodes_CheckType -[WARN] freeport: 10 out of 59 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestHealthIngressServiceNodes/associated_service -=== RUN TestHealthIngressServiceNodes/non-associated_service -2021-10-31T23:13:07.518Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:07.545Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:07.547 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8151340a-172f-4a4e-1e24-58a6d3ed9be3 Address:127.0.0.1:16491}]" -13:07.547 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16491 [Follower]" leader= -13:07.547 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8151340a-172f-4a4e-1e24-58a6d3ed9be3.dc1 127.0.0.1 -13:07.548 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8151340a-172f-4a4e-1e24-58a6d3ed9be3 127.0.0.1 -2021-10-31T23:13:07.548Z [INFO] agent.router: Initializing LAN area manager -13:07.548 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16479 network=udp -13:07.548 [INFO] TestAgent.server: Adding LAN server: server="Node-8151340a-172f-4a4e-1e24-58a6d3ed9be3 (Addr: tcp/127.0.0.1:16491) (DC: dc1)" -13:07.548 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8151340a-172f-4a4e-1e24-58a6d3ed9be3.dc1 area=wan -13:07.549 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16479 network=tcp -13:07.549 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16480 network=tcp -13:07.549 [INFO] TestAgent: started state syncer -13:07.549 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16492 network=tcp -13:07.605 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:07.606 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16491 [Candidate]" term=2 -13:07.606 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:07.606 [DEBUG] TestAgent.server.raft: vote granted: from=8151340a-172f-4a4e-1e24-58a6d3ed9be3 term=2 tally=1 -13:07.606 [INFO] TestAgent.server.raft: election won: tally=1 -13:07.606 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16491 [Leader]" -13:07.606 [INFO] TestAgent.server: cluster leadership acquired -13:07.606 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16491 -13:07.607 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:07.608 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:07.608 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:07.608 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:07.608 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:07.608 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:07.608 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8151340a-172f-4a4e-1e24-58a6d3ed9be3 -13:07.608 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8151340a-172f-4a4e-1e24-58a6d3ed9be3 -13:07.609 [INFO] TestAgent.server: New leader elected: payload=Node-8151340a-172f-4a4e-1e24-58a6d3ed9be3 -13:07.609 [INFO] TestAgent.server: federation state anti-entropy synced -13:07.662 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:07.662 [INFO] TestAgent: Synced node info -13:08.162 [INFO] TestAgent: Requesting shutdown -13:08.162 [INFO] TestAgent.server: shutting down server -13:08.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:08.162 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:08.162 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:08.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:08.162 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:08.164 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:08.164 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:08.164 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:08.164 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:08.164 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:08.164 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:08.165Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:08.165Z [INFO] agent.router.manager: shutting down -13:08.165 [INFO] TestAgent: consul server down -13:08.165 [INFO] TestAgent: shutdown complete -13:08.165 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16479 network=tcp -13:08.165 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16479 network=udp -13:08.165 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16480 network=tcp -13:08.665 [INFO] TestAgent: Waiting for endpoints to shut down -13:08.665 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:07.491Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:07.513Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:07.519 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fc6afb37-6e25-d67a-f62b-fa0d9a3a02d4 Address:127.0.0.1:16477}]" -13:07.520 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fc6afb37-6e25-d67a-f62b-fa0d9a3a02d4.dc1 127.0.0.1 -13:07.520 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16477 [Follower]" leader= -13:07.520 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fc6afb37-6e25-d67a-f62b-fa0d9a3a02d4 127.0.0.1 -2021-10-31T23:13:07.520Z [INFO] agent.router: Initializing LAN area manager -13:07.521 [INFO] TestAgent.server: Adding LAN server: server="Node-fc6afb37-6e25-d67a-f62b-fa0d9a3a02d4 (Addr: tcp/127.0.0.1:16477) (DC: dc1)" -13:07.521 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16472 network=udp -13:07.521 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fc6afb37-6e25-d67a-f62b-fa0d9a3a02d4.dc1 area=wan -13:07.521 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16472 network=tcp -13:07.521 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16473 network=tcp -13:07.521 [INFO] TestAgent: started state syncer -13:07.521 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16478 network=tcp -13:07.586 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:07.586 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16477 [Candidate]" term=2 -13:07.586 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:07.586 [DEBUG] TestAgent.server.raft: vote granted: from=fc6afb37-6e25-d67a-f62b-fa0d9a3a02d4 term=2 tally=1 -13:07.586 [INFO] TestAgent.server.raft: election won: tally=1 -13:07.586 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16477 [Leader]" -13:07.586 [INFO] TestAgent.server: cluster leadership acquired -13:07.586 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16477 -13:07.587 [INFO] TestAgent.server: New leader elected: payload=Node-fc6afb37-6e25-d67a-f62b-fa0d9a3a02d4 -13:07.587 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:07.589 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:07.589 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:07.589 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:07.589 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:07.589 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:07.589 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fc6afb37-6e25-d67a-f62b-fa0d9a3a02d4 -13:07.589 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fc6afb37-6e25-d67a-f62b-fa0d9a3a02d4 -13:07.594 [INFO] TestAgent.server: federation state anti-entropy synced -13:08.162 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:08.164 [INFO] TestAgent: Requesting shutdown -13:08.164 [INFO] TestAgent.server: shutting down server -13:08.164 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:08.164 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:08.164 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:08.164 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:08.164 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:08.164 [INFO] TestAgent: Synced node info -13:08.164 [DEBUG] TestAgent: Node info in sync -13:08.164 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:08.164 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:08.164 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:08.164 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:08.164 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:08.164 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:08.165Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:08.165Z [INFO] agent.router.manager: shutting down -13:08.165 [INFO] TestAgent: consul server down -13:08.165 [INFO] TestAgent: shutdown complete -13:08.165 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16472 network=tcp -13:08.165 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16472 network=udp -13:08.165 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16473 network=tcp -13:08.665 [INFO] TestAgent: Waiting for endpoints to shut down -13:08.665 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_InmemKeyrings/server_with_keys ---- PASS: TestHealthConnectServiceNodes_PassingFilter (1.21s) - --- PASS: TestHealthConnectServiceNodes_PassingFilter/bc_no_query_value (0.00s) - --- PASS: TestHealthConnectServiceNodes_PassingFilter/passing_true (0.00s) - --- PASS: TestHealthConnectServiceNodes_PassingFilter/passing_false (0.00s) - --- PASS: TestHealthConnectServiceNodes_PassingFilter/passing_bad (0.00s) +2022-12-04T05:49:21.575Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.585Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.586 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1918bbed-1d8a-b151-34ff-d6c6791e293f Address:127.0.0.1:23286}]" +49:21.586 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23286 [Follower]" leader= +49:21.586 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1918bbed-1d8a-b151-34ff-d6c6791e293f.dc1 127.0.0.1 +49:21.587 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1918bbed-1d8a-b151-34ff-d6c6791e293f 127.0.0.1 +2022-12-04T05:49:21.587Z [INFO] agent.router: Initializing LAN area manager +49:21.587 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23344 network=udp +49:21.587 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1918bbed-1d8a-b151-34ff-d6c6791e293f.dc1 area=wan +49:21.587 [INFO] TestAgent.server: Adding LAN server: server="Node-1918bbed-1d8a-b151-34ff-d6c6791e293f (Addr: tcp/127.0.0.1:23286) (DC: dc1)" +49:21.587 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23344 network=tcp +49:21.588 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23345 network=tcp +49:21.588 [INFO] TestAgent: started state syncer +49:21.588 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23287 network=tcp +49:21.643 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.643 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23286 [Candidate]" term=2 +49:21.644 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.644 [DEBUG] TestAgent.server.raft: vote granted: from=1918bbed-1d8a-b151-34ff-d6c6791e293f term=2 tally=1 +49:21.644 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.644 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23286 [Leader]" +49:21.644 [INFO] TestAgent.server: cluster leadership acquired +49:21.644 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23286 +49:21.645 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.645 [INFO] TestAgent.server: New leader elected: payload=Node-1918bbed-1d8a-b151-34ff-d6c6791e293f +49:21.646 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.646 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.646 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.646 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.646 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.646 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1918bbed-1d8a-b151-34ff-d6c6791e293f +49:21.646 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1918bbed-1d8a-b151-34ff-d6c6791e293f +49:21.647 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.712 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.712 [INFO] TestAgent: Synced node info +49:22.080 [WARN] TestAgent.server.rpc: Attempting to apply large raft entry: size_in_bytes=4719032 +49:22.118 [INFO] TestAgent: Requesting shutdown +49:22.118 [INFO] TestAgent.server: shutting down server +49:22.118 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.118 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.118 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.118 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.118 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.118 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.118 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.118 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.118 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.118 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.118 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.118Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.118Z [INFO] agent.router.manager: shutting down +49:22.118 [INFO] TestAgent: consul server down +49:22.118 [INFO] TestAgent: shutdown complete +49:22.118 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23344 network=tcp +49:22.118 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23344 network=udp +49:22.118 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23345 network=tcp +49:22.619 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.619 [INFO] TestAgent: Endpoints down +--- PASS: TestTxnEndpoint_Bad_Size_Net (5.89s) + --- PASS: TestTxnEndpoint_Bad_Size_Net/exceeds_default_limits (1.14s) + --- PASS: TestTxnEndpoint_Bad_Size_Net/exceeds_configured_max_txn_len (1.16s) + --- PASS: TestTxnEndpoint_Bad_Size_Net/exceeds_default_max_kv_value_size (1.30s) + --- PASS: TestTxnEndpoint_Bad_Size_Net/allowed (1.24s) + --- PASS: TestTxnEndpoint_Bad_Size_Net/allowed_kv_max_backward_compatible (1.06s) +=== CONT TestFilterNonPassing +--- PASS: TestFilterNonPassing (0.00s) === CONT TestHealthChecksInState_DistanceSort -2021-10-31T23:13:06.678Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:06.726Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:06.734 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:705bb8fb-2f1d-ce22-77dd-f9d6a5ae73d9 Address:127.0.0.1:16386}]" -13:06.735 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-705bb8fb-2f1d-ce22-77dd-f9d6a5ae73d9.dc1 127.0.0.1 -13:06.736 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-705bb8fb-2f1d-ce22-77dd-f9d6a5ae73d9 127.0.0.1 -2021-10-31T23:13:06.736Z [INFO] agent.router: Initializing LAN area manager -13:06.736 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16381 network=udp -13:06.736 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16386 [Follower]" leader= -13:06.737 [INFO] TestAgent.server: Adding LAN server: server="Node-705bb8fb-2f1d-ce22-77dd-f9d6a5ae73d9 (Addr: tcp/127.0.0.1:16386) (DC: dc1)" -13:06.737 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-705bb8fb-2f1d-ce22-77dd-f9d6a5ae73d9.dc1 area=wan -13:06.737 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16381 network=tcp -13:06.743 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16382 network=tcp -13:06.743 [INFO] TestAgent: started state syncer -13:06.743 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16387 network=tcp -13:06.793 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:06.793 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16386 [Candidate]" term=2 -13:06.794 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:06.794 [DEBUG] TestAgent.server.raft: vote granted: from=705bb8fb-2f1d-ce22-77dd-f9d6a5ae73d9 term=2 tally=1 -13:06.794 [INFO] TestAgent.server.raft: election won: tally=1 -13:06.794 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16386 [Leader]" -13:06.794 [INFO] TestAgent.server: cluster leadership acquired -13:06.794 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16386 -13:06.794 [INFO] TestAgent.server: New leader elected: payload=Node-705bb8fb-2f1d-ce22-77dd-f9d6a5ae73d9 -13:06.799 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:06.801 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:06.801 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:06.801 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:06.801 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:06.801 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:06.801 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-705bb8fb-2f1d-ce22-77dd-f9d6a5ae73d9 -13:06.801 [INFO] TestAgent.server: member joined, marking health alive: member=Node-705bb8fb-2f1d-ce22-77dd-f9d6a5ae73d9 -13:06.801 [INFO] TestAgent.server: federation state anti-entropy synced -13:06.810 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:06.810 [INFO] TestAgent: Synced node info -13:08.148 [INFO] TestAgent: Requesting shutdown -13:08.171 [INFO] TestAgent.server: shutting down server -13:08.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:08.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:08.171 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:08.171 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:08.171 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:08.172 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:08.172 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:08.172 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:08.172 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:08.172 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:08.172 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:08.172Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:08.172Z [INFO] agent.router.manager: shutting down -13:08.172 [INFO] TestAgent: consul server down -13:08.172 [INFO] TestAgent: shutdown complete -13:08.172 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16381 network=tcp -13:08.172 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16381 network=udp -13:08.172 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16382 network=tcp -13:08.677 [INFO] TestAgent: Waiting for endpoints to shut down -13:08.677 [INFO] TestAgent: Endpoints down -=== RUN TestTxnEndpoint_Bad_Size_Net/exceeds_default_max_kv_value_size -2021-10-31T23:13:07.577Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:07.616Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:07.638 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:75f86a7a-d883-df15-622d-4223246ec6e0 Address:127.0.0.1:16484}]" -13:07.639 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-75f86a7a-d883-df15-622d-4223246ec6e0.dc1 127.0.0.1 -13:07.640 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-75f86a7a-d883-df15-622d-4223246ec6e0 127.0.0.1 -2021-10-31T23:13:07.640Z [INFO] agent.router: Initializing LAN area manager -13:07.640 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16493 network=udp -13:07.640 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16484 [Follower]" leader= -13:07.640 [INFO] TestAgent.server: Adding LAN server: server="Node-75f86a7a-d883-df15-622d-4223246ec6e0 (Addr: tcp/127.0.0.1:16484) (DC: dc1)" -13:07.640 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-75f86a7a-d883-df15-622d-4223246ec6e0.dc1 area=wan -13:07.640 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16493 network=tcp -13:07.670 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16494 network=tcp -13:07.670 [INFO] TestAgent: started state syncer -13:07.670 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16485 network=tcp -13:07.679 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:07.679 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16484 [Candidate]" term=2 -13:07.679 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:07.679 [DEBUG] TestAgent.server.raft: vote granted: from=75f86a7a-d883-df15-622d-4223246ec6e0 term=2 tally=1 -13:07.679 [INFO] TestAgent.server.raft: election won: tally=1 -13:07.679 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16484 [Leader]" -13:07.679 [INFO] TestAgent.server: cluster leadership acquired -13:07.679 [INFO] TestAgent.server: New leader elected: payload=Node-75f86a7a-d883-df15-622d-4223246ec6e0 -13:07.679 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16484 -13:07.680 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:07.727 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:07.727 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:07.727 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:07.727 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:07.727 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:07.727 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-75f86a7a-d883-df15-622d-4223246ec6e0 -13:07.727 [INFO] TestAgent.server: member joined, marking health alive: member=Node-75f86a7a-d883-df15-622d-4223246ec6e0 -13:07.729 [INFO] TestAgent.server: federation state anti-entropy synced -13:08.046 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:08.171 [INFO] TestAgent: Requesting shutdown -13:08.171 [INFO] TestAgent.server: shutting down server -13:08.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:08.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:08.171 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:08.171 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:08.171 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:08.172 [INFO] TestAgent: Synced node info -13:08.172 [DEBUG] TestAgent: Node info in sync -13:08.172 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:08.172 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:08.172 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:08.172 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:08.172 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:08.172 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:08.172Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:08.172Z [INFO] agent.router.manager: shutting down -13:08.172 [INFO] TestAgent: consul server down -13:08.172 [INFO] TestAgent: shutdown complete -13:08.172 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16493 network=tcp -13:08.172 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16493 network=udp -13:08.172 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16494 network=tcp -13:08.681 [INFO] TestAgent: Waiting for endpoints to shut down -13:08.681 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthConnectServiceNodes_Filter (1.13s) -2021-10-31T23:13:07.434Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:07.460Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:07.466 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b2653130-8e97-adea-6840-7b810ac3b5a7 Address:127.0.0.1:16470}]" -13:07.467 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b2653130-8e97-adea-6840-7b810ac3b5a7.dc1 127.0.0.1 -13:07.467 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b2653130-8e97-adea-6840-7b810ac3b5a7 127.0.0.1 -2021-10-31T23:13:07.467Z [INFO] agent.router: Initializing LAN area manager -13:07.468 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16465 network=udp -13:07.468 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16470 [Follower]" leader= -13:07.468 [INFO] TestAgent.server: Adding LAN server: server="Node-b2653130-8e97-adea-6840-7b810ac3b5a7 (Addr: tcp/127.0.0.1:16470) (DC: dc1)" -13:07.468 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b2653130-8e97-adea-6840-7b810ac3b5a7.dc1 area=wan -13:07.468 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16465 network=tcp -13:07.469 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16466 network=tcp -13:07.469 [INFO] TestAgent: started state syncer -13:07.469 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16471 network=tcp -13:07.520 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:07.520 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16470 [Candidate]" term=2 -13:07.521 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:07.521 [DEBUG] TestAgent.server.raft: vote granted: from=b2653130-8e97-adea-6840-7b810ac3b5a7 term=2 tally=1 -13:07.521 [INFO] TestAgent.server.raft: election won: tally=1 -13:07.521 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16470 [Leader]" -13:07.521 [INFO] TestAgent.server: cluster leadership acquired -13:07.521 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16470 -13:07.521 [INFO] TestAgent.server: New leader elected: payload=Node-b2653130-8e97-adea-6840-7b810ac3b5a7 -13:07.527 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:07.528 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:07.528 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:07.528 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:07.528 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:07.528 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:07.528 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b2653130-8e97-adea-6840-7b810ac3b5a7 -13:07.528 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b2653130-8e97-adea-6840-7b810ac3b5a7 -13:07.529 [INFO] TestAgent.server: federation state anti-entropy synced -13:08.167 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:08.168 [INFO] TestAgent: Synced node info -13:08.170 [INFO] TestAgent: Requesting shutdown -13:08.170 [INFO] TestAgent.server: shutting down server -13:08.170 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:08.170 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:08.170 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:08.170 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:08.170 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:08.171 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:08.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:08.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:08.171 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:08.171 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:08.171 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:08.171Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:08.171Z [INFO] agent.router.manager: shutting down -13:08.171 [INFO] TestAgent: consul server down -13:08.171 [INFO] TestAgent: shutdown complete -13:08.171 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16465 network=tcp -13:08.171 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16465 network=udp -13:08.171 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16466 network=tcp -13:08.681 [INFO] TestAgent: Waiting for endpoints to shut down -13:08.681 [INFO] TestAgent: Endpoints down -=== CONT TestIntentionsSpecificUpdate_good ---- PASS: TestHealthServiceChecks_NodeMetaFilter (1.28s) +2022-12-04T05:49:22.040Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.050Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.051 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0af21904-6c4e-9789-9157-8b92310afa7a Address:127.0.0.1:23300}]" +49:22.051 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23300 [Follower]" leader= +49:22.051 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0af21904-6c4e-9789-9157-8b92310afa7a.dc1 127.0.0.1 +49:22.052 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0af21904-6c4e-9789-9157-8b92310afa7a 127.0.0.1 +2022-12-04T05:49:22.052Z [INFO] agent.router: Initializing LAN area manager +49:22.052 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0af21904-6c4e-9789-9157-8b92310afa7a.dc1 area=wan +49:22.052 [INFO] TestAgent.server: Adding LAN server: server="Node-0af21904-6c4e-9789-9157-8b92310afa7a (Addr: tcp/127.0.0.1:23300) (DC: dc1)" +49:22.052 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23330 network=udp +49:22.052 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23330 network=tcp +49:22.053 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23331 network=tcp +49:22.053 [INFO] TestAgent: started state syncer +49:22.053 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23301 network=tcp +49:22.112 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.112 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23300 [Candidate]" term=2 +49:22.113 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.113 [DEBUG] TestAgent.server.raft: vote granted: from=0af21904-6c4e-9789-9157-8b92310afa7a term=2 tally=1 +49:22.113 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.113 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23300 [Leader]" +49:22.113 [INFO] TestAgent.server: cluster leadership acquired +49:22.113 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23300 +49:22.114 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.115 [INFO] TestAgent.server: New leader elected: payload=Node-0af21904-6c4e-9789-9157-8b92310afa7a +49:22.115 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.115 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.115 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.115 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.115 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.115 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0af21904-6c4e-9789-9157-8b92310afa7a +49:22.115 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0af21904-6c4e-9789-9157-8b92310afa7a +49:22.115 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.130 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:22.131 [INFO] TestAgent: Synced node info +49:22.136 [INFO] TestAgent: Requesting shutdown +49:22.137 [INFO] TestAgent.server: shutting down server +49:22.137 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.137 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.137 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.137 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.137 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.137 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.137 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.137 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.137 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.137 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.137 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.137Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.137Z [INFO] agent.router.manager: shutting down +49:22.137 [INFO] TestAgent: consul server down +49:22.137 [INFO] TestAgent: shutdown complete +49:22.137 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23330 network=tcp +49:22.137 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23330 network=udp +49:22.137 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23331 network=tcp +49:22.638 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.638 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceNodes_NodeMetaFilter (0.61s) === CONT TestHealthChecksInState_Filter -2021-10-31T23:13:08.169Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:08.223Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:08.224 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b0a3b51e-2d5a-c2f2-d585-0581f86f337d Address:127.0.0.1:16540}]" -13:08.225 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b0a3b51e-2d5a-c2f2-d585-0581f86f337d.dc1 127.0.0.1 -13:08.225 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16540 [Follower]" leader= -13:08.238 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b0a3b51e-2d5a-c2f2-d585-0581f86f337d 127.0.0.1 -2021-10-31T23:13:08.238Z [INFO] agent.router: Initializing LAN area manager -13:08.238 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16521 network=udp -13:08.238 [INFO] TestAgent.server: Adding LAN server: server="Node-b0a3b51e-2d5a-c2f2-d585-0581f86f337d (Addr: tcp/127.0.0.1:16540) (DC: dc1)" -13:08.238 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b0a3b51e-2d5a-c2f2-d585-0581f86f337d.dc1 area=wan -13:08.239 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16521 network=tcp -13:08.239 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16522 network=tcp -13:08.239 [INFO] TestAgent: started state syncer -13:08.239 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16541 network=tcp -13:08.273 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:08.273 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16540 [Candidate]" term=2 -13:08.273 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:08.273 [DEBUG] TestAgent.server.raft: vote granted: from=b0a3b51e-2d5a-c2f2-d585-0581f86f337d term=2 tally=1 -13:08.273 [INFO] TestAgent.server.raft: election won: tally=1 -13:08.273 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16540 [Leader]" -13:08.273 [INFO] TestAgent.server: cluster leadership acquired -13:08.273 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16540 -13:08.273 [INFO] TestAgent.server: New leader elected: payload=Node-b0a3b51e-2d5a-c2f2-d585-0581f86f337d -13:08.294 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:08.296 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:08.296 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:08.296 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:08.296 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:08.296 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:08.296 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b0a3b51e-2d5a-c2f2-d585-0581f86f337d -13:08.296 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b0a3b51e-2d5a-c2f2-d585-0581f86f337d -13:08.298 [INFO] TestAgent.server: federation state anti-entropy synced -13:08.327 [INFO] TestAgent: Requesting shutdown -13:08.327 [INFO] TestAgent.server: shutting down server -13:08.327 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:08.327 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:08.327 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:08.327 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:08.327 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:08.327 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:08.327 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:08.327 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:08.327 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:08.327 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:08.327 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:08.327 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:08.327Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:08.327Z [INFO] agent.router.manager: shutting down -13:08.328 [INFO] TestAgent: consul server down -13:08.328 [INFO] TestAgent: shutdown complete -13:08.328 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16521 network=tcp -13:08.328 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16521 network=udp -13:08.328 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16522 network=tcp -13:08.829 [INFO] TestAgent: Waiting for endpoints to shut down -13:08.829 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Apply_IngressGateway (1.20s) -=== CONT TestHealthNodeChecks_Filtering -[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:08.213Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:08.246Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:08.247 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:399133bc-1334-1533-1131-8180e188d3cb Address:127.0.0.1:16498}]" -13:08.248 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-399133bc-1334-1533-1131-8180e188d3cb.dc1 127.0.0.1 -13:08.249 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-399133bc-1334-1533-1131-8180e188d3cb 127.0.0.1 -2021-10-31T23:13:08.249Z [INFO] agent.router: Initializing LAN area manager -13:08.249 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16563 network=udp -13:08.249 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16498 [Follower]" leader= -13:08.249 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-399133bc-1334-1533-1131-8180e188d3cb.dc1 area=wan -13:08.250 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16563 network=tcp -13:08.250 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16564 network=tcp -13:08.250 [INFO] TestAgent: started state syncer -13:08.250 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16499 network=tcp -13:08.261 [INFO] TestAgent.server: Adding LAN server: server="Node-399133bc-1334-1533-1131-8180e188d3cb (Addr: tcp/127.0.0.1:16498) (DC: dc1)" -13:08.293 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:08.293 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16498 [Candidate]" term=2 -13:08.294 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:08.294 [DEBUG] TestAgent.server.raft: vote granted: from=399133bc-1334-1533-1131-8180e188d3cb term=2 tally=1 -13:08.294 [INFO] TestAgent.server.raft: election won: tally=1 -13:08.294 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16498 [Leader]" -13:08.294 [INFO] TestAgent.server: cluster leadership acquired -13:08.294 [INFO] TestAgent.server: New leader elected: payload=Node-399133bc-1334-1533-1131-8180e188d3cb -13:08.294 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16498 -13:08.295 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:08.297 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:08.297 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:08.297 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:08.297 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:08.297 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:08.297 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-399133bc-1334-1533-1131-8180e188d3cb -13:08.297 [INFO] TestAgent.server: member joined, marking health alive: member=Node-399133bc-1334-1533-1131-8180e188d3cb -13:08.306 [INFO] TestAgent.server: federation state anti-entropy synced -13:08.486 [INFO] TestAgent: Requesting shutdown -13:08.486 [INFO] TestAgent.server: shutting down server -13:08.486 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:08.486 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:08.486 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:08.486 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:08.486 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:08.486 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:08.486 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:08.486 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:08.486 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:08.486 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:08.486 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:08.486 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:08.486Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:08.486Z [INFO] agent.router.manager: shutting down -13:08.486 [INFO] TestAgent: consul server down -13:08.486 [INFO] TestAgent: shutdown complete -13:08.486 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16563 network=tcp -13:08.486 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16563 network=udp -13:08.486 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16564 network=tcp -13:08.987 [INFO] TestAgent: Waiting for endpoints to shut down -13:08.987 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_LoadKeyrings/server_with_keys -2021-10-31T23:13:08.208Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:08.250Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:08.252 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dd46cd4f-05fc-dbcd-3f5d-4ea7508d4f0f Address:127.0.0.1:16554}]" -13:08.253 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dd46cd4f-05fc-dbcd-3f5d-4ea7508d4f0f.dc1 127.0.0.1 -13:08.253 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dd46cd4f-05fc-dbcd-3f5d-4ea7508d4f0f 127.0.0.1 -2021-10-31T23:13:08.253Z [INFO] agent.router: Initializing LAN area manager -13:08.253 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16554 [Follower]" leader= -13:08.254 [INFO] TestAgent.server: Adding LAN server: server="Node-dd46cd4f-05fc-dbcd-3f5d-4ea7508d4f0f (Addr: tcp/127.0.0.1:16554) (DC: dc1)" -13:08.254 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dd46cd4f-05fc-dbcd-3f5d-4ea7508d4f0f.dc1 area=wan -13:08.254 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16549 network=tcp -13:08.262 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16549 network=udp -13:08.262 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16550 network=tcp -13:08.262 [INFO] TestAgent: started state syncer -13:08.262 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16555 network=tcp -13:08.295 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:08.295 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16554 [Candidate]" term=2 -13:08.296 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:08.296 [DEBUG] TestAgent.server.raft: vote granted: from=dd46cd4f-05fc-dbcd-3f5d-4ea7508d4f0f term=2 tally=1 -13:08.296 [INFO] TestAgent.server.raft: election won: tally=1 -13:08.296 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16554 [Leader]" -13:08.296 [INFO] TestAgent.server: cluster leadership acquired -13:08.296 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16554 -13:08.297 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:08.298 [INFO] TestAgent.server: New leader elected: payload=Node-dd46cd4f-05fc-dbcd-3f5d-4ea7508d4f0f -13:08.309 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:08.309 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:08.309 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:08.309 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:08.309 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:08.309 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dd46cd4f-05fc-dbcd-3f5d-4ea7508d4f0f -13:08.309 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dd46cd4f-05fc-dbcd-3f5d-4ea7508d4f0f -13:08.310 [INFO] TestAgent.server: federation state anti-entropy synced -13:08.515 [INFO] TestAgent: Requesting shutdown -13:08.515 [INFO] TestAgent.server: shutting down server -13:08.515 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:08.515 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:08.515 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:08.515 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:08.515 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:08.515 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:08.515 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:08.515 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:08.515 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:08.515 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:08.515 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:08.515 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:08.515Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:08.515Z [INFO] agent.router.manager: shutting down -13:08.515 [INFO] TestAgent: consul server down -13:08.515 [INFO] TestAgent: shutdown complete -13:08.515 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16549 network=tcp -13:08.515 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16549 network=udp -13:08.515 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16550 network=tcp -13:09.018 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.018 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsSpecificDelete_good (0.85s) -=== CONT TestEventList_ACLFilter -2021-10-31T23:13:07.729Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:08.206Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:08.208 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:72af16cd-096a-ff8e-cbfe-ac2317861f3e Address:127.0.0.1:16519}]" -13:08.208 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16519 [Follower]" leader= -13:08.208 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-72af16cd-096a-ff8e-cbfe-ac2317861f3e.dc1 127.0.0.1 -13:08.209 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-72af16cd-096a-ff8e-cbfe-ac2317861f3e 127.0.0.1 -2021-10-31T23:13:08.209Z [INFO] agent.router: Initializing LAN area manager -13:08.209 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16486 network=udp -13:08.209 [INFO] TestAgent.server: Adding LAN server: server="Node-72af16cd-096a-ff8e-cbfe-ac2317861f3e (Addr: tcp/127.0.0.1:16519) (DC: dc1)" -13:08.209 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-72af16cd-096a-ff8e-cbfe-ac2317861f3e.dc1 area=wan -13:08.222 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16486 network=tcp -13:08.222 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16487 network=tcp -13:08.222 [INFO] TestAgent: started state syncer -13:08.222 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16520 network=tcp -13:08.250 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:08.250 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16519 [Candidate]" term=2 -13:08.250 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:08.250 [DEBUG] TestAgent.server.raft: vote granted: from=72af16cd-096a-ff8e-cbfe-ac2317861f3e term=2 tally=1 -13:08.250 [INFO] TestAgent.server.raft: election won: tally=1 -13:08.250 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16519 [Leader]" -13:08.250 [INFO] TestAgent.server: cluster leadership acquired -13:08.250 [INFO] TestAgent.server: New leader elected: payload=Node-72af16cd-096a-ff8e-cbfe-ac2317861f3e -13:08.250 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16519 -13:08.251 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:08.252 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:08.252 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:08.252 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:08.252 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:08.252 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:08.252 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-72af16cd-096a-ff8e-cbfe-ac2317861f3e -13:08.252 [INFO] TestAgent.server: member joined, marking health alive: member=Node-72af16cd-096a-ff8e-cbfe-ac2317861f3e -13:08.253 [INFO] TestAgent.server: federation state anti-entropy synced -13:08.511 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:08.511 [INFO] TestAgent: Synced node info -13:08.525 [INFO] TestAgent: Requesting shutdown -13:08.525 [INFO] TestAgent.server: shutting down server -13:08.525 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:08.525 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:08.525 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:08.525 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:08.525 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:08.525 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:08.525 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:08.525 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:08.525 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:08.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:08.526 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:08.526Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:08.526Z [INFO] agent.router.manager: shutting down -13:08.526 [INFO] TestAgent: consul server down -13:08.526 [INFO] TestAgent: shutdown complete -13:08.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16486 network=tcp -13:08.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16486 network=udp -13:08.526 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16487 network=tcp -13:09.029 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.029 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:08.231Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:08.260Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:08.270 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7f6802fb-53ff-eff2-4cda-3c961a47cf43 Address:127.0.0.1:16512}]" -13:08.271 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7f6802fb-53ff-eff2-4cda-3c961a47cf43.dc1 127.0.0.1 -13:08.271 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7f6802fb-53ff-eff2-4cda-3c961a47cf43 127.0.0.1 -2021-10-31T23:13:08.271Z [INFO] agent.router: Initializing LAN area manager -13:08.271 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16507 network=udp -13:08.272 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16512 [Follower]" leader= -13:08.272 [INFO] TestAgent.server: Adding LAN server: server="Node-7f6802fb-53ff-eff2-4cda-3c961a47cf43 (Addr: tcp/127.0.0.1:16512) (DC: dc1)" -13:08.272 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7f6802fb-53ff-eff2-4cda-3c961a47cf43.dc1 area=wan -13:08.272 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16507 network=tcp -13:08.273 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16508 network=tcp -13:08.273 [INFO] TestAgent: started state syncer -13:08.273 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16513 network=tcp -13:08.333 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:08.334 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16512 [Candidate]" term=2 -13:08.334 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:08.334 [DEBUG] TestAgent.server.raft: vote granted: from=7f6802fb-53ff-eff2-4cda-3c961a47cf43 term=2 tally=1 -13:08.334 [INFO] TestAgent.server.raft: election won: tally=1 -13:08.334 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16512 [Leader]" -13:08.334 [INFO] TestAgent.server: cluster leadership acquired -13:08.334 [INFO] TestAgent.server: New leader elected: payload=Node-7f6802fb-53ff-eff2-4cda-3c961a47cf43 -13:08.334 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16512 -13:08.335 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:08.337 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:08.337 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:08.337 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:08.337 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:08.337 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:08.337 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7f6802fb-53ff-eff2-4cda-3c961a47cf43 -13:08.337 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7f6802fb-53ff-eff2-4cda-3c961a47cf43 -13:08.339 [INFO] TestAgent.server: federation state anti-entropy synced -13:08.526 [INFO] TestAgent: Requesting shutdown -13:08.526 [INFO] TestAgent.server: shutting down server -13:08.526 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:08.526 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:08.526 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:08.526 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:08.526 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:08.526 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:08.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:08.526 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:08.526 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:08.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:08.526 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:08.526 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:08.526Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:08.526Z [INFO] agent.router.manager: shutting down -13:08.526 [INFO] TestAgent: consul server down -13:08.526 [INFO] TestAgent: shutdown complete -13:08.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16507 network=tcp -13:08.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16507 network=udp -13:08.527 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16508 network=tcp -13:09.029 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.029 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceChecks (1.39s) +2022-12-04T05:49:21.625Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.634Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.635 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:372e9875-489e-85ea-c7a6-844c36eaa72a Address:127.0.0.1:23384}]" +49:21.635 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23384 [Follower]" leader= +49:21.636 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-372e9875-489e-85ea-c7a6-844c36eaa72a.dc1 127.0.0.1 +49:21.636 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-372e9875-489e-85ea-c7a6-844c36eaa72a 127.0.0.1 +2022-12-04T05:49:21.636Z [INFO] agent.router: Initializing LAN area manager +49:21.636 [INFO] TestAgent.server: Adding LAN server: server="Node-372e9875-489e-85ea-c7a6-844c36eaa72a (Addr: tcp/127.0.0.1:23384) (DC: dc1)" +49:21.636 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-372e9875-489e-85ea-c7a6-844c36eaa72a.dc1 area=wan +49:21.637 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23302 network=tcp +49:21.637 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23302 network=udp +49:21.637 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23303 network=tcp +49:21.637 [INFO] TestAgent: started state syncer +49:21.637 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23385 network=tcp +49:21.705 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.705 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23384 [Candidate]" term=2 +49:21.705 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.705 [DEBUG] TestAgent.server.raft: vote granted: from=372e9875-489e-85ea-c7a6-844c36eaa72a term=2 tally=1 +49:21.705 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.705 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23384 [Leader]" +49:21.705 [INFO] TestAgent.server: cluster leadership acquired +49:21.705 [INFO] TestAgent.server: New leader elected: payload=Node-372e9875-489e-85ea-c7a6-844c36eaa72a +49:21.706 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23384 +49:21.707 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.709 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.709 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.709 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.709 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.709 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.709 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-372e9875-489e-85ea-c7a6-844c36eaa72a +49:21.709 [INFO] TestAgent.server: member joined, marking health alive: member=Node-372e9875-489e-85ea-c7a6-844c36eaa72a +49:21.709 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.971 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.972 [INFO] TestAgent: Synced node info +49:21.972 [DEBUG] TestAgent: Node info in sync +49:22.141 [INFO] TestAgent: Requesting shutdown +49:22.141 [INFO] TestAgent.server: shutting down server +49:22.141 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.141 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.141 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.141 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.141 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.141 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.141 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.141 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:22.141 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.141 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.141 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:22.141Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.141Z [INFO] agent.router.manager: shutting down +49:22.141 [INFO] TestAgent: consul server down +49:22.141 [INFO] TestAgent: shutdown complete +49:22.141 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23302 network=tcp +49:22.141 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23302 network=udp +49:22.142 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23303 network=tcp +49:22.642 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.642 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceNodes_DistanceSort (1.03s) === CONT TestHealthChecksInState_NodeMetaFilter ---- PASS: TestHealthNodeChecks (0.84s) -=== CONT TestEventList_Filter -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:08.185Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:08.218Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:08.220 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7caf43b1-ca80-67cb-e27d-a43158f4aecd Address:127.0.0.1:16561}]" -13:08.221 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7caf43b1-ca80-67cb-e27d-a43158f4aecd.dc1 127.0.0.1 -13:08.221 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16561 [Follower]" leader= -13:08.230 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7caf43b1-ca80-67cb-e27d-a43158f4aecd 127.0.0.1 -2021-10-31T23:13:08.230Z [INFO] agent.router: Initializing LAN area manager -13:08.230 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16542 network=udp -13:08.230 [INFO] TestAgent.server: Adding LAN server: server="Node-7caf43b1-ca80-67cb-e27d-a43158f4aecd (Addr: tcp/127.0.0.1:16561) (DC: dc1)" -13:08.230 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7caf43b1-ca80-67cb-e27d-a43158f4aecd.dc1 area=wan -13:08.230 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16542 network=tcp -13:08.231 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16543 network=tcp -13:08.231 [INFO] TestAgent: started state syncer -13:08.231 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16562 network=tcp -13:08.289 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:08.290 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16561 [Candidate]" term=2 -13:08.290 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:08.290 [DEBUG] TestAgent.server.raft: vote granted: from=7caf43b1-ca80-67cb-e27d-a43158f4aecd term=2 tally=1 -13:08.290 [INFO] TestAgent.server.raft: election won: tally=1 -13:08.290 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16561 [Leader]" -13:08.290 [INFO] TestAgent.server: cluster leadership acquired -13:08.290 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16561 -13:08.292 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:08.294 [INFO] TestAgent.server: New leader elected: payload=Node-7caf43b1-ca80-67cb-e27d-a43158f4aecd -13:08.302 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:08.302 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:08.302 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:08.302 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:08.302 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:08.302 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7caf43b1-ca80-67cb-e27d-a43158f4aecd -13:08.302 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7caf43b1-ca80-67cb-e27d-a43158f4aecd -13:08.303 [INFO] TestAgent.server: federation state anti-entropy synced -13:08.526 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:08.526 [INFO] TestAgent: Synced node info -13:08.526 [DEBUG] TestAgent: Node info in sync -13:08.659 [INFO] TestAgent: Requesting shutdown -13:08.660 [INFO] TestAgent.server: shutting down server -13:08.660 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:08.660 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:08.660 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:08.660 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:08.660 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:08.660 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:08.660 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:08.660 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:08.660 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:08.660 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:08.660 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:08.660Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:08.660Z [INFO] agent.router.manager: shutting down -13:08.660 [INFO] TestAgent: consul server down -13:08.660 [INFO] TestAgent: shutdown complete -13:08.660 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16542 network=tcp -13:08.660 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16542 network=udp -13:08.660 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16543 network=tcp -13:09.161 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.186 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthIngressServiceNodes (1.45s) - --- PASS: TestHealthIngressServiceNodes/associated_service (0.00s) - --- PASS: TestHealthIngressServiceNodes/non-associated_service (0.00s) -=== CONT TestIntentionsSpecificGet_invalidId -2021-10-31T23:13:08.213Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:08.257Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:08.259 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d20ccccf-0187-9ff6-5d23-34af01b056ff Address:127.0.0.1:16547}]" -13:08.260 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d20ccccf-0187-9ff6-5d23-34af01b056ff.dc1 127.0.0.1 -13:08.260 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d20ccccf-0187-9ff6-5d23-34af01b056ff 127.0.0.1 -2021-10-31T23:13:08.260Z [INFO] agent.router: Initializing LAN area manager -13:08.261 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16500 network=udp -13:08.261 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16547 [Follower]" leader= -13:08.261 [INFO] TestAgent.server: Adding LAN server: server="Node-d20ccccf-0187-9ff6-5d23-34af01b056ff (Addr: tcp/127.0.0.1:16547) (DC: dc1)" -13:08.261 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d20ccccf-0187-9ff6-5d23-34af01b056ff.dc1 area=wan -13:08.261 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16500 network=tcp -13:08.262 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16501 network=tcp -13:08.262 [INFO] TestAgent: started state syncer -13:08.262 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16548 network=tcp -13:08.309 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:08.309 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16547 [Candidate]" term=2 -13:08.309 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:08.316 [DEBUG] TestAgent.server.raft: vote granted: from=d20ccccf-0187-9ff6-5d23-34af01b056ff term=2 tally=1 -13:08.316 [INFO] TestAgent.server.raft: election won: tally=1 -13:08.316 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16547 [Leader]" -13:08.316 [INFO] TestAgent.server: cluster leadership acquired -13:08.317 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16547 -13:08.322 [INFO] TestAgent.server: New leader elected: payload=Node-d20ccccf-0187-9ff6-5d23-34af01b056ff -13:08.330 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:08.332 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:08.332 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:08.332 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:08.332 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:08.332 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:08.332 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d20ccccf-0187-9ff6-5d23-34af01b056ff -13:08.332 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d20ccccf-0187-9ff6-5d23-34af01b056ff -13:08.334 [INFO] TestAgent.server: federation state anti-entropy synced -13:08.658 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:08.658 [INFO] TestAgent: Synced node info -13:08.670 [INFO] TestAgent: Requesting shutdown -13:08.670 [INFO] TestAgent.server: shutting down server -13:08.670 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:08.670 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:08.670 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:08.670 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:08.670 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:08.671 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:08.671 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:08.671 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:08.671 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:08.671 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:08.671 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:08.671Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:08.671Z [INFO] agent.router.manager: shutting down -13:08.671 [INFO] TestAgent: consul server down -13:08.671 [INFO] TestAgent: shutdown complete -13:08.671 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16500 network=tcp -13:08.671 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16500 network=udp -13:08.671 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16501 network=tcp -13:09.174 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.190 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthConnectServiceNodes (1.02s) -=== CONT TestEventList -2021-10-31T23:13:08.227Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:08.263Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:08.264 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4402f472-bf1a-4e76-790a-5a88d6fb2f0e Address:127.0.0.1:16533}]" -13:08.265 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4402f472-bf1a-4e76-790a-5a88d6fb2f0e.dc1 127.0.0.1 -13:08.265 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4402f472-bf1a-4e76-790a-5a88d6fb2f0e 127.0.0.1 -13:08.265 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16533 [Follower]" leader= -2021-10-31T23:13:08.275Z [INFO] agent.router: Initializing LAN area manager -13:08.294 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4402f472-bf1a-4e76-790a-5a88d6fb2f0e.dc1 area=wan -13:08.294 [INFO] TestAgent.server: Adding LAN server: server="Node-4402f472-bf1a-4e76-790a-5a88d6fb2f0e (Addr: tcp/127.0.0.1:16533) (DC: dc1)" -13:08.294 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16514 network=tcp -13:08.298 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16514 network=udp -13:08.299 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16515 network=tcp -13:08.299 [INFO] TestAgent: started state syncer -13:08.299 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16534 network=tcp -13:08.305 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:08.305 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16533 [Candidate]" term=2 -13:08.306 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:08.306 [DEBUG] TestAgent.server.raft: vote granted: from=4402f472-bf1a-4e76-790a-5a88d6fb2f0e term=2 tally=1 -13:08.306 [INFO] TestAgent.server.raft: election won: tally=1 -13:08.306 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16533 [Leader]" -13:08.306 [INFO] TestAgent.server: cluster leadership acquired -13:08.306 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16533 -13:08.307 [INFO] TestAgent.server: New leader elected: payload=Node-4402f472-bf1a-4e76-790a-5a88d6fb2f0e -13:08.308 [INFO] TestAgent: Synced node info -13:08.308 [DEBUG] TestAgent: Node info in sync -13:08.308 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:08.311 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:08.311 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:08.311 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:08.311 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:08.311 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:08.311 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4402f472-bf1a-4e76-790a-5a88d6fb2f0e -13:08.311 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4402f472-bf1a-4e76-790a-5a88d6fb2f0e -13:08.312 [INFO] TestAgent.server: federation state anti-entropy synced -13:08.683 [INFO] TestAgent: Requesting shutdown -13:08.683 [INFO] TestAgent.server: shutting down server -13:08.683 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:08.683 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:08.683 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:08.683 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:08.683 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:08.683 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:08.683 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:08.683 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:08.683 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:08.683 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:08.683 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:08.683Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:08.683Z [INFO] agent.router.manager: shutting down -13:08.683 [INFO] TestAgent: consul server down -13:08.683 [INFO] TestAgent: shutdown complete -13:08.683 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16514 network=tcp -13:08.683 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16514 network=udp -13:08.683 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16515 network=tcp -13:09.193 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.193 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceNodes_CheckType (1.00s) +2022-12-04T05:49:21.836Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.849Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.850 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9ebff7cc-e167-39b3-3aef-ecf140f22cf4 Address:127.0.0.1:23349}]" +49:21.850 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23349 [Follower]" leader= +49:21.850 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9ebff7cc-e167-39b3-3aef-ecf140f22cf4.dc1 127.0.0.1 +49:21.851 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9ebff7cc-e167-39b3-3aef-ecf140f22cf4 127.0.0.1 +2022-12-04T05:49:21.851Z [INFO] agent.router: Initializing LAN area manager +49:21.851 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9ebff7cc-e167-39b3-3aef-ecf140f22cf4.dc1 area=wan +49:21.851 [INFO] TestAgent.server: Adding LAN server: server="Node-9ebff7cc-e167-39b3-3aef-ecf140f22cf4 (Addr: tcp/127.0.0.1:23349) (DC: dc1)" +49:21.851 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23379 network=udp +49:21.851 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23379 network=tcp +49:21.852 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23380 network=tcp +49:21.852 [INFO] TestAgent: started state syncer +49:21.852 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23350 network=tcp +49:21.909 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.910 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23349 [Candidate]" term=2 +49:21.910 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.910 [DEBUG] TestAgent.server.raft: vote granted: from=9ebff7cc-e167-39b3-3aef-ecf140f22cf4 term=2 tally=1 +49:21.910 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.910 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23349 [Leader]" +49:21.910 [INFO] TestAgent.server: cluster leadership acquired +49:21.910 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23349 +49:21.910 [INFO] TestAgent.server: New leader elected: payload=Node-9ebff7cc-e167-39b3-3aef-ecf140f22cf4 +49:21.911 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.912 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.912 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.912 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.912 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.912 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.912 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9ebff7cc-e167-39b3-3aef-ecf140f22cf4 +49:21.912 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9ebff7cc-e167-39b3-3aef-ecf140f22cf4 +49:21.913 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.971 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.972 [INFO] TestAgent: Synced node info +49:22.193 [INFO] TestAgent: Requesting shutdown +49:22.193 [INFO] TestAgent.server: shutting down server +49:22.193 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.193 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.193 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.193 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.193 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.193 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.193 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.193 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.193 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.193 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.193 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.193Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.193Z [INFO] agent.router.manager: shutting down +49:22.194 [INFO] TestAgent: consul server down +49:22.194 [INFO] TestAgent: shutdown complete +49:22.194 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23379 network=tcp +49:22.194 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23379 network=udp +49:22.194 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23380 network=tcp +49:22.694 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.694 [INFO] TestAgent: Endpoints down +--- PASS: TestKVSEndpoint_AcquireRelease (0.87s) +=== CONT TestEventList_Blocking +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:22.112Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.121Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.122 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:81e732a2-f55b-7b51-f7d6-61c4c91bc859 Address:127.0.0.1:23307}]" +49:22.122 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23307 [Follower]" leader= +49:22.123 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-81e732a2-f55b-7b51-f7d6-61c4c91bc859.dc1 127.0.0.1 +49:22.123 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-81e732a2-f55b-7b51-f7d6-61c4c91bc859 127.0.0.1 +2022-12-04T05:49:22.123Z [INFO] agent.router: Initializing LAN area manager +49:22.123 [INFO] TestAgent.server: Adding LAN server: server="Node-81e732a2-f55b-7b51-f7d6-61c4c91bc859 (Addr: tcp/127.0.0.1:23307) (DC: dc1)" +49:22.123 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-81e732a2-f55b-7b51-f7d6-61c4c91bc859.dc1 area=wan +49:22.123 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23316 network=udp +49:22.123 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23316 network=tcp +49:22.124 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23317 network=tcp +49:22.124 [INFO] TestAgent: started state syncer +49:22.124 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23308 network=tcp +49:22.159 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.159 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23307 [Candidate]" term=2 +49:22.159 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.159 [DEBUG] TestAgent.server.raft: vote granted: from=81e732a2-f55b-7b51-f7d6-61c4c91bc859 term=2 tally=1 +49:22.159 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.159 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23307 [Leader]" +49:22.159 [INFO] TestAgent.server: cluster leadership acquired +49:22.159 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23307 +49:22.160 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.160 [INFO] TestAgent.server: New leader elected: payload=Node-81e732a2-f55b-7b51-f7d6-61c4c91bc859 +49:22.161 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.161 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.161 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.161 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.161 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.161 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-81e732a2-f55b-7b51-f7d6-61c4c91bc859 +49:22.161 [INFO] TestAgent.server: member joined, marking health alive: member=Node-81e732a2-f55b-7b51-f7d6-61c4c91bc859 +49:22.161 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.309 [INFO] TestAgent: Requesting shutdown +49:22.309 [INFO] TestAgent.server: shutting down server +49:22.309 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.309 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.309 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.309 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.309 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.309 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.309 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.309 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.309 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.309 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.309 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:22.309 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.309Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.309Z [INFO] agent.router.manager: shutting down +49:22.309 [INFO] TestAgent: consul server down +49:22.309 [INFO] TestAgent: shutdown complete +49:22.309 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23316 network=tcp +49:22.309 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23316 network=udp +49:22.309 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23317 network=tcp +49:22.811 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.811 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceNodes (0.73s) +=== CONT TestEventList_ACLFilter +2022-12-04T05:49:22.226Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.235Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.237 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:07141e2d-daad-800a-33e9-d5b1adeb5275 Address:127.0.0.1:23419}]" +49:22.237 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23419 [Follower]" leader= +49:22.237 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-07141e2d-daad-800a-33e9-d5b1adeb5275.dc1 127.0.0.1 +49:22.238 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-07141e2d-daad-800a-33e9-d5b1adeb5275 127.0.0.1 +2022-12-04T05:49:22.238Z [INFO] agent.router: Initializing LAN area manager +49:22.238 [INFO] TestAgent.server: Adding LAN server: server="Node-07141e2d-daad-800a-33e9-d5b1adeb5275 (Addr: tcp/127.0.0.1:23419) (DC: dc1)" +49:22.238 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-07141e2d-daad-800a-33e9-d5b1adeb5275.dc1 area=wan +49:22.238 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23400 network=udp +49:22.238 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23400 network=tcp +49:22.238 [WARN] TestAgent: Replacing socket: path=/tmp/consul-test/TestHTTPServer_UnixSocket_FileExists-consul680484069/test.sock +49:22.239 [INFO] TestAgent: Started HTTP server: address=/tmp/consul-test/TestHTTPServer_UnixSocket_FileExists-consul680484069/test.sock network=unix +49:22.239 [INFO] TestAgent: started state syncer +49:22.239 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23420 network=tcp +49:22.298 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.298 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23419 [Candidate]" term=2 +49:22.298 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.298 [DEBUG] TestAgent.server.raft: vote granted: from=07141e2d-daad-800a-33e9-d5b1adeb5275 term=2 tally=1 +49:22.298 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.298 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23419 [Leader]" +49:22.298 [INFO] TestAgent.server: cluster leadership acquired +49:22.298 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23419 +49:22.298 [INFO] TestAgent.server: New leader elected: payload=Node-07141e2d-daad-800a-33e9-d5b1adeb5275 +49:22.299 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.300 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.300 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.300 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.300 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.300 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.300 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-07141e2d-daad-800a-33e9-d5b1adeb5275 +49:22.300 [INFO] TestAgent.server: member joined, marking health alive: member=Node-07141e2d-daad-800a-33e9-d5b1adeb5275 +49:22.301 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.347 [INFO] TestAgent: Requesting shutdown +49:22.347 [INFO] TestAgent.server: shutting down server +49:22.347 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.347 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.347 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.347 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.347 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.347 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:22.347 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.347 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.347 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.347 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.347 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:22.347 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:22.347Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.347Z [INFO] agent.router.manager: shutting down +49:22.348 [INFO] TestAgent: consul server down +49:22.348 [INFO] TestAgent: shutdown complete +49:22.348 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23400 network=tcp +49:22.348 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23400 network=udp +49:22.348 [INFO] TestAgent: Stopping server: protocol=HTTP address=/tmp/consul-test/TestHTTPServer_UnixSocket_FileExists-consul680484069/test.sock network=unix +49:22.848 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.848 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPServer_UnixSocket_FileExists (0.63s) === CONT TestUUIDToUint64 --- PASS: TestUUIDToUint64 (0.00s) -=== CONT TestIntentionsSpecificGet_good -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestEventList_ACLFilter/no_token -=== RUN TestEventList_ACLFilter/root_token -2021-10-31T23:13:08.700Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:08.747Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:08.748 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7f5d9f37-cc77-453e-845e-910aae12e31d Address:127.0.0.1:16568}]" -13:08.749 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7f5d9f37-cc77-453e-845e-910aae12e31d.dc1 127.0.0.1 -13:08.749 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16568 [Follower]" leader= -13:08.754 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7f5d9f37-cc77-453e-845e-910aae12e31d 127.0.0.1 -2021-10-31T23:13:08.754Z [INFO] agent.router: Initializing LAN area manager -13:08.755 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16528 network=udp -13:08.755 [INFO] TestAgent.server: Adding LAN server: server="Node-7f5d9f37-cc77-453e-845e-910aae12e31d (Addr: tcp/127.0.0.1:16568) (DC: dc1)" -13:08.755 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7f5d9f37-cc77-453e-845e-910aae12e31d.dc1 area=wan -13:08.791 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16528 network=tcp -13:08.792 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16529 network=tcp -13:08.793 [INFO] TestAgent: started state syncer -13:08.793 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16569 network=tcp -13:08.796 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:08.796 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16568 [Candidate]" term=2 -13:08.796 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:08.796 [DEBUG] TestAgent.server.raft: vote granted: from=7f5d9f37-cc77-453e-845e-910aae12e31d term=2 tally=1 -13:08.796 [INFO] TestAgent.server.raft: election won: tally=1 -13:08.796 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16568 [Leader]" -13:08.796 [INFO] TestAgent.server: cluster leadership acquired -13:08.797 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16568 -13:08.797 [INFO] TestAgent.server: New leader elected: payload=Node-7f5d9f37-cc77-453e-845e-910aae12e31d -13:08.799 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:08.818 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:08.818 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:08.818 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:08.818 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:08.818 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:08.818 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7f5d9f37-cc77-453e-845e-910aae12e31d -13:08.818 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7f5d9f37-cc77-453e-845e-910aae12e31d -13:08.820 [INFO] TestAgent.server: federation state anti-entropy synced -13:08.998 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:08.999 [INFO] TestAgent: Synced node info -13:08.999 [DEBUG] TestAgent: Node info in sync -13:09.040 [INFO] TestAgent: Requesting shutdown -13:09.040 [INFO] TestAgent.server: shutting down server -13:09.040 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:09.040 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:09.040 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:09.040 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:09.040 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:09.040 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:09.040 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:09.040 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:09.040 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:09.040 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:09.041 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:09.041Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:09.041Z [INFO] agent.router.manager: shutting down -13:09.041 [INFO] TestAgent: consul server down -13:09.041 [INFO] TestAgent: shutdown complete -13:09.041 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16528 network=tcp -13:09.041 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16528 network=udp -13:09.041 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16529 network=tcp -13:09.541 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.541 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_InmemKeyrings/client_with_keys -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:08.732Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:08.789Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:08.816 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:66f56d40-45b4-3a6a-fe14-c8eae95c8137 Address:127.0.0.1:16596}]" -13:08.817 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16596 [Follower]" leader= -13:08.817 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-66f56d40-45b4-3a6a-fe14-c8eae95c8137.dc1 127.0.0.1 -13:08.818 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-66f56d40-45b4-3a6a-fe14-c8eae95c8137 127.0.0.1 -2021-10-31T23:13:08.818Z [INFO] agent.router: Initializing LAN area manager -13:08.819 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16591 network=udp -13:08.819 [INFO] TestAgent.server: Adding LAN server: server="Node-66f56d40-45b4-3a6a-fe14-c8eae95c8137 (Addr: tcp/127.0.0.1:16596) (DC: dc1)" -13:08.819 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-66f56d40-45b4-3a6a-fe14-c8eae95c8137.dc1 area=wan -13:08.819 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16591 network=tcp -13:08.821 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16592 network=tcp -13:08.843 [INFO] TestAgent: started state syncer -13:08.874 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16597 network=tcp -13:08.884 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:08.884 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16596 [Candidate]" term=2 -13:08.885 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:08.885 [DEBUG] TestAgent.server.raft: vote granted: from=66f56d40-45b4-3a6a-fe14-c8eae95c8137 term=2 tally=1 -13:08.885 [INFO] TestAgent.server.raft: election won: tally=1 -13:08.885 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16596 [Leader]" -13:08.885 [INFO] TestAgent.server: cluster leadership acquired -13:08.885 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16596 -13:08.885 [INFO] TestAgent.server: New leader elected: payload=Node-66f56d40-45b4-3a6a-fe14-c8eae95c8137 -13:08.886 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:08.888 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:08.888 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:08.888 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:08.888 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:08.888 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:08.888 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-66f56d40-45b4-3a6a-fe14-c8eae95c8137 -13:08.888 [INFO] TestAgent.server: member joined, marking health alive: member=Node-66f56d40-45b4-3a6a-fe14-c8eae95c8137 -13:08.888 [INFO] TestAgent.server: federation state anti-entropy synced -13:09.081 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:09.082 [INFO] TestAgent: Synced node info -13:09.195 [INFO] TestAgent: Requesting shutdown -13:09.195 [INFO] TestAgent.server: shutting down server -13:09.195 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:09.195 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:09.195 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:09.195 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:09.195 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:09.195 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:09.195 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:09.195 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:09.195 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:09.195 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:09.196 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:09.196Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:09.196Z [INFO] agent.router.manager: shutting down -13:09.196 [INFO] TestAgent: consul server down -13:09.196 [INFO] TestAgent: shutdown complete -13:09.196 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16591 network=tcp -13:09.196 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16591 network=udp -13:09.196 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16592 network=tcp -13:09.697 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.697 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthChecksInState_Filter (1.01s) === CONT TestEventList_EventBufOrder -2021-10-31T23:13:08.911Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:08.959Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:08.961 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5a600c0d-09d4-76ac-a187-c816a9b171a8 Address:127.0.0.1:16603}]" -13:08.962 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5a600c0d-09d4-76ac-a187-c816a9b171a8.dc1 127.0.0.1 -13:08.962 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16603 [Follower]" leader= -13:08.970 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5a600c0d-09d4-76ac-a187-c816a9b171a8 127.0.0.1 -2021-10-31T23:13:08.970Z [INFO] agent.router: Initializing LAN area manager -13:08.970 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16598 network=udp -13:08.971 [INFO] TestAgent.server: Adding LAN server: server="Node-5a600c0d-09d4-76ac-a187-c816a9b171a8 (Addr: tcp/127.0.0.1:16603) (DC: dc1)" -13:08.971 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5a600c0d-09d4-76ac-a187-c816a9b171a8.dc1 area=wan -13:08.971 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16598 network=tcp -13:08.971 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16599 network=tcp -13:08.971 [INFO] TestAgent: started state syncer -13:08.972 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16604 network=tcp -13:09.018 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:09.018 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16603 [Candidate]" term=2 -13:09.018 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:09.018 [DEBUG] TestAgent.server.raft: vote granted: from=5a600c0d-09d4-76ac-a187-c816a9b171a8 term=2 tally=1 -13:09.018 [INFO] TestAgent.server.raft: election won: tally=1 -13:09.018 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16603 [Leader]" -13:09.019 [INFO] TestAgent.server: cluster leadership acquired -13:09.019 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16603 -13:09.020 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:09.021 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:09.021 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:09.021 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:09.021 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:09.021 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:09.021 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a600c0d-09d4-76ac-a187-c816a9b171a8 -13:09.021 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5a600c0d-09d4-76ac-a187-c816a9b171a8 -13:09.021 [INFO] TestAgent.server: New leader elected: payload=Node-5a600c0d-09d4-76ac-a187-c816a9b171a8 -13:09.034 [INFO] TestAgent.server: federation state anti-entropy synced -13:09.199 [INFO] TestAgent: Requesting shutdown -13:09.199 [INFO] TestAgent.server: shutting down server -13:09.199 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:09.199 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:09.199 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:09.199 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:09.199 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:09.200 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:09.200 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:09.200 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:09.200 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:09.200 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:09.200 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:09.200 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:09.200Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:09.200Z [INFO] agent.router.manager: shutting down -13:09.200 [INFO] TestAgent: consul server down -13:09.200 [INFO] TestAgent: shutdown complete -13:09.200 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16598 network=tcp -13:09.200 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16598 network=udp -13:09.200 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16599 network=tcp -13:09.701 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.701 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthNodeChecks_Filtering (0.83s) -=== CONT TestIntentionsCreate_noBody -2021-10-31T23:13:08.735Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:08.822Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:08.838 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5397429b-cf8d-fee1-4056-6033aea8584f Address:127.0.0.1:16589}]" -13:08.872 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16589 [Follower]" leader= -13:08.872 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5397429b-cf8d-fee1-4056-6033aea8584f.dc1 127.0.0.1 -13:08.873 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5397429b-cf8d-fee1-4056-6033aea8584f 127.0.0.1 -2021-10-31T23:13:08.873Z [INFO] agent.router: Initializing LAN area manager -13:08.874 [INFO] TestAgent.server: Adding LAN server: server="Node-5397429b-cf8d-fee1-4056-6033aea8584f (Addr: tcp/127.0.0.1:16589) (DC: dc1)" -13:08.874 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5397429b-cf8d-fee1-4056-6033aea8584f.dc1 area=wan -13:08.885 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16584 network=udp -13:08.885 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16584 network=tcp -13:08.893 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16585 network=tcp -13:08.893 [INFO] TestAgent: started state syncer -13:08.893 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16590 network=tcp -13:08.937 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:08.937 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16589 [Candidate]" term=2 -13:08.937 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:08.937 [DEBUG] TestAgent.server.raft: vote granted: from=5397429b-cf8d-fee1-4056-6033aea8584f term=2 tally=1 -13:08.937 [INFO] TestAgent.server.raft: election won: tally=1 -13:08.937 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16589 [Leader]" -13:08.937 [INFO] TestAgent.server: cluster leadership acquired -13:08.938 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16589 -13:08.938 [INFO] TestAgent.server: New leader elected: payload=Node-5397429b-cf8d-fee1-4056-6033aea8584f -13:08.938 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:08.940 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:08.940 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:08.940 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:08.940 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:08.940 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:08.940 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5397429b-cf8d-fee1-4056-6033aea8584f -13:08.940 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5397429b-cf8d-fee1-4056-6033aea8584f -13:08.941 [INFO] TestAgent.server: federation state anti-entropy synced -13:09.018 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:09.018 [INFO] TestAgent: Synced node info -13:09.207 [INFO] TestAgent: Requesting shutdown -13:09.207 [INFO] TestAgent.server: shutting down server -13:09.207 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:09.207 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:09.207 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:09.207 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:09.207 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:09.207 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:09.207 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:09.207 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:09.207 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:09.207 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:09.207 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:09.207Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:09.207Z [INFO] agent.router.manager: shutting down -13:09.208 [INFO] TestAgent: consul server down -13:09.208 [INFO] TestAgent: shutdown complete -13:09.208 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16584 network=tcp -13:09.208 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16584 network=udp -13:09.208 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16585 network=tcp -13:09.709 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.709 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsSpecificUpdate_good (1.03s) +2022-12-04T05:49:22.002Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.014Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.017 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a471bf0f-6308-1020-2648-a88d88a38b35 Address:127.0.0.1:23356}]" +49:22.017 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23356 [Follower]" leader= +49:22.017 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a471bf0f-6308-1020-2648-a88d88a38b35.dc1 127.0.0.1 +49:22.018 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a471bf0f-6308-1020-2648-a88d88a38b35 127.0.0.1 +2022-12-04T05:49:22.018Z [INFO] agent.router: Initializing LAN area manager +49:22.018 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23337 network=udp +49:22.018 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a471bf0f-6308-1020-2648-a88d88a38b35.dc1 area=wan +49:22.018 [INFO] TestAgent.server: Adding LAN server: server="Node-a471bf0f-6308-1020-2648-a88d88a38b35 (Addr: tcp/127.0.0.1:23356) (DC: dc1)" +49:22.018 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23337 network=tcp +49:22.019 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23338 network=tcp +49:22.019 [INFO] TestAgent: started state syncer +49:22.019 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23357 network=tcp +49:22.077 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.077 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23356 [Candidate]" term=2 +49:22.077 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.077 [DEBUG] TestAgent.server.raft: vote granted: from=a471bf0f-6308-1020-2648-a88d88a38b35 term=2 tally=1 +49:22.077 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.077 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23356 [Leader]" +49:22.077 [INFO] TestAgent.server: cluster leadership acquired +49:22.078 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23356 +49:22.078 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.079 [INFO] TestAgent.server: New leader elected: payload=Node-a471bf0f-6308-1020-2648-a88d88a38b35 +49:22.089 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.089 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.089 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.089 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.089 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.089 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a471bf0f-6308-1020-2648-a88d88a38b35 +49:22.089 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a471bf0f-6308-1020-2648-a88d88a38b35 +49:22.091 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.352 [INFO] TestAgent: Requesting shutdown +49:22.352 [INFO] TestAgent.server: shutting down server +49:22.352 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.352 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.352 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.352 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.352 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.352 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.352 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.352 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:22.352 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.352 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.352 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.352 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.352Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.352Z [INFO] agent.router.manager: shutting down +49:22.352 [INFO] TestAgent: consul server down +49:22.352 [INFO] TestAgent: shutdown complete +49:22.352 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23337 network=tcp +49:22.352 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23337 network=udp +49:22.352 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23338 network=tcp +49:22.853 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.853 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceNodes_Filter (0.87s) +=== CONT TestEventList_Filter +2022-12-04T05:49:21.426Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:21.426Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.436Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:21.436Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.438 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:80d42ac1-39d8-62c2-a2a9-b1d38fd5a65b Address:127.0.0.1:23279}]" +49:21.438 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23279 [Follower]" leader= +49:21.438 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-80d42ac1-39d8-62c2-a2a9-b1d38fd5a65b.dc2 127.0.0.1 +49:21.439 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-80d42ac1-39d8-62c2-a2a9-b1d38fd5a65b 127.0.0.1 +2022-12-04T05:49:21.439Z [INFO] agent.router: Initializing LAN area manager +49:21.439 [INFO] TestAgent.server: Adding LAN server: server="Node-80d42ac1-39d8-62c2-a2a9-b1d38fd5a65b (Addr: tcp/127.0.0.1:23279) (DC: dc2)" +49:21.439 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=udp +49:21.439 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-80d42ac1-39d8-62c2-a2a9-b1d38fd5a65b.dc2 area=wan +49:21.439 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=tcp +49:21.439 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23275 network=tcp +49:21.440 [INFO] TestAgent: started state syncer +49:21.440 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23280 network=tcp +49:21.476 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.476 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23279 [Candidate]" term=2 +49:21.476 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.476 [DEBUG] TestAgent.server.raft: vote granted: from=80d42ac1-39d8-62c2-a2a9-b1d38fd5a65b term=2 tally=1 +49:21.476 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.476 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23279 [Leader]" +49:21.476 [INFO] TestAgent.server: cluster leadership acquired +49:21.476 [INFO] TestAgent.server: initializing acls +49:21.476 [INFO] TestAgent.server: New leader elected: payload=Node-80d42ac1-39d8-62c2-a2a9-b1d38fd5a65b +49:21.477 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:21.477 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:21.477 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:21.477 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:21.477 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-80d42ac1-39d8-62c2-a2a9-b1d38fd5a65b +49:21.477 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-80d42ac1-39d8-62c2-a2a9-b1d38fd5a65b.dc2 +49:21.477 [INFO] TestAgent.server: Updating LAN server: server="Node-80d42ac1-39d8-62c2-a2a9-b1d38fd5a65b (Addr: tcp/127.0.0.1:23279) (DC: dc2)" +49:21.477 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-80d42ac1-39d8-62c2-a2a9-b1d38fd5a65b.dc2 area=wan +49:21.477 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.478 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.478 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.478 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.478 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.478 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.478 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-80d42ac1-39d8-62c2-a2a9-b1d38fd5a65b +49:21.478 [INFO] TestAgent.server: member joined, marking health alive: member=Node-80d42ac1-39d8-62c2-a2a9-b1d38fd5a65b +49:21.479 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-80d42ac1-39d8-62c2-a2a9-b1d38fd5a65b +49:21.479 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.515 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.515 [INFO] TestAgent: Synced node info +49:21.887 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:24242] +49:21.888 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:24242 +49:21.888 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3d7f1831-b5f3-79d8-7af4-9c352fed27c9.dc1 127.0.0.1 +49:21.888 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +49:21.889 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3d7f1831-b5f3-79d8-7af4-9c352fed27c9.dc1 area=wan +49:21.890 [INFO] TestAgent: Requesting shutdown +49:21.890 [INFO] TestAgent.server: shutting down server +49:21.890 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.890 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.890 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.890 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.890 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:21.890 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:21.890 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.890 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.890 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.890 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:21.890 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.890 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.890 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.890 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:21.891 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +2022-12-04T05:49:21.891Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.891Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.891Z [INFO] agent.router.manager: shutting down +49:21.891 [INFO] TestAgent: consul server down +49:21.891 [INFO] TestAgent: shutdown complete +49:21.891 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=tcp +49:21.891 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=udp +49:21.891 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23275 network=tcp +49:22.391 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.391 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:21.041Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:21.041Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.059Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:21.059Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.061 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3d7f1831-b5f3-79d8-7af4-9c352fed27c9 Address:127.0.0.1:24243}]" +49:21.061 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24243 [Follower]" leader= +49:21.061 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3d7f1831-b5f3-79d8-7af4-9c352fed27c9.dc1 127.0.0.1 +49:21.062 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3d7f1831-b5f3-79d8-7af4-9c352fed27c9 127.0.0.1 +2022-12-04T05:49:21.062Z [INFO] agent.router: Initializing LAN area manager +49:21.062 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24238 network=udp +49:21.062 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3d7f1831-b5f3-79d8-7af4-9c352fed27c9.dc1 area=wan +49:21.062 [INFO] TestAgent.server: Adding LAN server: server="Node-3d7f1831-b5f3-79d8-7af4-9c352fed27c9 (Addr: tcp/127.0.0.1:24243) (DC: dc1)" +49:21.062 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24238 network=tcp +49:21.063 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24239 network=tcp +49:21.063 [INFO] TestAgent: started state syncer +49:21.063 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24244 network=tcp +49:21.107 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.107 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24243 [Candidate]" term=2 +49:21.107 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.107 [DEBUG] TestAgent.server.raft: vote granted: from=3d7f1831-b5f3-79d8-7af4-9c352fed27c9 term=2 tally=1 +49:21.107 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.107 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24243 [Leader]" +49:21.107 [INFO] TestAgent.server: cluster leadership acquired +49:21.107 [INFO] TestAgent.server: initializing acls +49:21.107 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:21.107 [INFO] TestAgent.server: New leader elected: payload=Node-3d7f1831-b5f3-79d8-7af4-9c352fed27c9 +49:21.107 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:21.107 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:21.108 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:21.108 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-3d7f1831-b5f3-79d8-7af4-9c352fed27c9 +49:21.108 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-3d7f1831-b5f3-79d8-7af4-9c352fed27c9.dc1 +49:21.108 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-3d7f1831-b5f3-79d8-7af4-9c352fed27c9.dc1 area=wan +49:21.108 [INFO] TestAgent.server: Updating LAN server: server="Node-3d7f1831-b5f3-79d8-7af4-9c352fed27c9 (Addr: tcp/127.0.0.1:24243) (DC: dc1)" +49:21.108 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.110 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.110 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.110 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.110 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.110 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.110 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3d7f1831-b5f3-79d8-7af4-9c352fed27c9 +49:21.110 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3d7f1831-b5f3-79d8-7af4-9c352fed27c9 +49:21.111 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.111 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3d7f1831-b5f3-79d8-7af4-9c352fed27c9 +49:21.273 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.274 [INFO] TestAgent: Synced node info +49:21.274 [DEBUG] TestAgent: Node info in sync +49:21.888 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:32788 +49:21.888 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-80d42ac1-39d8-62c2-a2a9-b1d38fd5a65b.dc2 127.0.0.1 +49:21.888 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-80d42ac1-39d8-62c2-a2a9-b1d38fd5a65b.dc2 area=wan +49:22.391 [INFO] TestAgent: Requesting shutdown +49:22.391 [INFO] TestAgent.server: shutting down server +49:22.391 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:22.391 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:22.391 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.391 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.391 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.391 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.391 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.391 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.391 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.391 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:22.391 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:22.391 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.391 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.391 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.391 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.391Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.391Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.391Z [INFO] agent.router.manager: shutting down +49:22.392 [INFO] TestAgent: consul server down +49:22.392 [INFO] TestAgent: shutdown complete +49:22.392 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24238 network=tcp +49:22.392 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24238 network=udp +49:22.392 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24239 network=tcp +49:22.892 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.892 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceNodes_WanTranslation (1.87s) +=== CONT TestEventList +2022-12-04T05:49:22.210Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.221Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.223 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:18eb49d8-49d5-e7c1-d909-bf69547deb86 Address:127.0.0.1:23398}]" +49:22.223 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23398 [Follower]" leader= +49:22.223 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-18eb49d8-49d5-e7c1-d909-bf69547deb86.dc1 127.0.0.1 +49:22.224 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-18eb49d8-49d5-e7c1-d909-bf69547deb86 127.0.0.1 +2022-12-04T05:49:22.224Z [INFO] agent.router: Initializing LAN area manager +49:22.224 [INFO] TestAgent.server: Adding LAN server: server="Node-18eb49d8-49d5-e7c1-d909-bf69547deb86 (Addr: tcp/127.0.0.1:23398) (DC: dc1)" +49:22.224 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23372 network=udp +49:22.224 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-18eb49d8-49d5-e7c1-d909-bf69547deb86.dc1 area=wan +49:22.224 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23372 network=tcp +49:22.225 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23373 network=tcp +49:22.225 [INFO] TestAgent: started state syncer +49:22.225 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23399 network=tcp +49:22.266 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.266 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23398 [Candidate]" term=2 +49:22.266 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.266 [DEBUG] TestAgent.server.raft: vote granted: from=18eb49d8-49d5-e7c1-d909-bf69547deb86 term=2 tally=1 +49:22.266 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.266 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23398 [Leader]" +49:22.266 [INFO] TestAgent.server: cluster leadership acquired +49:22.266 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23398 +49:22.266 [INFO] TestAgent.server: New leader elected: payload=Node-18eb49d8-49d5-e7c1-d909-bf69547deb86 +49:22.267 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.268 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.268 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.268 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.268 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.268 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.268 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-18eb49d8-49d5-e7c1-d909-bf69547deb86 +49:22.268 [INFO] TestAgent.server: member joined, marking health alive: member=Node-18eb49d8-49d5-e7c1-d909-bf69547deb86 +49:22.269 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.424 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:22.424 [INFO] TestAgent: Synced node info +49:22.439 [INFO] TestAgent: Requesting shutdown +49:22.440 [INFO] TestAgent.server: shutting down server +49:22.440 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.440 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.440 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.440 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.440 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.440 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.440 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.440 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.440 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.440 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:22.440 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:49:22.440Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.440Z [INFO] agent.router.manager: shutting down +49:22.440 [INFO] TestAgent: consul server down +49:22.440 [INFO] TestAgent: shutdown complete +49:22.440 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23372 network=tcp +49:22.440 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23372 network=udp +49:22.440 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23373 network=tcp +49:22.940 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.940 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceChecks (0.74s) +=== CONT TestDNS_Compression_Query +2022-12-04T05:49:22.017Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.030Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.032 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b0da15a6-4a2e-3ebf-f432-d8f68c2c81a5 Address:127.0.0.1:23328}]" +49:22.032 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23328 [Follower]" leader= +49:22.032 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b0da15a6-4a2e-3ebf-f432-d8f68c2c81a5.dc1 127.0.0.1 +49:22.033 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b0da15a6-4a2e-3ebf-f432-d8f68c2c81a5 127.0.0.1 +2022-12-04T05:49:22.033Z [INFO] agent.router: Initializing LAN area manager +49:22.033 [INFO] TestAgent.server: Adding LAN server: server="Node-b0da15a6-4a2e-3ebf-f432-d8f68c2c81a5 (Addr: tcp/127.0.0.1:23328) (DC: dc1)" +49:22.033 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b0da15a6-4a2e-3ebf-f432-d8f68c2c81a5.dc1 area=wan +49:22.033 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23365 network=udp +49:22.033 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23365 network=tcp +49:22.034 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23366 network=tcp +49:22.034 [INFO] TestAgent: started state syncer +49:22.034 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23329 network=tcp +49:22.089 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.089 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23328 [Candidate]" term=2 +49:22.089 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.089 [DEBUG] TestAgent.server.raft: vote granted: from=b0da15a6-4a2e-3ebf-f432-d8f68c2c81a5 term=2 tally=1 +49:22.089 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.089 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23328 [Leader]" +49:22.089 [INFO] TestAgent.server: cluster leadership acquired +49:22.089 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23328 +49:22.090 [INFO] TestAgent.server: New leader elected: payload=Node-b0da15a6-4a2e-3ebf-f432-d8f68c2c81a5 +49:22.091 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.097 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.097 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.097 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.097 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.097 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.097 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b0da15a6-4a2e-3ebf-f432-d8f68c2c81a5 +49:22.097 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b0da15a6-4a2e-3ebf-f432-d8f68c2c81a5 +49:22.100 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.143 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:22.144 [INFO] TestAgent: Synced node info +49:22.453 [INFO] TestAgent: Requesting shutdown +49:22.453 [INFO] TestAgent.server: shutting down server +49:22.453 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.453 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.453 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.453 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.453 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.453 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.453 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.453 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.453 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.453 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.453 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.454Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.454Z [INFO] agent.router.manager: shutting down +49:22.454 [INFO] TestAgent: consul server down +49:22.454 [INFO] TestAgent: shutdown complete +49:22.454 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23365 network=tcp +49:22.454 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23365 network=udp +49:22.454 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23366 network=tcp +49:22.954 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.954 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceChecks_DistanceSort (0.95s) === CONT TestEventFire_token -2021-10-31T23:13:08.567Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:08.567Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:08.614Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:08.614Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:08.616 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5e6b908d-7277-4bff-7cc5-9e7ac34cefb5 Address:127.0.0.1:16526}]" -13:08.617 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5e6b908d-7277-4bff-7cc5-9e7ac34cefb5.dc2 127.0.0.1 -13:08.617 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16526 [Follower]" leader= -13:08.618 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5e6b908d-7277-4bff-7cc5-9e7ac34cefb5 127.0.0.1 -2021-10-31T23:13:08.618Z [INFO] agent.router: Initializing LAN area manager -13:08.619 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16535 network=udp -13:08.619 [INFO] TestAgent.server: Adding LAN server: server="Node-5e6b908d-7277-4bff-7cc5-9e7ac34cefb5 (Addr: tcp/127.0.0.1:16526) (DC: dc2)" -13:08.619 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5e6b908d-7277-4bff-7cc5-9e7ac34cefb5.dc2 area=wan -13:08.619 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16535 network=tcp -13:08.619 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16536 network=tcp -13:08.619 [INFO] TestAgent: started state syncer -13:08.619 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16527 network=tcp -13:08.669 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:08.683 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:08.683 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16526 [Candidate]" term=2 -13:08.683 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:08.683 [DEBUG] TestAgent.server.raft: vote granted: from=5e6b908d-7277-4bff-7cc5-9e7ac34cefb5 term=2 tally=1 -13:08.683 [INFO] TestAgent.server.raft: election won: tally=1 -13:08.683 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16526 [Leader]" -13:08.684 [INFO] TestAgent.server: cluster leadership acquired -13:08.684 [INFO] TestAgent.server: initializing acls -13:08.684 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:08.684 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:08.684 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:08.684 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:08.685 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5e6b908d-7277-4bff-7cc5-9e7ac34cefb5 -13:08.685 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5e6b908d-7277-4bff-7cc5-9e7ac34cefb5.dc2 -13:08.685 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:08.685 [INFO] TestAgent.server: New leader elected: payload=Node-5e6b908d-7277-4bff-7cc5-9e7ac34cefb5 -13:08.686 [INFO] TestAgent.server: Updating LAN server: server="Node-5e6b908d-7277-4bff-7cc5-9e7ac34cefb5 (Addr: tcp/127.0.0.1:16526) (DC: dc2)" -13:08.686 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5e6b908d-7277-4bff-7cc5-9e7ac34cefb5.dc2 area=wan -13:08.687 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:08.687 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:08.687 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:08.687 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:08.687 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:08.687 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5e6b908d-7277-4bff-7cc5-9e7ac34cefb5 -13:08.687 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5e6b908d-7277-4bff-7cc5-9e7ac34cefb5 -13:08.687 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5e6b908d-7277-4bff-7cc5-9e7ac34cefb5 -13:08.688 [INFO] TestAgent.server: federation state anti-entropy synced -13:08.696 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:16560] -13:08.697 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:16560 -13:08.702 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-030b6d1c-d5f2-be19-0984-025fc842fde4.dc1 127.0.0.1 -13:08.702 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -13:08.703 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-030b6d1c-d5f2-be19-0984-025fc842fde4.dc1 area=wan -13:08.703 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:08.704 [INFO] TestAgent: Synced node info -13:08.704 [DEBUG] TestAgent: Node info in sync -13:08.708 [INFO] TestAgent: Requesting shutdown -13:08.708 [INFO] TestAgent.server: shutting down server -13:08.708 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:08.708 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:08.708 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:08.708 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:08.708 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:08.708 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:08.708 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:08.708 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:08.708 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:08.708 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:08.708 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:08.708 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:08.708 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:08.708 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:08.708 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:08.708Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:08.708Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:08.708Z [INFO] agent.router.manager: shutting down -13:08.708 [INFO] TestAgent: consul server down -13:08.708 [INFO] TestAgent: shutdown complete -13:08.708 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16535 network=tcp -13:08.708 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16535 network=udp -13:08.708 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16536 network=tcp -13:09.209 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.209 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:08.220Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:08.220Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:08.258Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:08.258Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:08.260 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:030b6d1c-d5f2-be19-0984-025fc842fde4 Address:127.0.0.1:16505}]" -13:08.260 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-030b6d1c-d5f2-be19-0984-025fc842fde4.dc1 127.0.0.1 -13:08.261 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-030b6d1c-d5f2-be19-0984-025fc842fde4 127.0.0.1 -2021-10-31T23:13:08.261Z [INFO] agent.router: Initializing LAN area manager -13:08.261 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16556 network=udp -13:08.261 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16505 [Follower]" leader= -13:08.262 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-030b6d1c-d5f2-be19-0984-025fc842fde4.dc1 area=wan -13:08.262 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16556 network=tcp -13:08.262 [INFO] TestAgent.server: Adding LAN server: server="Node-030b6d1c-d5f2-be19-0984-025fc842fde4 (Addr: tcp/127.0.0.1:16505) (DC: dc1)" -13:08.262 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16557 network=tcp -13:08.262 [INFO] TestAgent: started state syncer -13:08.263 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16506 network=tcp -13:08.307 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:08.307 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16505 [Candidate]" term=2 -13:08.307 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:08.307 [DEBUG] TestAgent.server.raft: vote granted: from=030b6d1c-d5f2-be19-0984-025fc842fde4 term=2 tally=1 -13:08.308 [INFO] TestAgent.server.raft: election won: tally=1 -13:08.308 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16505 [Leader]" -13:08.308 [INFO] TestAgent.server: cluster leadership acquired -13:08.308 [INFO] TestAgent.server: initializing acls -13:08.308 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:08.309 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:08.309 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:08.309 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:08.309 [INFO] TestAgent.server: New leader elected: payload=Node-030b6d1c-d5f2-be19-0984-025fc842fde4 -13:08.309 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-030b6d1c-d5f2-be19-0984-025fc842fde4 -13:08.309 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-030b6d1c-d5f2-be19-0984-025fc842fde4.dc1 -13:08.309 [INFO] TestAgent.server: Updating LAN server: server="Node-030b6d1c-d5f2-be19-0984-025fc842fde4 (Addr: tcp/127.0.0.1:16505) (DC: dc1)" -13:08.310 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-030b6d1c-d5f2-be19-0984-025fc842fde4.dc1 area=wan -13:08.317 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:08.327 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:08.327 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:08.327 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:08.327 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:08.327 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:08.328 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-030b6d1c-d5f2-be19-0984-025fc842fde4 -13:08.328 [INFO] TestAgent.server: member joined, marking health alive: member=Node-030b6d1c-d5f2-be19-0984-025fc842fde4 -13:08.328 [INFO] TestAgent.server: federation state anti-entropy synced -13:08.329 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-030b6d1c-d5f2-be19-0984-025fc842fde4 -13:08.696 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:08.697 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:42520 -13:08.697 [INFO] TestAgent: Synced node info -13:08.697 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5e6b908d-7277-4bff-7cc5-9e7ac34cefb5.dc2 127.0.0.1 -13:08.698 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5e6b908d-7277-4bff-7cc5-9e7ac34cefb5.dc2 area=wan -13:09.209 [INFO] TestAgent: Requesting shutdown -13:09.210 [INFO] TestAgent.server: shutting down server -13:09.210 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:09.210 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:09.210 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:09.210 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:09.210 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:09.210 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:09.210 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:09.210 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:09.210 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:09.210 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:09.210 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:09.210 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:09.210 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:09.210 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:09.210 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:09.210Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:09.210Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:09.210Z [INFO] agent.router.manager: shutting down -13:09.210 [INFO] TestAgent: consul server down -13:09.210 [INFO] TestAgent: shutdown complete -13:09.210 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16556 network=tcp -13:09.210 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16556 network=udp -13:09.210 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16557 network=tcp -13:09.713 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.713 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthServiceNodes_WanTranslation (1.53s) -=== CONT TestEventList_Blocking -2021-10-31T23:13:08.708Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:08.754Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:08.787 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c667bc60-9ae2-dde3-dd99-76b6308da065 Address:127.0.0.1:16575}]" -13:08.788 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c667bc60-9ae2-dde3-dd99-76b6308da065.dc1 127.0.0.1 -13:08.791 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16575 [Follower]" leader= -13:08.798 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c667bc60-9ae2-dde3-dd99-76b6308da065 127.0.0.1 -2021-10-31T23:13:08.798Z [INFO] agent.router: Initializing LAN area manager -13:08.798 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16570 network=udp -13:08.798 [INFO] TestAgent.server: Adding LAN server: server="Node-c667bc60-9ae2-dde3-dd99-76b6308da065 (Addr: tcp/127.0.0.1:16575) (DC: dc1)" -13:08.798 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c667bc60-9ae2-dde3-dd99-76b6308da065.dc1 area=wan -13:08.799 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16570 network=tcp -13:08.800 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16571 network=tcp -13:08.800 [INFO] TestAgent: started state syncer -13:08.800 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16576 network=tcp -13:08.875 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:08.875 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16575 [Candidate]" term=2 -13:08.875 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:08.875 [DEBUG] TestAgent.server.raft: vote granted: from=c667bc60-9ae2-dde3-dd99-76b6308da065 term=2 tally=1 -13:08.875 [INFO] TestAgent.server.raft: election won: tally=1 -13:08.875 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16575 [Leader]" -13:08.875 [INFO] TestAgent.server: cluster leadership acquired -13:08.875 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16575 -13:08.876 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:08.877 [INFO] TestAgent.server: New leader elected: payload=Node-c667bc60-9ae2-dde3-dd99-76b6308da065 -13:08.886 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:08.886 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:08.886 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:08.886 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:08.886 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:08.886 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c667bc60-9ae2-dde3-dd99-76b6308da065 -13:08.886 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c667bc60-9ae2-dde3-dd99-76b6308da065 -13:08.887 [INFO] TestAgent.server: federation state anti-entropy synced -13:09.041 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:09.042 [INFO] TestAgent: Synced node info -13:09.214 [INFO] TestAgent: Requesting shutdown -13:09.214 [INFO] TestAgent.server: shutting down server -13:09.214 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:09.214 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:09.214 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:09.214 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:09.214 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:09.214 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:09.214 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:09.214 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:09.214 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:09.214 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:09.214 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:09.214Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:09.214Z [INFO] agent.router.manager: shutting down -13:09.214 [INFO] TestAgent: consul server down -13:09.214 [INFO] TestAgent: shutdown complete -13:09.214 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16570 network=tcp -13:09.214 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16570 network=udp -13:09.214 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16571 network=tcp -13:09.717 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.717 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthChecksInState_DistanceSort (1.06s) -=== CONT TestIntentionsCreate_good -2021-10-31T23:13:09.019Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:09.068Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:09.078 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b14a8317-e8f8-cc0e-8b84-aff9d70d43e7 Address:127.0.0.1:16610}]" -13:09.079 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b14a8317-e8f8-cc0e-8b84-aff9d70d43e7.dc1 127.0.0.1 -13:09.079 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b14a8317-e8f8-cc0e-8b84-aff9d70d43e7 127.0.0.1 -2021-10-31T23:13:09.079Z [INFO] agent.router: Initializing LAN area manager -13:09.080 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16605 network=udp -13:09.080 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16610 [Follower]" leader= -13:09.080 [INFO] TestAgent.server: Adding LAN server: server="Node-b14a8317-e8f8-cc0e-8b84-aff9d70d43e7 (Addr: tcp/127.0.0.1:16610) (DC: dc1)" -13:09.080 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b14a8317-e8f8-cc0e-8b84-aff9d70d43e7.dc1 area=wan -13:09.080 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16605 network=tcp -13:09.081 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16606 network=tcp -13:09.081 [INFO] TestAgent: started state syncer -13:09.081 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16611 network=tcp -13:09.121 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:09.121 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16610 [Candidate]" term=2 -13:09.121 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:09.121 [DEBUG] TestAgent.server.raft: vote granted: from=b14a8317-e8f8-cc0e-8b84-aff9d70d43e7 term=2 tally=1 -13:09.121 [INFO] TestAgent.server.raft: election won: tally=1 -13:09.121 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16610 [Leader]" -13:09.122 [INFO] TestAgent.server: cluster leadership acquired -13:09.122 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16610 -13:09.122 [INFO] TestAgent.server: New leader elected: payload=Node-b14a8317-e8f8-cc0e-8b84-aff9d70d43e7 -13:09.123 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:09.125 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:09.125 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:09.125 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:09.125 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:09.125 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:09.125 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b14a8317-e8f8-cc0e-8b84-aff9d70d43e7 -13:09.125 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b14a8317-e8f8-cc0e-8b84-aff9d70d43e7 -13:09.126 [INFO] TestAgent.server: federation state anti-entropy synced -13:09.206 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:09.206 [INFO] TestAgent: Synced node info -13:09.206 [DEBUG] TestAgent: Node info in sync -13:09.262 [INFO] TestAgent: Requesting shutdown -13:09.262 [INFO] TestAgent.server: shutting down server -13:09.262 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:09.262 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:09.262 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:09.262 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:09.262 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:09.262 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:09.262 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:09.262 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:09.262 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:09.262 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:09.262 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:09.262Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:09.262Z [INFO] agent.router.manager: shutting down -13:09.262 [INFO] TestAgent: consul server down -13:09.262 [INFO] TestAgent: shutdown complete -13:09.262 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16605 network=tcp -13:09.262 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16605 network=udp -13:09.262 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16606 network=tcp -13:09.794 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.794 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_LoadKeyrings/client_with_keys -2021-10-31T23:13:09.059Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:09.084Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:09.114 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:efc9cd4a-b257-cbb8-f411-5cfbef28d35f Address:127.0.0.1:16637}]" -13:09.115 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-efc9cd4a-b257-cbb8-f411-5cfbef28d35f.dc1 127.0.0.1 -13:09.116 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-efc9cd4a-b257-cbb8-f411-5cfbef28d35f 127.0.0.1 -2021-10-31T23:13:09.116Z [INFO] agent.router: Initializing LAN area manager -13:09.118 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16637 [Follower]" leader= -13:09.119 [INFO] TestAgent.server: Adding LAN server: server="Node-efc9cd4a-b257-cbb8-f411-5cfbef28d35f (Addr: tcp/127.0.0.1:16637) (DC: dc1)" -13:09.120 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-efc9cd4a-b257-cbb8-f411-5cfbef28d35f.dc1 area=wan -13:09.121 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16632 network=tcp -13:09.126 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16632 network=udp -13:09.128 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16633 network=tcp -13:09.128 [INFO] TestAgent: started state syncer -13:09.129 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16624 network=tcp -13:09.157 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:09.158 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16637 [Candidate]" term=2 -13:09.186 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:09.186 [DEBUG] TestAgent.server.raft: vote granted: from=efc9cd4a-b257-cbb8-f411-5cfbef28d35f term=2 tally=1 -13:09.186 [INFO] TestAgent.server.raft: election won: tally=1 -13:09.186 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16637 [Leader]" -13:09.196 [INFO] TestAgent.server: cluster leadership acquired -13:09.196 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16637 -13:09.197 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:09.197 [INFO] TestAgent.server: New leader elected: payload=Node-efc9cd4a-b257-cbb8-f411-5cfbef28d35f -13:09.199 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:09.199 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:09.199 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:09.199 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:09.199 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:09.199 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-efc9cd4a-b257-cbb8-f411-5cfbef28d35f -13:09.199 [INFO] TestAgent.server: member joined, marking health alive: member=Node-efc9cd4a-b257-cbb8-f411-5cfbef28d35f -13:09.200 [INFO] TestAgent.server: federation state anti-entropy synced -13:09.265 [INFO] TestAgent: Requesting shutdown -13:09.265 [INFO] TestAgent.server: shutting down server -13:09.265 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:09.265 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:09.265 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:09.265 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:09.265 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:09.265 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:09.265 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:09.265 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:09.265 [INFO] TestAgent: Synced node info -13:09.265 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:09.265 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:09.265 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:09.265 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:09.266Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:09.266Z [INFO] agent.router.manager: shutting down -13:09.266 [INFO] TestAgent: consul server down -13:09.266 [INFO] TestAgent: shutdown complete -13:09.266 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16632 network=tcp -13:09.266 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16632 network=udp -13:09.266 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16633 network=tcp -13:09.801 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.801 [INFO] TestAgent: Endpoints down ---- PASS: TestHealthChecksInState_NodeMetaFilter (0.77s) +2022-12-04T05:49:22.166Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.175Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.177 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b339f1b5-517e-fee2-7b47-fd2c15fbb431 Address:127.0.0.1:23391}]" +49:22.177 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23391 [Follower]" leader= +49:22.177 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b339f1b5-517e-fee2-7b47-fd2c15fbb431.dc1 127.0.0.1 +49:22.178 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b339f1b5-517e-fee2-7b47-fd2c15fbb431 127.0.0.1 +2022-12-04T05:49:22.178Z [INFO] agent.router: Initializing LAN area manager +49:22.178 [INFO] TestAgent.server: Adding LAN server: server="Node-b339f1b5-517e-fee2-7b47-fd2c15fbb431 (Addr: tcp/127.0.0.1:23391) (DC: dc1)" +49:22.178 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b339f1b5-517e-fee2-7b47-fd2c15fbb431.dc1 area=wan +49:22.178 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23295 network=udp +49:22.178 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23295 network=tcp +49:22.178 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23296 network=tcp +49:22.178 [INFO] TestAgent: started state syncer +49:22.178 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23392 network=tcp +49:22.245 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.245 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23391 [Candidate]" term=2 +49:22.245 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.245 [DEBUG] TestAgent.server.raft: vote granted: from=b339f1b5-517e-fee2-7b47-fd2c15fbb431 term=2 tally=1 +49:22.245 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.245 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23391 [Leader]" +49:22.245 [INFO] TestAgent.server: cluster leadership acquired +49:22.245 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23391 +49:22.246 [INFO] TestAgent.server: New leader elected: payload=Node-b339f1b5-517e-fee2-7b47-fd2c15fbb431 +49:22.246 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.247 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.247 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.247 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.247 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.247 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.247 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b339f1b5-517e-fee2-7b47-fd2c15fbb431 +49:22.247 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b339f1b5-517e-fee2-7b47-fd2c15fbb431 +49:22.248 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.353 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:22.354 [INFO] TestAgent: Synced node info +49:22.511 [INFO] TestAgent: Requesting shutdown +49:22.511 [INFO] TestAgent.server: shutting down server +49:22.511 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.511 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.511 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.511 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.511 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.511 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.511 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.511 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.511 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.511 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.511 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.511Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.511Z [INFO] agent.router.manager: shutting down +49:22.511 [INFO] TestAgent: consul server down +49:22.511 [INFO] TestAgent: shutdown complete +49:22.511 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23295 network=tcp +49:22.511 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23295 network=udp +49:22.511 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23296 network=tcp +49:23.012 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.012 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceChecks_Filtering (0.86s) +=== CONT TestDNS_Compression_trimUDPResponse +--- PASS: TestDNS_Compression_trimUDPResponse (0.01s) === CONT TestEventFire -[WARN] freeport: 10 out of 66 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:09.233Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:09.275Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:09.276 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c2895a74-75a8-cac2-6efd-133985e732a2 Address:127.0.0.1:16658}]" -13:09.277 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16658 [Follower]" leader= -13:09.277 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c2895a74-75a8-cac2-6efd-133985e732a2.dc1 127.0.0.1 -13:09.278 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c2895a74-75a8-cac2-6efd-133985e732a2 127.0.0.1 -2021-10-31T23:13:09.278Z [INFO] agent.router: Initializing LAN area manager -13:09.278 [INFO] TestAgent.server: Adding LAN server: server="Node-c2895a74-75a8-cac2-6efd-133985e732a2 (Addr: tcp/127.0.0.1:16658) (DC: dc1)" -13:09.278 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c2895a74-75a8-cac2-6efd-133985e732a2.dc1 area=wan -13:09.278 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16653 network=tcp -13:09.279 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16653 network=udp -13:09.279 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16654 network=tcp -13:09.279 [INFO] TestAgent: started state syncer -13:09.279 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16659 network=tcp -13:09.337 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:09.337 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16658 [Candidate]" term=2 -13:09.338 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:09.338 [DEBUG] TestAgent.server.raft: vote granted: from=c2895a74-75a8-cac2-6efd-133985e732a2 term=2 tally=1 -13:09.338 [INFO] TestAgent.server.raft: election won: tally=1 -13:09.338 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16658 [Leader]" -13:09.338 [INFO] TestAgent.server: cluster leadership acquired -13:09.338 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16658 -13:09.339 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:09.340 [INFO] TestAgent.server: New leader elected: payload=Node-c2895a74-75a8-cac2-6efd-133985e732a2 -13:09.340 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:09.340 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:09.340 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:09.340 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:09.340 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:09.340 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c2895a74-75a8-cac2-6efd-133985e732a2 -13:09.340 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c2895a74-75a8-cac2-6efd-133985e732a2 -13:09.350 [INFO] TestAgent.server: federation state anti-entropy synced -13:09.369 [INFO] TestAgent: Requesting shutdown -13:09.369 [INFO] TestAgent.server: shutting down server -13:09.369 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:09.369 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:09.369 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:09.369 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:09.369 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:09.369 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:09.369 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:09.369 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:09.369 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:09.369 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:09.369 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:09.369 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:09.369Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:09.369Z [INFO] agent.router.manager: shutting down -13:09.369 [INFO] TestAgent: consul server down -13:09.369 [INFO] TestAgent: shutdown complete -13:09.369 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16653 network=tcp -13:09.369 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16653 network=udp -13:09.370 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16654 network=tcp -13:09.873 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.873 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsSpecificGet_good (0.70s) -=== CONT TestDNS_Compression_ReverseLookup -2021-10-31T23:13:09.225Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:09.258Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:09.259 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d131a5f4-a965-fefc-2038-f7b8023ae97d Address:127.0.0.1:16651}]" -13:09.260 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d131a5f4-a965-fefc-2038-f7b8023ae97d.dc1 127.0.0.1 -13:09.260 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d131a5f4-a965-fefc-2038-f7b8023ae97d 127.0.0.1 -2021-10-31T23:13:09.260Z [INFO] agent.router: Initializing LAN area manager -13:09.261 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16646 network=udp -13:09.261 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16651 [Follower]" leader= -13:09.261 [INFO] TestAgent.server: Adding LAN server: server="Node-d131a5f4-a965-fefc-2038-f7b8023ae97d (Addr: tcp/127.0.0.1:16651) (DC: dc1)" -13:09.261 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d131a5f4-a965-fefc-2038-f7b8023ae97d.dc1 area=wan -13:09.261 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16646 network=tcp -13:09.274 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16647 network=tcp -13:09.274 [INFO] TestAgent: started state syncer -13:09.274 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16638 network=tcp -13:09.305 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:09.306 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16651 [Candidate]" term=2 -13:09.306 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:09.306 [DEBUG] TestAgent.server.raft: vote granted: from=d131a5f4-a965-fefc-2038-f7b8023ae97d term=2 tally=1 -13:09.306 [INFO] TestAgent.server.raft: election won: tally=1 -13:09.306 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16651 [Leader]" -13:09.306 [INFO] TestAgent.server: cluster leadership acquired -13:09.306 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16651 -13:09.307 [INFO] TestAgent.server: New leader elected: payload=Node-d131a5f4-a965-fefc-2038-f7b8023ae97d -13:09.307 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:09.309 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:09.309 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:09.309 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:09.309 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:09.309 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:09.309 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d131a5f4-a965-fefc-2038-f7b8023ae97d -13:09.309 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d131a5f4-a965-fefc-2038-f7b8023ae97d -13:09.318 [INFO] TestAgent: Requesting shutdown -13:09.318 [INFO] TestAgent.server: federation state anti-entropy synced -13:09.318 [INFO] TestAgent.server: shutting down server -13:09.318 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:09.318 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:09.318 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:09.318 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:09.318 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:09.318 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:09.318 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:09.318 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:09.318 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:09.318 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:09.318 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:09.318 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:09.318Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:09.318Z [INFO] agent.router.manager: shutting down -13:09.318 [INFO] TestAgent: consul server down -13:09.318 [INFO] TestAgent: shutdown complete -13:09.318 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16646 network=tcp -13:09.318 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16646 network=udp -13:09.318 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16647 network=tcp -13:09.818 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.886 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsSpecificGet_invalidId (0.71s) -=== CONT TestIntentionsCheck_noDestination -2021-10-31T23:13:09.073Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:09.150Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:09.155 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:47a87bd4-b6a1-9d98-de60-4bdfe991f885 Address:127.0.0.1:16630}]" -13:09.156 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-47a87bd4-b6a1-9d98-de60-4bdfe991f885.dc1 127.0.0.1 -13:09.157 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-47a87bd4-b6a1-9d98-de60-4bdfe991f885 127.0.0.1 -2021-10-31T23:13:09.157Z [INFO] agent.router: Initializing LAN area manager -13:09.157 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16630 [Follower]" leader= -13:09.158 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16625 network=udp -13:09.186 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-47a87bd4-b6a1-9d98-de60-4bdfe991f885.dc1 area=wan -13:09.186 [INFO] TestAgent.server: Adding LAN server: server="Node-47a87bd4-b6a1-9d98-de60-4bdfe991f885 (Addr: tcp/127.0.0.1:16630) (DC: dc1)" -13:09.190 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16625 network=tcp -13:09.190 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16626 network=tcp -13:09.190 [INFO] TestAgent: started state syncer -13:09.196 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16645 network=tcp -13:09.206 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:09.206 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16630 [Candidate]" term=2 -13:09.206 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:09.206 [DEBUG] TestAgent.server.raft: vote granted: from=47a87bd4-b6a1-9d98-de60-4bdfe991f885 term=2 tally=1 -13:09.206 [INFO] TestAgent.server.raft: election won: tally=1 -13:09.206 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16630 [Leader]" -13:09.206 [INFO] TestAgent.server: cluster leadership acquired -13:09.207 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16630 -13:09.207 [INFO] TestAgent.server: New leader elected: payload=Node-47a87bd4-b6a1-9d98-de60-4bdfe991f885 -13:09.207 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:09.209 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:09.209 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:09.209 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:09.209 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:09.209 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:09.209 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-47a87bd4-b6a1-9d98-de60-4bdfe991f885 -13:09.209 [INFO] TestAgent.server: member joined, marking health alive: member=Node-47a87bd4-b6a1-9d98-de60-4bdfe991f885 -13:09.210 [INFO] TestAgent.server: federation state anti-entropy synced -13:09.326 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:09.326 [INFO] TestAgent: Synced node info -13:09.326 [DEBUG] TestAgent: Node info in sync -13:09.346 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:09.346 [DEBUG] TestAgent: Node info in sync -13:09.434 [DEBUG] TestAgent.server: User event: event=test -13:09.434 [DEBUG] TestAgent.server: User event: event=foo -13:09.434 [DEBUG] TestAgent: new event: event_name=test event_id=8e110694-4e11-daae-8a38-54cb85c8265e -13:09.434 [DEBUG] TestAgent: new event: event_name=foo event_id=f23c382b-bc47-6b09-fd04-b3aca2c057fc -13:09.462 [INFO] TestAgent: Requesting shutdown -13:09.462 [INFO] TestAgent.server: shutting down server -13:09.462 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:09.462 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:09.462 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:09.462 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:09.462 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:09.462 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:09.462 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:09.462 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:09.462 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:09.462 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:09.462 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:09.462Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:09.462Z [INFO] agent.router.manager: shutting down -13:09.462 [INFO] TestAgent: consul server down -13:09.462 [INFO] TestAgent: shutdown complete -13:09.462 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16625 network=tcp -13:09.462 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16625 network=udp -13:09.462 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16626 network=tcp -13:09.965 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.965 [INFO] TestAgent: Endpoints down ---- PASS: TestEventList_Filter (0.94s) +2022-12-04T05:49:22.420Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.430Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.431 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9a6b46d8-6f49-d6cb-f38e-2add7a7cc731 Address:127.0.0.1:23461}]" +49:22.431 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23461 [Follower]" leader= +49:22.432 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9a6b46d8-6f49-d6cb-f38e-2add7a7cc731.dc1 127.0.0.1 +49:22.432 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9a6b46d8-6f49-d6cb-f38e-2add7a7cc731 127.0.0.1 +2022-12-04T05:49:22.432Z [INFO] agent.router: Initializing LAN area manager +49:22.432 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23435 network=udp +49:22.432 [INFO] TestAgent.server: Adding LAN server: server="Node-9a6b46d8-6f49-d6cb-f38e-2add7a7cc731 (Addr: tcp/127.0.0.1:23461) (DC: dc1)" +49:22.432 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9a6b46d8-6f49-d6cb-f38e-2add7a7cc731.dc1 area=wan +49:22.433 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23435 network=tcp +49:22.433 [INFO] TestAgent: Started HTTP server: address=/tmp/consul-test/TestHTTPServer_UnixSocket-consul713867826/test.sock network=unix +49:22.433 [INFO] TestAgent: started state syncer +49:22.433 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23462 network=tcp +49:22.499 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.500 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23461 [Candidate]" term=2 +49:22.500 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.500 [DEBUG] TestAgent.server.raft: vote granted: from=9a6b46d8-6f49-d6cb-f38e-2add7a7cc731 term=2 tally=1 +49:22.500 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.500 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23461 [Leader]" +49:22.500 [INFO] TestAgent.server: cluster leadership acquired +49:22.500 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23461 +49:22.500 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.501 [INFO] TestAgent.server: New leader elected: payload=Node-9a6b46d8-6f49-d6cb-f38e-2add7a7cc731 +49:22.502 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.502 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.502 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.502 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.502 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.502 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9a6b46d8-6f49-d6cb-f38e-2add7a7cc731 +49:22.502 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9a6b46d8-6f49-d6cb-f38e-2add7a7cc731 +49:22.502 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.504 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:22.505 [INFO] TestAgent: Synced node info +49:22.505 [DEBUG] TestAgent: Node info in sync +49:22.522 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=@ latency=1.148902ms +49:22.523 [INFO] TestAgent: Requesting shutdown +49:22.523 [INFO] TestAgent.server: shutting down server +49:22.523 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.523 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.523 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.523 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.523 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.523 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.523 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.523 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.523 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.523 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.523 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.524Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.524Z [INFO] agent.router.manager: shutting down +49:22.524 [INFO] TestAgent: consul server down +49:22.524 [INFO] TestAgent: shutdown complete +49:22.524 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23435 network=tcp +49:22.524 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23435 network=udp +49:22.524 [INFO] TestAgent: Stopping server: protocol=HTTP address=/tmp/consul-test/TestHTTPServer_UnixSocket-consul713867826/test.sock network=unix +49:23.024 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.024 [INFO] TestAgent: Endpoints down +--- PASS: TestHTTPServer_UnixSocket (0.61s) === CONT TestDNS_ReloadConfig_DuringQuery -2021-10-31T23:13:09.227Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:09.264Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:09.270 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a961f355-f379-413e-400b-26a3db39facc Address:127.0.0.1:16644}]" -13:09.271 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a961f355-f379-413e-400b-26a3db39facc.dc1 127.0.0.1 -13:09.271 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a961f355-f379-413e-400b-26a3db39facc 127.0.0.1 -2021-10-31T23:13:09.271Z [INFO] agent.router: Initializing LAN area manager -13:09.271 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16639 network=udp -13:09.271 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16644 [Follower]" leader= -13:09.272 [INFO] TestAgent.server: Adding LAN server: server="Node-a961f355-f379-413e-400b-26a3db39facc (Addr: tcp/127.0.0.1:16644) (DC: dc1)" -13:09.272 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a961f355-f379-413e-400b-26a3db39facc.dc1 area=wan -13:09.272 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16639 network=tcp -13:09.273 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16640 network=tcp -13:09.273 [INFO] TestAgent: started state syncer -13:09.273 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16652 network=tcp -13:09.307 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:09.307 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16644 [Candidate]" term=2 -13:09.307 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:09.307 [DEBUG] TestAgent.server.raft: vote granted: from=a961f355-f379-413e-400b-26a3db39facc term=2 tally=1 -13:09.307 [INFO] TestAgent.server.raft: election won: tally=1 -13:09.307 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16644 [Leader]" -13:09.309 [INFO] TestAgent.server: cluster leadership acquired -13:09.310 [INFO] TestAgent.server: New leader elected: payload=Node-a961f355-f379-413e-400b-26a3db39facc -13:09.310 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16644 -13:09.311 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:09.312 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:09.312 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:09.312 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:09.312 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:09.312 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:09.312 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a961f355-f379-413e-400b-26a3db39facc -13:09.312 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a961f355-f379-413e-400b-26a3db39facc -13:09.313 [INFO] TestAgent.server: federation state anti-entropy synced -13:09.502 [DEBUG] TestAgent.server: User event: event=test -13:09.502 [DEBUG] TestAgent: new event: event_name=test event_id=87ec0144-6001-79e5-717b-d511d6bec993 -13:09.530 [INFO] TestAgent: Requesting shutdown -13:09.530 [INFO] TestAgent.server: shutting down server -13:09.530 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:09.530 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:09.530 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:09.530 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:09.530 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:09.530 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:09.530 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:09.530 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:09.530 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:09.530 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:09.530 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:09.530 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:09.530Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:09.530Z [INFO] agent.router.manager: shutting down -13:09.530 [INFO] TestAgent: consul server down -13:09.530 [INFO] TestAgent: shutdown complete -13:09.530 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16639 network=tcp -13:09.530 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16639 network=udp -13:09.530 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16640 network=tcp -13:10.031 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.055 [INFO] TestAgent: Endpoints down ---- PASS: TestEventList (0.86s) -=== CONT TestDNS_Compression_Query -2021-10-31T23:13:09.049Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:09.049Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:09.078Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:09.078Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:09.080 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fe4309cd-a80a-a668-d570-e560ee605dc1 Address:127.0.0.1:16619}]" -13:09.081 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16619 [Follower]" leader= -13:09.086 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fe4309cd-a80a-a668-d570-e560ee605dc1.dc1 127.0.0.1 -13:09.087 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fe4309cd-a80a-a668-d570-e560ee605dc1 127.0.0.1 -2021-10-31T23:13:09.087Z [INFO] agent.router: Initializing LAN area manager -13:09.087 [INFO] TestAgent.server: Adding LAN server: server="Node-fe4309cd-a80a-a668-d570-e560ee605dc1 (Addr: tcp/127.0.0.1:16619) (DC: dc1)" -13:09.087 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fe4309cd-a80a-a668-d570-e560ee605dc1.dc1 area=wan -13:09.087 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16612 network=tcp -13:09.087 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16612 network=udp -13:09.088 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16613 network=tcp -13:09.088 [INFO] TestAgent: started state syncer -13:09.088 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16631 network=tcp -13:09.127 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:09.127 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16619 [Candidate]" term=2 -13:09.127 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:09.127 [DEBUG] TestAgent.server.raft: vote granted: from=fe4309cd-a80a-a668-d570-e560ee605dc1 term=2 tally=1 -13:09.127 [INFO] TestAgent.server.raft: election won: tally=1 -13:09.127 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16619 [Leader]" -13:09.128 [INFO] TestAgent.server: cluster leadership acquired -13:09.128 [INFO] TestAgent.server: initializing acls -13:09.129 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:09.129 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:09.130 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:09.130 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:09.131 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:09.131 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:09.131 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-fe4309cd-a80a-a668-d570-e560ee605dc1 -13:09.131 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-fe4309cd-a80a-a668-d570-e560ee605dc1.dc1 -13:09.132 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:09.134 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-fe4309cd-a80a-a668-d570-e560ee605dc1.dc1 area=wan -13:09.134 [INFO] TestAgent.server: New leader elected: payload=Node-fe4309cd-a80a-a668-d570-e560ee605dc1 -13:09.134 [INFO] TestAgent.server: Updating LAN server: server="Node-fe4309cd-a80a-a668-d570-e560ee605dc1 (Addr: tcp/127.0.0.1:16619) (DC: dc1)" -13:09.151 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:09.151 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:09.151 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:09.151 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:09.151 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:09.151 [INFO] TestAgent.server: federation state anti-entropy synced -13:09.151 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe4309cd-a80a-a668-d570-e560ee605dc1 -13:09.151 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fe4309cd-a80a-a668-d570-e560ee605dc1 -13:09.152 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe4309cd-a80a-a668-d570-e560ee605dc1 -13:09.483 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:09.483 [INFO] TestAgent: Synced node info -13:09.516 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fe4309cd-a80a-a668-d570-e560ee605dc1 -13:09.516 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fe4309cd-a80a-a668-d570-e560ee605dc1 -13:09.517 [DEBUG] TestAgent.server: User event: event=foo -13:09.517 [DEBUG] TestAgent: new event: event_name=foo event_id=2ef77513-c6d6-6788-ac4c-8394af0f7041 -13:09.546 [INFO] TestAgent: Requesting shutdown -13:09.546 [INFO] TestAgent.server: shutting down server -13:09.546 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:09.546 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:09.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:09.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:09.546 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:09.546 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:09.546 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:09.546 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:09.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:09.546 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:09.546 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:09.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:09.546 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:09.546 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:09.546 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:09.546Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:09.546Z [INFO] agent.router.manager: shutting down -13:09.546 [INFO] TestAgent: consul server down -13:09.546 [INFO] TestAgent: shutdown complete -13:09.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16612 network=tcp -13:09.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16612 network=udp -13:09.546 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16613 network=tcp -13:10.047 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.065 [INFO] TestAgent: Endpoints down ---- PASS: TestEventList_ACLFilter (1.05s) - --- PASS: TestEventList_ACLFilter/no_token (0.00s) - --- PASS: TestEventList_ACLFilter/root_token (0.03s) -=== CONT TestDNS_ConfigReload -[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! -13:09.663 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-9e3cc169-395b-07a8-ee57-804e83ed23e7 127.0.0.1 -2021-10-31T23:13:09.663Z [INFO] agent.router: Initializing LAN area manager -13:09.664 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16660 network=udp -13:09.664 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16660 network=tcp -13:09.664 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16661 network=tcp -13:09.665 [INFO] TestAgent: started state syncer -13:09.665 [INFO] TestAgent: Requesting shutdown -13:09.665 [INFO] TestAgent.client: shutting down client -13:09.665 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -13:09.665 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16666 network=tcp -13:09.665 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -2021-10-31T23:13:09.665Z [WARN] agent.router.manager: No servers available -13:09.665 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -13:09.665 [INFO] TestAgent: consul client down -13:09.665 [INFO] TestAgent: shutdown complete -13:09.665 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16660 network=tcp -13:09.665 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16660 network=udp -13:09.669 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16661 network=tcp -13:10.170 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.170 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_InmemKeyrings/ignore_files -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -13:09.871 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-d22da2a4-0558-310f-e4ea-71281135c9ff 127.0.0.1 -2021-10-31T23:13:09.871Z [INFO] agent.router: Initializing LAN area manager -13:09.871 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy -13:09.871 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16695 network=udp -13:09.874 [INFO] TestAgent.client.serf.lan: serf: Attempting re-join to previously known node: Node-6790e579-7a82-d6e9-0ee6-0d06b74d1a72: 127.0.0.1:16251 -2021-10-31T23:13:09.874Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.874Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.874Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.874Z [WARN] agent.router.manager: No servers available -13:09.874 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:16251: connect: connection refused -13:09.874 [INFO] TestAgent.client.serf.lan: serf: Attempting re-join to previously known node: Node-0b522c41-4596-57bf-f084-5c080e62cb05: 127.0.0.1:16174 -13:09.874 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:16174: connect: connection refused -13:09.874 [WARN] TestAgent.client.serf.lan: serf: Failed to re-join any previously known node -2021-10-31T23:13:09.874Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.874Z [WARN] agent.router.manager: No servers available -13:09.874 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16695 network=tcp -2021-10-31T23:13:09.874Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.874Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.874Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -13:09.875 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" -13:09.875 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16696 network=tcp -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -13:09.875 [INFO] TestAgent: started state syncer -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.875Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -13:09.876 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -13:09.876 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -13:09.876 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -13:09.876 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -13:09.876 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -13:09.876 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -13:09.876 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16701 network=tcp -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -13:09.876 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -13:09.876 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" -13:09.876 [DEBUG] TestAgent: removed service: service=web-sidecar-proxy -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -13:09.876 [INFO] TestAgent: Requesting shutdown -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -13:09.876 [INFO] TestAgent.client: shutting down client -2021-10-31T23:13:09.876Z [WARN] agent.router.manager: No servers available -13:09.876 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -13:09.876 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -13:09.876 [INFO] TestAgent: consul client down -13:09.876 [INFO] TestAgent: shutdown complete -13:09.876 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16695 network=tcp -13:09.877 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16695 network=udp -13:09.877 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16696 network=tcp -13:10.381 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.381 [INFO] TestAgent: Endpoints down -13:05.127 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-6790e579-7a82-d6e9-0ee6-0d06b74d1a72 127.0.0.1 -2021-10-31T23:13:05.127Z [INFO] agent.router: Initializing LAN area manager -13:05.127 [DEBUG] TestAgent: added local registration for service: service=web-sidecar-proxy -13:05.127 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16248 network=udp -2021-10-31T23:13:05.134Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.134Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.134Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.134Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.135Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.135Z [WARN] agent.router.manager: No servers available -13:05.135 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16248 network=tcp -2021-10-31T23:13:05.135Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.135Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.135Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.135Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.135Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.135Z [WARN] agent.router.manager: No servers available -13:05.135 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -13:05.136 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" -13:05.136 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16249 network=tcp -13:05.142 [INFO] TestAgent: started state syncer -13:05.154 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -2021-10-31T23:13:05.154Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.154Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.154Z [WARN] agent.router.manager: No servers available -13:05.154 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -13:05.154 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -13:05.154 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -13:05.156 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -2021-10-31T23:13:05.157Z [WARN] agent.router.manager: No servers available -13:05.157 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -13:05.186 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -13:05.186 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -13:05.186 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -13:05.186 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16254 network=tcp -13:05.198 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:16174] -13:05.198 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:16174 -2021-10-31T23:13:05.198Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.198Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.198Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.199Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.199Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.199Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.199Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.199Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.199Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.199Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.199Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.199Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.199Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.199Z [WARN] agent.router.manager: No servers available -13:05.201 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" -2021-10-31T23:13:05.201Z [WARN] agent.router.manager: No servers available -13:05.202 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -13:05.202 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -13:05.202 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" -13:05.202 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -13:05.202 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -13:05.202 [ERROR] TestAgent: error handling service update: error="error watching service config: No known Consul servers" -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -13:05.202 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -13:05.202 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -13:05.202 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" -13:05.202 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" -13:05.202 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.202Z [WARN] agent.router.manager: No servers available -2021-10-31T23:13:05.203Z [WARN] agent.router.manager: No servers available -13:05.203 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" -13:05.203 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=discovery-chain:redis error="error filling agent cache: No known Consul servers" -13:05.218 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-0b522c41-4596-57bf-f084-5c080e62cb05 127.0.0.1 -13:05.218 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -13:05.218 [DEBUG] TestAgent: systemd notify failed: error="No socket" -2021-10-31T23:13:05.218Z [WARN] agent.router.manager: No servers available -13:05.218 [INFO] TestAgent.client: adding server: server="Node-0b522c41-4596-57bf-f084-5c080e62cb05 (Addr: tcp/127.0.0.1:16162) (DC: dc1)" -13:05.313 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader -13:05.512 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader -13:05.512 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-6790e579-7a82-d6e9-0ee6-0d06b74d1a72 -13:05.714 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader -13:05.714 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-6790e579-7a82-d6e9-0ee6-0d06b74d1a72 -13:05.746 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:05.747 [INFO] TestAgent: Synced node info -13:05.747 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -13:05.747 [DEBUG] TestAgent: Node info in sync -13:05.747 [DEBUG] TestAgent: Service in sync: service=web-sidecar-proxy -13:05.922 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-6790e579-7a82-d6e9-0ee6-0d06b74d1a72 -13:05.934 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader -13:05.934 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-6790e579-7a82-d6e9-0ee6-0d06b74d1a72 -13:08.703 [DEBUG] TestAgent: Node info in sync -13:08.707 [INFO] TestAgent: Synced service: service=web-sidecar-proxy -13:08.707 [DEBUG] TestAgent: Node info in sync -13:08.707 [DEBUG] TestAgent: Service in sync: service=web-sidecar-proxy -13:08.708 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -13:08.708 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -13:08.708 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -13:08.708 [ERROR] TestAgent.proxycfg: watch error: service_id=web-sidecar-proxy id=service-http-checks:web error="error filling agent cache: Internal cache failure: service 'web' not in agent state" -13:08.715 [INFO] TestAgent: Requesting shutdown -13:08.715 [INFO] TestAgent.client: shutting down client -13:08.715 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -13:08.715 [INFO] TestAgent: consul client down -13:08.715 [INFO] TestAgent: shutdown complete -13:08.715 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16248 network=tcp -13:08.715 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16248 network=udp -13:08.715 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16249 network=tcp -13:08.715 [ERROR] TestAgent.client: RPC failed to server: method=Intention.Match server=127.0.0.1:16162 error="rpc error making call: EOF" -13:08.715 [ERROR] TestAgent.client: RPC failed to server: method=Health.ServiceNodes server=127.0.0.1:16162 error="rpc error making call: EOF" -13:08.715 [ERROR] TestAgent.client: RPC failed to server: method=DiscoveryChain.Get server=127.0.0.1:16162 error="rpc error making call: EOF" -13:08.715 [ERROR] TestAgent.client: RPC failed to server: method=ConnectCA.Roots server=127.0.0.1:16162 error="rpc error making call: EOF" -13:08.715 [ERROR] TestAgent.client: RPC failed to server: method=ConnectCA.Roots server=127.0.0.1:16162 error="rpc error making call: EOF" -13:08.715 [ERROR] TestAgent.client: RPC failed to server: method=DiscoveryChain.Get server=127.0.0.1:16162 error="rpc error making call: EOF" -13:08.715 [ERROR] TestAgent.client: RPC failed to server: method=ConfigEntry.ResolveServiceConfig server=127.0.0.1:16162 error="rpc error making call: EOF" -13:09.216 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.216 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:04.746Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:04.899Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:04.901 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0b522c41-4596-57bf-f084-5c080e62cb05 Address:127.0.0.1:16162}]" -13:04.901 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16162 [Follower]" leader= -13:04.910 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0b522c41-4596-57bf-f084-5c080e62cb05.dc1 127.0.0.1 -13:04.911 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0b522c41-4596-57bf-f084-5c080e62cb05 127.0.0.1 -2021-10-31T23:13:04.911Z [INFO] agent.router: Initializing LAN area manager -13:04.911 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16171 network=udp -13:04.911 [INFO] TestAgent.server: Adding LAN server: server="Node-0b522c41-4596-57bf-f084-5c080e62cb05 (Addr: tcp/127.0.0.1:16162) (DC: dc1)" -13:04.911 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0b522c41-4596-57bf-f084-5c080e62cb05.dc1 area=wan -13:04.911 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16171 network=tcp -13:04.912 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16172 network=tcp -13:04.912 [INFO] TestAgent: started state syncer -13:04.912 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16163 network=tcp -13:04.954 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:04.954 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16162 [Candidate]" term=2 -13:04.964 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:04.964 [DEBUG] TestAgent.server.raft: vote granted: from=0b522c41-4596-57bf-f084-5c080e62cb05 term=2 tally=1 -13:04.964 [INFO] TestAgent.server.raft: election won: tally=1 -13:04.964 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16162 [Leader]" -13:04.968 [INFO] TestAgent.server: cluster leadership acquired -13:04.974 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16162 -13:04.976 [INFO] TestAgent.server: New leader elected: payload=Node-0b522c41-4596-57bf-f084-5c080e62cb05 -13:05.029 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:05.030 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:05.030 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:05.030 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:05.030 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:05.030 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:05.030 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0b522c41-4596-57bf-f084-5c080e62cb05 -13:05.030 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0b522c41-4596-57bf-f084-5c080e62cb05 -13:05.031 [INFO] TestAgent.server: federation state anti-entropy synced -13:05.040 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:05.041 [INFO] TestAgent: Synced node info -13:05.198 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:45508 -13:05.214 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6790e579-7a82-d6e9-0ee6-0d06b74d1a72 127.0.0.1 -13:05.214 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6790e579-7a82-d6e9-0ee6-0d06b74d1a72 -13:05.334 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-6790e579-7a82-d6e9-0ee6-0d06b74d1a72 -13:05.531 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-6790e579-7a82-d6e9-0ee6-0d06b74d1a72 -13:05.743 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-6790e579-7a82-d6e9-0ee6-0d06b74d1a72 -13:05.923 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-6790e579-7a82-d6e9-0ee6-0d06b74d1a72 -13:07.514 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:07.514 [DEBUG] TestAgent: Node info in sync -13:07.514 [DEBUG] TestAgent: Node info in sync -13:09.216 [INFO] TestAgent: Requesting shutdown -13:09.216 [INFO] TestAgent.server: shutting down server -13:09.216 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:09.216 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:09.216 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:09.216 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:09.216 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:09.216 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:09.216 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:09.216 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:09.216 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:09.216 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:09.216 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:09.216Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:09.216Z [INFO] agent.router.manager: shutting down -13:09.216 [INFO] TestAgent: consul server down -13:09.216 [INFO] TestAgent: shutdown complete -13:09.217 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16171 network=tcp -13:09.217 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16171 network=udp -13:09.217 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16172 network=tcp -13:09.417 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-6790e579-7a82-d6e9-0ee6-0d06b74d1a72 (timeout reached) -13:09.717 [INFO] TestAgent: Waiting for endpoints to shut down -13:09.717 [INFO] TestAgent: Endpoints down -13:09.914 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-6790e579-7a82-d6e9-0ee6-0d06b74d1a72 has failed, no acks received ---- PASS: TestServiceManager_PersistService_ConfigFiles (5.69s) -=== CONT TestDNS_Compression_trimUDPResponse ---- PASS: TestDNS_Compression_trimUDPResponse (0.03s) -=== CONT TestDNS_trimUDPResponse_NoTrim ---- PASS: TestDNS_trimUDPResponse_NoTrim (0.04s) +2022-12-04T05:49:22.286Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.295Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.297 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:365691dd-dffc-b706-7f98-dd5536c47d09 Address:127.0.0.1:23447}]" +49:22.297 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23447 [Follower]" leader= +49:22.297 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-365691dd-dffc-b706-7f98-dd5536c47d09.dc1 127.0.0.1 +49:22.298 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-365691dd-dffc-b706-7f98-dd5536c47d09 127.0.0.1 +2022-12-04T05:49:22.298Z [INFO] agent.router: Initializing LAN area manager +49:22.298 [INFO] TestAgent.server: Adding LAN server: server="Node-365691dd-dffc-b706-7f98-dd5536c47d09 (Addr: tcp/127.0.0.1:23447) (DC: dc1)" +49:22.298 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-365691dd-dffc-b706-7f98-dd5536c47d09.dc1 area=wan +49:22.298 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23407 network=udp +49:22.298 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23407 network=tcp +49:22.298 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23408 network=tcp +49:22.298 [INFO] TestAgent: started state syncer +49:22.298 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23448 network=tcp +49:22.336 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.336 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23447 [Candidate]" term=2 +49:22.336 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.336 [DEBUG] TestAgent.server.raft: vote granted: from=365691dd-dffc-b706-7f98-dd5536c47d09 term=2 tally=1 +49:22.336 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.336 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23447 [Leader]" +49:22.336 [INFO] TestAgent.server: cluster leadership acquired +49:22.336 [INFO] TestAgent.server: New leader elected: payload=Node-365691dd-dffc-b706-7f98-dd5536c47d09 +49:22.336 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23447 +49:22.337 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.338 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.338 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.338 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.338 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.338 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.338 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-365691dd-dffc-b706-7f98-dd5536c47d09 +49:22.338 [INFO] TestAgent.server: member joined, marking health alive: member=Node-365691dd-dffc-b706-7f98-dd5536c47d09 +49:22.338 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.374 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:22.374 [INFO] TestAgent: Synced node info +49:22.527 [INFO] TestAgent: Requesting shutdown +49:22.527 [INFO] TestAgent.server: shutting down server +49:22.527 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.527 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.527 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.527 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.527 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.527 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.527 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.527 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.527 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.527 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.527 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.527Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.527Z [INFO] agent.router.manager: shutting down +49:22.527 [INFO] TestAgent: consul server down +49:22.527 [INFO] TestAgent: shutdown complete +49:22.527 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23407 network=tcp +49:22.527 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23407 network=udp +49:22.527 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23408 network=tcp +49:23.027 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.027 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthNodeChecks_Filtering (0.75s) === CONT TestDNS_syncExtra --- PASS: TestDNS_syncExtra (0.00s) +=== CONT TestDNS_ConfigReload +[WARN] freeport: 10 out of 80 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:22.189Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.198Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.199 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:61029058-3fbc-0e2f-9feb-8bd597af58a6 Address:127.0.0.1:23370}]" +49:22.199 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23370 [Follower]" leader= +49:22.200 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-61029058-3fbc-0e2f-9feb-8bd597af58a6.dc1 127.0.0.1 +49:22.200 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-61029058-3fbc-0e2f-9feb-8bd597af58a6 127.0.0.1 +2022-12-04T05:49:22.200Z [INFO] agent.router: Initializing LAN area manager +49:22.200 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23309 network=udp +49:22.200 [INFO] TestAgent.server: Adding LAN server: server="Node-61029058-3fbc-0e2f-9feb-8bd597af58a6 (Addr: tcp/127.0.0.1:23370) (DC: dc1)" +49:22.200 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-61029058-3fbc-0e2f-9feb-8bd597af58a6.dc1 area=wan +49:22.201 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23309 network=tcp +49:22.201 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23310 network=tcp +49:22.201 [INFO] TestAgent: started state syncer +49:22.201 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23371 network=tcp +49:22.252 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.252 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23370 [Candidate]" term=2 +49:22.252 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.252 [DEBUG] TestAgent.server.raft: vote granted: from=61029058-3fbc-0e2f-9feb-8bd597af58a6 term=2 tally=1 +49:22.252 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.252 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23370 [Leader]" +49:22.252 [INFO] TestAgent.server: cluster leadership acquired +49:22.252 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23370 +49:22.253 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.254 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.254 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.254 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.254 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.254 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.254 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-61029058-3fbc-0e2f-9feb-8bd597af58a6 +49:22.254 [INFO] TestAgent.server: member joined, marking health alive: member=Node-61029058-3fbc-0e2f-9feb-8bd597af58a6 +49:22.254 [INFO] TestAgent.server: New leader elected: payload=Node-61029058-3fbc-0e2f-9feb-8bd597af58a6 +49:22.255 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.318 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:22.318 [INFO] TestAgent: Synced node info +49:22.552 [INFO] TestAgent: Requesting shutdown +49:22.552 [INFO] TestAgent.server: shutting down server +49:22.552 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.552 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.552 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.552 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.552 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.552 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.552 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.552 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.552 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.552 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.552 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.552Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.552Z [INFO] agent.router.manager: shutting down +49:22.553 [INFO] TestAgent: consul server down +49:22.553 [INFO] TestAgent: shutdown complete +49:22.553 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23309 network=tcp +49:22.553 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23309 network=udp +49:22.553 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23310 network=tcp +49:23.053 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.053 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthServiceChecks_NodeMetaFilter (0.87s) === CONT TestDNS_Compression_Recurse -13:09.957 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-85c0e2b3-9870-b92f-aa45-1acf44311e7b 127.0.0.1 -2021-10-31T23:13:09.957Z [INFO] agent.router: Initializing LAN area manager -13:09.961 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16730 network=udp -13:09.961 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16730 network=tcp -13:09.962 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16731 network=tcp -13:09.962 [INFO] TestAgent: started state syncer -13:09.965 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16715 network=tcp -2021-10-31T23:13:09.969Z [WARN] agent.router.manager: No servers available -13:09.969 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -13:09.995 [INFO] TestAgent: Requesting shutdown -13:09.997 [INFO] TestAgent.client: shutting down client -13:09.997 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -13:09.997 [INFO] TestAgent: consul client down -13:09.997 [INFO] TestAgent: shutdown complete -13:09.997 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16730 network=tcp -13:09.997 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16730 network=udp -13:09.997 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16731 network=tcp -13:10.498 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.498 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_LoadKeyrings (2.33s) - --- PASS: TestAgent_LoadKeyrings/no_keys (0.82s) - --- PASS: TestAgent_LoadKeyrings/server_with_keys (0.81s) - --- PASS: TestAgent_LoadKeyrings/client_with_keys (0.70s) +=== RUN TestEventList_ACLFilter/no_token +=== RUN TestEventList_ACLFilter/root_token +2022-12-04T05:49:22.180Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.189Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.190 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5150ed80-1a91-d02c-1473-3960b6212540 Address:127.0.0.1:23321}]" +49:22.190 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23321 [Follower]" leader= +49:22.191 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5150ed80-1a91-d02c-1473-3960b6212540.dc1 127.0.0.1 +49:22.191 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5150ed80-1a91-d02c-1473-3960b6212540 127.0.0.1 +2022-12-04T05:49:22.191Z [INFO] agent.router: Initializing LAN area manager +49:22.191 [INFO] TestAgent.server: Adding LAN server: server="Node-5150ed80-1a91-d02c-1473-3960b6212540 (Addr: tcp/127.0.0.1:23321) (DC: dc1)" +49:22.191 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5150ed80-1a91-d02c-1473-3960b6212540.dc1 area=wan +49:22.191 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23393 network=udp +49:22.191 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23393 network=tcp +49:22.192 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23394 network=tcp +49:22.192 [INFO] TestAgent: started state syncer +49:22.192 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23322 network=tcp +49:22.228 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.228 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23321 [Candidate]" term=2 +49:22.228 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.228 [DEBUG] TestAgent.server.raft: vote granted: from=5150ed80-1a91-d02c-1473-3960b6212540 term=2 tally=1 +49:22.228 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.228 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23321 [Leader]" +49:22.228 [INFO] TestAgent.server: cluster leadership acquired +49:22.228 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23321 +49:22.229 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.230 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.230 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.230 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.230 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.230 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.230 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5150ed80-1a91-d02c-1473-3960b6212540 +49:22.230 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5150ed80-1a91-d02c-1473-3960b6212540 +49:22.231 [INFO] TestAgent.server: New leader elected: payload=Node-5150ed80-1a91-d02c-1473-3960b6212540 +49:22.231 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.481 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:22.481 [INFO] TestAgent: Synced node info +49:22.481 [DEBUG] TestAgent: Node info in sync +49:22.627 [INFO] TestAgent: Requesting shutdown +49:22.627 [INFO] TestAgent.server: shutting down server +49:22.627 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.627 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.627 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.627 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.627 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.627 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.627 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.627 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.627 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.628 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.628 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.628Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.628Z [INFO] agent.router.manager: shutting down +49:22.628 [INFO] TestAgent: consul server down +49:22.628 [INFO] TestAgent: shutdown complete +49:22.628 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23393 network=tcp +49:22.628 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23393 network=udp +49:22.628 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23394 network=tcp +49:23.128 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.128 [INFO] TestAgent: Endpoints down +--- PASS: TestOperator_RaftPeer (1.71s) + --- PASS: TestOperator_RaftPeer/#00 (0.75s) + --- PASS: TestOperator_RaftPeer/#01 (0.96s) +=== CONT TestAgent_RerouteExistingHTTPChecks +2022-12-04T05:49:22.652Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.661Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.662 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5bfdc6ca-cbf8-53c9-39b7-8baa33bf9a32 Address:127.0.0.1:23440}]" +49:22.662 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23440 [Follower]" leader= +49:22.662 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5bfdc6ca-cbf8-53c9-39b7-8baa33bf9a32.dc1 127.0.0.1 +49:22.663 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5bfdc6ca-cbf8-53c9-39b7-8baa33bf9a32 127.0.0.1 +2022-12-04T05:49:22.663Z [INFO] agent.router: Initializing LAN area manager +49:22.663 [INFO] TestAgent.server: Adding LAN server: server="Node-5bfdc6ca-cbf8-53c9-39b7-8baa33bf9a32 (Addr: tcp/127.0.0.1:23440) (DC: dc1)" +49:22.663 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5bfdc6ca-cbf8-53c9-39b7-8baa33bf9a32.dc1 area=wan +49:22.663 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23442 network=udp +49:22.663 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23442 network=tcp +49:22.664 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23443 network=tcp +49:22.664 [INFO] TestAgent: started state syncer +49:22.664 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23441 network=tcp +49:22.716 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.716 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23440 [Candidate]" term=2 +49:22.716 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.716 [DEBUG] TestAgent.server.raft: vote granted: from=5bfdc6ca-cbf8-53c9-39b7-8baa33bf9a32 term=2 tally=1 +49:22.716 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.716 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23440 [Leader]" +49:22.716 [INFO] TestAgent.server: cluster leadership acquired +49:22.716 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23440 +49:22.717 [INFO] TestAgent.server: New leader elected: payload=Node-5bfdc6ca-cbf8-53c9-39b7-8baa33bf9a32 +49:22.717 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.718 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.718 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.718 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.718 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.718 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.718 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5bfdc6ca-cbf8-53c9-39b7-8baa33bf9a32 +49:22.718 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5bfdc6ca-cbf8-53c9-39b7-8baa33bf9a32 +49:22.718 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.781 [INFO] TestAgent: Requesting shutdown +49:22.781 [INFO] TestAgent.server: shutting down server +49:22.781 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.781 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.781 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.781 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.781 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.781 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.781 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.781 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:22.781 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.781 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.781 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.781 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.782Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.782Z [INFO] agent.router.manager: shutting down +49:22.782 [INFO] TestAgent: consul server down +49:22.782 [INFO] TestAgent: shutdown complete +49:22.782 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23442 network=tcp +49:22.782 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23442 network=udp +49:22.782 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23443 network=tcp +49:23.282 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.282 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthChecksInState_NodeMetaFilter (0.64s) +=== CONT TestDNS_Compression_ReverseLookup +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:22.648Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.657Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.658 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6f5129c8-e802-6b8f-3afb-3fdf216c55aa Address:127.0.0.1:23426}]" +49:22.659 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23426 [Follower]" leader= +49:22.659 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6f5129c8-e802-6b8f-3afb-3fdf216c55aa.dc1 127.0.0.1 +49:22.659 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6f5129c8-e802-6b8f-3afb-3fdf216c55aa 127.0.0.1 +2022-12-04T05:49:22.659Z [INFO] agent.router: Initializing LAN area manager +49:22.659 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6f5129c8-e802-6b8f-3afb-3fdf216c55aa.dc1 area=wan +49:22.660 [INFO] TestAgent.server: Adding LAN server: server="Node-6f5129c8-e802-6b8f-3afb-3fdf216c55aa (Addr: tcp/127.0.0.1:23426) (DC: dc1)" +49:22.660 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23421 network=udp +49:22.660 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23421 network=tcp +49:22.660 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23422 network=tcp +49:22.660 [INFO] TestAgent: started state syncer +49:22.660 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23427 network=tcp +49:22.716 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.716 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23426 [Candidate]" term=2 +49:22.716 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.716 [DEBUG] TestAgent.server.raft: vote granted: from=6f5129c8-e802-6b8f-3afb-3fdf216c55aa term=2 tally=1 +49:22.716 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.716 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23426 [Leader]" +49:22.716 [INFO] TestAgent.server: cluster leadership acquired +49:22.716 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23426 +49:22.717 [INFO] TestAgent.server: New leader elected: payload=Node-6f5129c8-e802-6b8f-3afb-3fdf216c55aa +49:22.717 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.718 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.718 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.718 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.718 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.718 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.718 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6f5129c8-e802-6b8f-3afb-3fdf216c55aa +49:22.718 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6f5129c8-e802-6b8f-3afb-3fdf216c55aa +49:22.719 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.850 [INFO] TestAgent: Requesting shutdown +49:22.850 [INFO] TestAgent.server: shutting down server +49:22.850 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.850 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.850 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.850 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.850 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.850 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.851 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.851 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.851 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.851 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:22.851 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.851 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.851Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.851Z [INFO] agent.router.manager: shutting down +49:22.851 [INFO] TestAgent: consul server down +49:22.851 [INFO] TestAgent: shutdown complete +49:22.851 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23421 network=tcp +49:22.851 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23421 network=udp +49:22.851 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23422 network=tcp +49:23.351 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.351 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthChecksInState_Filter (0.71s) === CONT TestDNS_trimUDPResponse_TrimSizeEDNS ---- PASS: TestDNS_trimUDPResponse_TrimSizeEDNS (0.02s) -=== CONT TestDNS_PreparedQuery_AgentSource -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:09.744Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:09.865Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:09.875 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:866513b9-1216-f9b3-2965-624611fc6db5 Address:127.0.0.1:16686}]" -13:09.875 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-866513b9-1216-f9b3-2965-624611fc6db5.dc1 127.0.0.1 -13:09.876 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-866513b9-1216-f9b3-2965-624611fc6db5 127.0.0.1 -2021-10-31T23:13:09.876Z [INFO] agent.router: Initializing LAN area manager -13:09.876 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16686 [Follower]" leader= -13:09.876 [INFO] TestAgent.server: Adding LAN server: server="Node-866513b9-1216-f9b3-2965-624611fc6db5 (Addr: tcp/127.0.0.1:16686) (DC: dc1)" -13:09.876 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-866513b9-1216-f9b3-2965-624611fc6db5.dc1 area=wan -13:09.876 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16681 network=tcp -13:09.876 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16681 network=udp -13:09.877 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16682 network=tcp -13:09.877 [INFO] TestAgent: started state syncer -13:09.877 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16694 network=tcp -13:09.917 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:09.917 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16686 [Candidate]" term=2 -13:09.918 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:09.918 [DEBUG] TestAgent.server.raft: vote granted: from=866513b9-1216-f9b3-2965-624611fc6db5 term=2 tally=1 -13:09.918 [INFO] TestAgent.server.raft: election won: tally=1 -13:09.918 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16686 [Leader]" -13:09.918 [INFO] TestAgent.server: cluster leadership acquired -13:09.918 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16686 -13:09.919 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:09.921 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:09.921 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:09.921 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:09.921 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:09.921 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:09.921 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-866513b9-1216-f9b3-2965-624611fc6db5 -13:09.921 [INFO] TestAgent.server: member joined, marking health alive: member=Node-866513b9-1216-f9b3-2965-624611fc6db5 -13:09.921 [INFO] TestAgent.server: New leader elected: payload=Node-866513b9-1216-f9b3-2965-624611fc6db5 -13:09.926 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:09.930 [INFO] TestAgent: Synced node info -13:09.930 [DEBUG] TestAgent: Node info in sync -13:09.936 [INFO] TestAgent.server: federation state anti-entropy synced -13:10.058 [DEBUG] TestAgent.server: User event: event=test -13:10.058 [DEBUG] TestAgent: new event: event_name=test event_id=d69a6469-0446-fc66-90c3-1be2a90c194b -13:10.110 [DEBUG] TestAgent.server: User event: event=second -13:10.110 [DEBUG] TestAgent: new event: event_name=second event_id=8932dd8d-68dc-55dd-703f-88dfc2a1f42a -13:10.137 [INFO] TestAgent: Requesting shutdown -13:10.137 [INFO] TestAgent.server: shutting down server -13:10.137 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:10.137 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:10.137 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:10.137 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:10.137 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.137 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:10.137 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:10.137 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:10.137 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:10.137 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:10.137 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:10.137Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.137Z [INFO] agent.router.manager: shutting down -13:10.137 [INFO] TestAgent: consul server down -13:10.137 [INFO] TestAgent: shutdown complete -13:10.137 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16681 network=tcp -13:10.137 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16681 network=udp -13:10.137 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16682 network=tcp -13:10.637 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.637 [INFO] TestAgent: Endpoints down ---- PASS: TestEventList_Blocking (0.92s) +--- PASS: TestDNS_trimUDPResponse_TrimSizeEDNS (0.01s) === CONT TestDNS_trimUDPResponse_TrimSize ---- PASS: TestDNS_trimUDPResponse_TrimSize (0.02s) -=== CONT TestDNS_trimUDPResponse_TrimLimit -2021-10-31T23:13:09.735Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:09.888Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:09.890 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9c04f2cc-1590-0ac8-52e6-d883cbe82db1 Address:127.0.0.1:16672}]" -13:09.891 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16672 [Follower]" leader= -13:09.891 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9c04f2cc-1590-0ac8-52e6-d883cbe82db1.dc1 127.0.0.1 -13:09.891 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9c04f2cc-1590-0ac8-52e6-d883cbe82db1 127.0.0.1 -2021-10-31T23:13:09.891Z [INFO] agent.router: Initializing LAN area manager -13:09.892 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16667 network=udp -13:09.892 [INFO] TestAgent.server: Adding LAN server: server="Node-9c04f2cc-1590-0ac8-52e6-d883cbe82db1 (Addr: tcp/127.0.0.1:16672) (DC: dc1)" -13:09.892 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9c04f2cc-1590-0ac8-52e6-d883cbe82db1.dc1 area=wan -13:09.892 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16667 network=tcp -13:09.893 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16668 network=tcp -13:09.893 [INFO] TestAgent: started state syncer -13:09.893 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16673 network=tcp -13:09.940 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:09.941 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16672 [Candidate]" term=2 -13:09.941 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:09.941 [DEBUG] TestAgent.server.raft: vote granted: from=9c04f2cc-1590-0ac8-52e6-d883cbe82db1 term=2 tally=1 -13:09.941 [INFO] TestAgent.server.raft: election won: tally=1 -13:09.941 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16672 [Leader]" -13:09.941 [INFO] TestAgent.server: cluster leadership acquired -13:09.941 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16672 -13:09.942 [INFO] TestAgent.server: New leader elected: payload=Node-9c04f2cc-1590-0ac8-52e6-d883cbe82db1 -13:09.942 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:09.944 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:09.944 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:09.944 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:09.944 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:09.944 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:09.944 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9c04f2cc-1590-0ac8-52e6-d883cbe82db1 -13:09.944 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9c04f2cc-1590-0ac8-52e6-d883cbe82db1 -13:09.944 [INFO] TestAgent.server: federation state anti-entropy synced -13:10.134 [DEBUG] TestAgent.server: User event: event=foo -13:10.134 [DEBUG] TestAgent.server: User event: event=bar -13:10.134 [DEBUG] TestAgent.server: User event: event=foo -13:10.134 [DEBUG] TestAgent.server: User event: event=foo -13:10.134 [DEBUG] TestAgent.server: User event: event=bar -13:10.134 [DEBUG] TestAgent: new event: event_name=foo event_id=6377a7a4-6d1f-e0dd-d66e-4c58f479c05a -13:10.134 [DEBUG] TestAgent: new event: event_name=bar event_id=7bc8d324-2a55-f228-078d-10f952aee261 -13:10.134 [DEBUG] TestAgent: new event: event_name=foo event_id=ad997614-1e77-cbe4-9b04-2bbc1ab2eeb0 -13:10.134 [DEBUG] TestAgent: new event: event_name=foo event_id=4e33806b-f32a-e43b-91a1-a7963ac8bef7 -13:10.134 [DEBUG] TestAgent: new event: event_name=bar event_id=a76d43fb-c4ea-5c5d-8278-57293cca60cf -13:10.160 [INFO] TestAgent: Requesting shutdown -13:10.160 [INFO] TestAgent.server: shutting down server -13:10.160 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:10.160 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:10.160 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:10.160 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:10.160 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.160 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:10.160 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:10.161 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:10.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:10.161 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:10.161 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:10.161 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:10.161Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.161Z [INFO] agent.router.manager: shutting down -13:10.161 [INFO] TestAgent: consul server down -13:10.161 [INFO] TestAgent: shutdown complete -13:10.161 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16667 network=tcp -13:10.161 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16667 network=udp -13:10.161 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16668 network=tcp -13:10.661 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.661 [INFO] TestAgent: Endpoints down ---- PASS: TestEventList_EventBufOrder (0.96s) +2022-12-04T05:49:22.629Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.639Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.640 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:90cc0ad9-76ca-02f9-07fa-b016a8f402fc Address:127.0.0.1:23433}]" +49:22.641 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-90cc0ad9-76ca-02f9-07fa-b016a8f402fc.dc1 127.0.0.1 +49:22.641 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-90cc0ad9-76ca-02f9-07fa-b016a8f402fc 127.0.0.1 +2022-12-04T05:49:22.641Z [INFO] agent.router: Initializing LAN area manager +49:22.641 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23428 network=udp +49:22.641 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23433 [Follower]" leader= +49:22.641 [INFO] TestAgent.server: Adding LAN server: server="Node-90cc0ad9-76ca-02f9-07fa-b016a8f402fc (Addr: tcp/127.0.0.1:23433) (DC: dc1)" +49:22.641 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-90cc0ad9-76ca-02f9-07fa-b016a8f402fc.dc1 area=wan +49:22.642 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23428 network=tcp +49:22.642 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23429 network=tcp +49:22.642 [INFO] TestAgent: started state syncer +49:22.642 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23434 network=tcp +49:22.704 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.704 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23433 [Candidate]" term=2 +49:22.704 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.704 [DEBUG] TestAgent.server.raft: vote granted: from=90cc0ad9-76ca-02f9-07fa-b016a8f402fc term=2 tally=1 +49:22.704 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.704 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23433 [Leader]" +49:22.704 [INFO] TestAgent.server: cluster leadership acquired +49:22.704 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23433 +49:22.704 [INFO] TestAgent.server: New leader elected: payload=Node-90cc0ad9-76ca-02f9-07fa-b016a8f402fc +49:22.705 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.706 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.706 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.706 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.706 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.706 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.706 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-90cc0ad9-76ca-02f9-07fa-b016a8f402fc +49:22.706 [INFO] TestAgent.server: member joined, marking health alive: member=Node-90cc0ad9-76ca-02f9-07fa-b016a8f402fc +49:22.707 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.784 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:22.784 [INFO] TestAgent: Synced node info +49:22.868 [INFO] TestAgent: Requesting shutdown +49:22.868 [INFO] TestAgent.server: shutting down server +49:22.868 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.868 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.868 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.868 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.868 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.868 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.868 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.868 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.868 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.868 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.868 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.868Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.868Z [INFO] agent.router.manager: shutting down +49:22.868 [INFO] TestAgent: consul server down +49:22.868 [INFO] TestAgent: shutdown complete +49:22.868 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23428 network=tcp +49:22.868 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23428 network=udp +49:22.868 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23429 network=tcp +49:23.369 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.369 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthChecksInState_DistanceSort (0.75s) === CONT TestDNS_InvalidQueries -2021-10-31T23:13:09.728Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:09.791Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:09.810 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:45844388-e12e-28a0-5de1-e9031ef20c07 Address:127.0.0.1:16679}]" -13:09.811 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-45844388-e12e-28a0-5de1-e9031ef20c07.dc1 127.0.0.1 -13:09.812 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-45844388-e12e-28a0-5de1-e9031ef20c07 127.0.0.1 -2021-10-31T23:13:09.812Z [INFO] agent.router: Initializing LAN area manager -13:09.862 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16674 network=udp -13:09.862 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16679 [Follower]" leader= -13:09.862 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-45844388-e12e-28a0-5de1-e9031ef20c07.dc1 area=wan -13:09.862 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16674 network=tcp -13:09.863 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16675 network=tcp -13:09.863 [INFO] TestAgent: started state syncer -13:09.863 [INFO] TestAgent.server: Adding LAN server: server="Node-45844388-e12e-28a0-5de1-e9031ef20c07 (Addr: tcp/127.0.0.1:16679) (DC: dc1)" -13:09.863 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16687 network=tcp -13:09.917 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:09.917 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16679 [Candidate]" term=2 -13:09.918 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:09.918 [DEBUG] TestAgent.server.raft: vote granted: from=45844388-e12e-28a0-5de1-e9031ef20c07 term=2 tally=1 -13:09.918 [INFO] TestAgent.server.raft: election won: tally=1 -13:09.918 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16679 [Leader]" -13:09.918 [INFO] TestAgent.server: cluster leadership acquired -13:09.918 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16679 -13:09.919 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:09.920 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:09.920 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:09.920 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:09.920 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:09.920 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:09.920 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-45844388-e12e-28a0-5de1-e9031ef20c07 -13:09.920 [INFO] TestAgent.server: member joined, marking health alive: member=Node-45844388-e12e-28a0-5de1-e9031ef20c07 -13:09.921 [INFO] TestAgent.server: New leader elected: payload=Node-45844388-e12e-28a0-5de1-e9031ef20c07 -13:09.921 [INFO] TestAgent.server: federation state anti-entropy synced -13:10.178 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:10.178 [INFO] TestAgent: Synced node info -13:10.178 [DEBUG] TestAgent: Node info in sync -13:10.178 [INFO] TestAgent: Requesting shutdown -13:10.178 [INFO] TestAgent.server: shutting down server -13:10.178 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:10.178 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:10.178 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:10.178 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:10.178 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.178 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:10.178 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:10.178 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:10.178 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:10.178 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:10.178 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:10.178Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.178Z [INFO] agent.router.manager: shutting down -13:10.179 [INFO] TestAgent: consul server down -13:10.179 [INFO] TestAgent: shutdown complete -13:10.179 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16674 network=tcp -13:10.179 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16674 network=udp -13:10.179 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16675 network=tcp -13:10.680 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.680 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsCreate_noBody (0.98s) -=== CONT TestDNS_NonExistingLookupEmptyAorAAAA -2021-10-31T23:13:09.752Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:09.752Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:09.870Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:09.870Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:09.872 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e9eb2c71-13d3-91a4-1343-e4697d798600 Address:127.0.0.1:16693}]" -13:09.872 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e9eb2c71-13d3-91a4-1343-e4697d798600.dc1 127.0.0.1 -13:09.873 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e9eb2c71-13d3-91a4-1343-e4697d798600 127.0.0.1 -2021-10-31T23:13:09.873Z [INFO] agent.router: Initializing LAN area manager -13:09.873 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16688 network=udp -13:09.873 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16693 [Follower]" leader= -13:09.874 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e9eb2c71-13d3-91a4-1343-e4697d798600.dc1 area=wan -13:09.874 [INFO] TestAgent.server: Adding LAN server: server="Node-e9eb2c71-13d3-91a4-1343-e4697d798600 (Addr: tcp/127.0.0.1:16693) (DC: dc1)" -13:09.876 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16688 network=tcp -13:09.890 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16689 network=tcp -13:09.890 [INFO] TestAgent: started state syncer -13:09.891 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16680 network=tcp -13:09.925 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:09.961 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:09.961 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16693 [Candidate]" term=2 -13:09.962 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:09.962 [DEBUG] TestAgent.server.raft: vote granted: from=e9eb2c71-13d3-91a4-1343-e4697d798600 term=2 tally=1 -13:09.962 [INFO] TestAgent.server.raft: election won: tally=1 -13:09.962 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16693 [Leader]" -13:09.962 [INFO] TestAgent.server: cluster leadership acquired -13:09.962 [INFO] TestAgent.server: initializing acls -13:09.962 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:09.962 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:09.963 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:09.963 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:09.963 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:09.963 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:09.963 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e9eb2c71-13d3-91a4-1343-e4697d798600 -13:09.963 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e9eb2c71-13d3-91a4-1343-e4697d798600.dc1 -13:09.964 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:09.966 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e9eb2c71-13d3-91a4-1343-e4697d798600.dc1 area=wan -13:09.966 [INFO] TestAgent.server: New leader elected: payload=Node-e9eb2c71-13d3-91a4-1343-e4697d798600 -13:09.966 [INFO] TestAgent.server: Updating LAN server: server="Node-e9eb2c71-13d3-91a4-1343-e4697d798600 (Addr: tcp/127.0.0.1:16693) (DC: dc1)" -13:09.994 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:09.994 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:09.994 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:09.994 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:09.994 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:09.994 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e9eb2c71-13d3-91a4-1343-e4697d798600 -13:09.994 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e9eb2c71-13d3-91a4-1343-e4697d798600 -13:09.999 [INFO] TestAgent.server: federation state anti-entropy synced -13:09.999 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e9eb2c71-13d3-91a4-1343-e4697d798600 -13:10.186 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e9eb2c71-13d3-91a4-1343-e4697d798600 -13:10.186 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e9eb2c71-13d3-91a4-1343-e4697d798600 -13:10.187 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=foo accessorID= -13:10.187 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=bar accessorID= -13:10.187 [INFO] TestAgent: Requesting shutdown -13:10.187 [INFO] TestAgent.server: shutting down server -13:10.187 [DEBUG] TestAgent.server: User event: event=baz -13:10.187 [DEBUG] TestAgent: new event: event_name=baz event_id=aa69809a-6020-d2ad-f605-32d1f8e36218 -13:10.187 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:10.187 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:10.187 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:10.187 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:10.187 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:10.187 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:10.187 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:10.187 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:10.187 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:10.187 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:10.187 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:10.187 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.187 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:10.187 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:10.188 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:10.188Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.188Z [INFO] agent.router.manager: shutting down -13:10.188 [INFO] TestAgent: consul server down -13:10.188 [INFO] TestAgent: shutdown complete -13:10.188 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16688 network=tcp -13:10.188 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16688 network=udp -13:10.188 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16689 network=tcp -13:10.689 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.689 [INFO] TestAgent: Endpoints down ---- PASS: TestEventFire_token (0.98s) +--- PASS: TestDNS_trimUDPResponse_TrimSize (0.01s) === CONT TestDNS_PreparedQuery_AllowStale ---- PASS: TestDNS_trimUDPResponse_TrimLimit (0.04s) +2022-12-04T05:49:22.595Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.611Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.613 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b870f68e-5e54-6809-a17d-5a3326c8c42b Address:127.0.0.1:23412}]" +49:22.613 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23412 [Follower]" leader= +49:22.614 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b870f68e-5e54-6809-a17d-5a3326c8c42b.dc1 127.0.0.1 +49:22.614 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b870f68e-5e54-6809-a17d-5a3326c8c42b 127.0.0.1 +2022-12-04T05:49:22.614Z [INFO] agent.router: Initializing LAN area manager +49:22.614 [INFO] TestAgent.server: Adding LAN server: server="Node-b870f68e-5e54-6809-a17d-5a3326c8c42b (Addr: tcp/127.0.0.1:23412) (DC: dc1)" +49:22.614 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b870f68e-5e54-6809-a17d-5a3326c8c42b.dc1 area=wan +49:22.614 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23463 network=udp +49:22.614 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23463 network=tcp +49:22.615 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23464 network=tcp +49:22.615 [INFO] TestAgent: started state syncer +49:22.615 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23413 network=tcp +49:22.662 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.662 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23412 [Candidate]" term=2 +49:22.662 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.662 [DEBUG] TestAgent.server.raft: vote granted: from=b870f68e-5e54-6809-a17d-5a3326c8c42b term=2 tally=1 +49:22.662 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.662 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23412 [Leader]" +49:22.662 [INFO] TestAgent.server: cluster leadership acquired +49:22.662 [INFO] TestAgent.server: New leader elected: payload=Node-b870f68e-5e54-6809-a17d-5a3326c8c42b +49:22.662 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23412 +49:22.663 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.664 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.664 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.664 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.664 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.664 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.664 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b870f68e-5e54-6809-a17d-5a3326c8c42b +49:22.664 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b870f68e-5e54-6809-a17d-5a3326c8c42b +49:22.665 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.803 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:22.804 [INFO] TestAgent: Synced node info +49:23.046 [INFO] TestAgent: Requesting shutdown +49:23.046 [INFO] TestAgent.server: shutting down server +49:23.046 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.046 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.046 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.046 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.046 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.046 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.046 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.047 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.047 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.047 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:23.047 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:23.047Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.047Z [INFO] agent.router.manager: shutting down +49:23.047 [INFO] TestAgent: consul server down +49:23.047 [INFO] TestAgent: shutdown complete +49:23.047 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23463 network=tcp +49:23.047 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23463 network=udp +49:23.047 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23464 network=tcp +49:23.547 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.547 [INFO] TestAgent: Endpoints down +--- PASS: TestHealthNodeChecks (0.97s) +=== CONT TestDNS_trimUDPResponse_TrimLimit +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +--- PASS: TestDNS_trimUDPResponse_TrimLimit (0.01s) === CONT TestDNS_AltDomains_Overlap -2021-10-31T23:13:09.931Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:09.997Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:09.999 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ffc5cd0c-72b4-3919-a9e3-5596aba07b71 Address:127.0.0.1:16714}]" -13:10.000 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ffc5cd0c-72b4-3919-a9e3-5596aba07b71.dc1 127.0.0.1 -13:10.001 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ffc5cd0c-72b4-3919-a9e3-5596aba07b71 127.0.0.1 -2021-10-31T23:13:10.001Z [INFO] agent.router: Initializing LAN area manager -13:10.001 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16709 network=udp -13:10.001 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16714 [Follower]" leader= -13:10.003 [INFO] TestAgent.server: Adding LAN server: server="Node-ffc5cd0c-72b4-3919-a9e3-5596aba07b71 (Addr: tcp/127.0.0.1:16714) (DC: dc1)" -13:10.003 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ffc5cd0c-72b4-3919-a9e3-5596aba07b71.dc1 area=wan -13:10.003 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16709 network=tcp -13:10.004 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16710 network=tcp -13:10.005 [INFO] TestAgent: started state syncer -13:10.005 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16743 network=tcp -13:10.045 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:10.046 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16714 [Candidate]" term=2 -13:10.066 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:10.066 [DEBUG] TestAgent.server.raft: vote granted: from=ffc5cd0c-72b4-3919-a9e3-5596aba07b71 term=2 tally=1 -13:10.066 [INFO] TestAgent.server.raft: election won: tally=1 -13:10.066 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16714 [Leader]" -13:10.066 [INFO] TestAgent.server: cluster leadership acquired -13:10.066 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16714 -13:10.066 [INFO] TestAgent.server: New leader elected: payload=Node-ffc5cd0c-72b4-3919-a9e3-5596aba07b71 -13:10.067 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:10.068 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:10.068 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:10.068 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:10.068 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:10.068 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:10.068 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ffc5cd0c-72b4-3919-a9e3-5596aba07b71 -13:10.068 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ffc5cd0c-72b4-3919-a9e3-5596aba07b71 -13:10.069 [INFO] TestAgent.server: federation state anti-entropy synced -13:10.110 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:10.110 [INFO] TestAgent: Synced node info -13:10.267 [INFO] TestAgent: Requesting shutdown -13:10.267 [INFO] TestAgent.server: shutting down server -13:10.267 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:10.267 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:10.267 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:10.267 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:10.267 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.267 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:10.267 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:10.267 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:10.267 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:10.267 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:10.267 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:10.267Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.267Z [INFO] agent.router.manager: shutting down -13:10.267 [INFO] TestAgent: consul server down -13:10.267 [INFO] TestAgent: shutdown complete -13:10.267 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16709 network=tcp -13:10.267 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16709 network=udp -13:10.267 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16710 network=tcp -13:10.768 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.768 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsCheck_noDestination (0.87s) -=== CONT TestDNS_NonExistingLookup -2021-10-31T23:13:09.752Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:09.890Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:09.892 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7683c504-f450-82ba-49ab-6b6a8e8740da Address:127.0.0.1:16707}]" -13:09.892 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7683c504-f450-82ba-49ab-6b6a8e8740da.dc1 127.0.0.1 -13:09.893 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7683c504-f450-82ba-49ab-6b6a8e8740da 127.0.0.1 -2021-10-31T23:13:09.893Z [INFO] agent.router: Initializing LAN area manager -13:09.893 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16707 [Follower]" leader= -13:09.894 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7683c504-f450-82ba-49ab-6b6a8e8740da.dc1 area=wan -13:09.894 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16702 network=udp -13:09.894 [INFO] TestAgent.server: Adding LAN server: server="Node-7683c504-f450-82ba-49ab-6b6a8e8740da (Addr: tcp/127.0.0.1:16707) (DC: dc1)" -13:09.894 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16702 network=tcp -13:09.894 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16703 network=tcp -13:09.894 [INFO] TestAgent: started state syncer -13:09.894 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16729 network=tcp -13:09.965 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:09.966 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16707 [Candidate]" term=2 -13:09.966 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:09.966 [DEBUG] TestAgent.server.raft: vote granted: from=7683c504-f450-82ba-49ab-6b6a8e8740da term=2 tally=1 -13:09.966 [INFO] TestAgent.server.raft: election won: tally=1 -13:09.966 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16707 [Leader]" -13:09.966 [INFO] TestAgent.server: cluster leadership acquired -13:09.966 [INFO] TestAgent.server: New leader elected: payload=Node-7683c504-f450-82ba-49ab-6b6a8e8740da -13:09.966 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16707 -13:09.967 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:09.968 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:09.968 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:09.968 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:09.968 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:09.968 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:09.968 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7683c504-f450-82ba-49ab-6b6a8e8740da -13:09.968 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7683c504-f450-82ba-49ab-6b6a8e8740da -13:10.006 [INFO] TestAgent.server: federation state anti-entropy synced -13:10.150 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:10.150 [INFO] TestAgent: Synced node info -13:10.294 [INFO] TestAgent: Requesting shutdown -13:10.294 [INFO] TestAgent.server: shutting down server -13:10.294 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:10.294 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:10.294 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:10.294 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:10.294 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.294 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:10.294 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:10.295 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:10.295 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:10.295 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:10.295 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:10.295Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.295Z [INFO] agent.router.manager: shutting down -13:10.295 [INFO] TestAgent: consul server down -13:10.295 [INFO] TestAgent: shutdown complete -13:10.295 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16702 network=tcp -13:10.295 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16702 network=udp -13:10.295 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16703 network=tcp -13:10.797 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.797 [INFO] TestAgent: Endpoints down ---- PASS: TestIntentionsCreate_good (1.08s) +2022-12-04T05:49:22.706Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.724Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.726 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:195a6c85-cbe5-7d93-3ca4-61fd176a344b Address:127.0.0.1:23496}]" +49:22.726 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23496 [Follower]" leader= +49:22.726 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-195a6c85-cbe5-7d93-3ca4-61fd176a344b.dc1 127.0.0.1 +49:22.727 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-195a6c85-cbe5-7d93-3ca4-61fd176a344b 127.0.0.1 +2022-12-04T05:49:22.727Z [INFO] agent.router: Initializing LAN area manager +49:22.727 [INFO] TestAgent.server: Adding LAN server: server="Node-195a6c85-cbe5-7d93-3ca4-61fd176a344b (Addr: tcp/127.0.0.1:23496) (DC: dc1)" +49:22.727 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-195a6c85-cbe5-7d93-3ca4-61fd176a344b.dc1 area=wan +49:22.727 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23449 network=udp +49:22.727 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23449 network=tcp +49:22.727 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23450 network=tcp +49:22.728 [INFO] TestAgent: started state syncer +49:22.728 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23497 network=tcp +49:22.780 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.780 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23496 [Candidate]" term=2 +49:22.780 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.780 [DEBUG] TestAgent.server.raft: vote granted: from=195a6c85-cbe5-7d93-3ca4-61fd176a344b term=2 tally=1 +49:22.780 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.780 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23496 [Leader]" +49:22.781 [INFO] TestAgent.server: cluster leadership acquired +49:22.781 [INFO] TestAgent.server: New leader elected: payload=Node-195a6c85-cbe5-7d93-3ca4-61fd176a344b +49:22.781 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23496 +49:22.781 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.783 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.783 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.783 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.783 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.783 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.783 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-195a6c85-cbe5-7d93-3ca4-61fd176a344b +49:22.783 [INFO] TestAgent.server: member joined, marking health alive: member=Node-195a6c85-cbe5-7d93-3ca4-61fd176a344b +49:22.784 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.009 [DEBUG] TestAgent.server: User event: event=test +49:23.009 [DEBUG] TestAgent: new event: event_name=test event_id=6da20315-b585-c9fb-58a8-d9db52429888 +49:23.060 [DEBUG] TestAgent.server: User event: event=second +49:23.060 [DEBUG] TestAgent: new event: event_name=second event_id=2f36e95e-5c2d-2062-f32b-e52bc3413b39 +49:23.085 [INFO] TestAgent: Requesting shutdown +49:23.085 [INFO] TestAgent.server: shutting down server +49:23.085 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.085 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.085 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.085 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.085 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.085 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.085 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.085 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:23.085 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.085 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.085 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:23.085 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:23.085Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.085Z [INFO] agent.router.manager: shutting down +49:23.085 [INFO] TestAgent: consul server down +49:23.085 [INFO] TestAgent: shutdown complete +49:23.085 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23449 network=tcp +49:23.085 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23449 network=udp +49:23.085 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23450 network=tcp +49:23.586 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.586 [INFO] TestAgent: Endpoints down +--- PASS: TestEventList_Blocking (0.89s) +=== CONT TestDNS_trimUDPResponse_NoTrim +--- PASS: TestDNS_trimUDPResponse_NoTrim (0.01s) +=== CONT TestDNS_NonExistentDC_Server +2022-12-04T05:49:22.965Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:22.965Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.975Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:22.975Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.977 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c09ba833-2569-35c7-47af-0b4c15c83e43 Address:127.0.0.1:23524}]" +49:22.977 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23524 [Follower]" leader= +49:22.977 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c09ba833-2569-35c7-47af-0b4c15c83e43.dc1 127.0.0.1 +49:22.978 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c09ba833-2569-35c7-47af-0b4c15c83e43 127.0.0.1 +2022-12-04T05:49:22.978Z [INFO] agent.router: Initializing LAN area manager +49:22.978 [INFO] TestAgent.server: Adding LAN server: server="Node-c09ba833-2569-35c7-47af-0b4c15c83e43 (Addr: tcp/127.0.0.1:23524) (DC: dc1)" +49:22.978 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c09ba833-2569-35c7-47af-0b4c15c83e43.dc1 area=wan +49:22.978 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23512 network=udp +49:22.978 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23512 network=tcp +49:22.979 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23513 network=tcp +49:22.979 [INFO] TestAgent: started state syncer +49:22.979 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23525 network=tcp +49:23.015 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.015 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23524 [Candidate]" term=2 +49:23.015 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.015 [DEBUG] TestAgent.server.raft: vote granted: from=c09ba833-2569-35c7-47af-0b4c15c83e43 term=2 tally=1 +49:23.015 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.015 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23524 [Leader]" +49:23.015 [INFO] TestAgent.server: cluster leadership acquired +49:23.015 [INFO] TestAgent.server: New leader elected: payload=Node-c09ba833-2569-35c7-47af-0b4c15c83e43 +49:23.015 [INFO] TestAgent.server: initializing acls +49:23.015 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:23.015 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:23.016 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:23.016 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:23.016 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:23.016 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:23.016 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c09ba833-2569-35c7-47af-0b4c15c83e43 +49:23.016 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c09ba833-2569-35c7-47af-0b4c15c83e43.dc1 +49:23.017 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.017 [INFO] TestAgent.server: Updating LAN server: server="Node-c09ba833-2569-35c7-47af-0b4c15c83e43 (Addr: tcp/127.0.0.1:23524) (DC: dc1)" +49:23.017 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c09ba833-2569-35c7-47af-0b4c15c83e43.dc1 area=wan +49:23.018 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.018 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.018 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.018 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.018 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.018 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c09ba833-2569-35c7-47af-0b4c15c83e43 +49:23.018 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c09ba833-2569-35c7-47af-0b4c15c83e43 +49:23.018 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c09ba833-2569-35c7-47af-0b4c15c83e43 +49:23.018 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.101 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-c09ba833-2569-35c7-47af-0b4c15c83e43 +49:23.101 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-c09ba833-2569-35c7-47af-0b4c15c83e43 +49:23.112 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=foo accessorID= +49:23.112 [WARN] TestAgent.server.internal: user event blocked by ACLs: event=bar accessorID= +49:23.112 [INFO] TestAgent: Requesting shutdown +49:23.113 [INFO] TestAgent.server: shutting down server +49:23.113 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:23.113 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:23.113 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.113 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.113 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.113 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.113 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.113 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.113 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.113 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:23.113 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:23.113 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:23.113 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.113 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.113 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:23.113 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:23.113Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.113Z [INFO] agent.router.manager: shutting down +49:23.113 [INFO] TestAgent: consul server down +49:23.113 [INFO] TestAgent: shutdown complete +49:23.113 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23512 network=tcp +49:23.113 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23512 network=udp +49:23.113 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23513 network=tcp +49:23.616 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.616 [INFO] TestAgent: Endpoints down +--- PASS: TestEventFire_token (0.66s) +=== CONT TestDNS_PreparedQuery_AgentSource +2022-12-04T05:49:22.821Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:22.821Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.831Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:22.831Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.832 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9ec42b39-0ab7-76c9-c29b-1a2be7fad93b Address:127.0.0.1:23503}]" +49:22.832 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23503 [Follower]" leader= +49:22.832 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9ec42b39-0ab7-76c9-c29b-1a2be7fad93b.dc1 127.0.0.1 +49:22.833 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9ec42b39-0ab7-76c9-c29b-1a2be7fad93b 127.0.0.1 +2022-12-04T05:49:22.833Z [INFO] agent.router: Initializing LAN area manager +49:22.833 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23498 network=udp +49:22.833 [INFO] TestAgent.server: Adding LAN server: server="Node-9ec42b39-0ab7-76c9-c29b-1a2be7fad93b (Addr: tcp/127.0.0.1:23503) (DC: dc1)" +49:22.833 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9ec42b39-0ab7-76c9-c29b-1a2be7fad93b.dc1 area=wan +49:22.833 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23498 network=tcp +49:22.834 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23499 network=tcp +49:22.834 [INFO] TestAgent: started state syncer +49:22.834 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23504 network=tcp +49:22.868 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.868 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23503 [Candidate]" term=2 +49:22.868 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.868 [DEBUG] TestAgent.server.raft: vote granted: from=9ec42b39-0ab7-76c9-c29b-1a2be7fad93b term=2 tally=1 +49:22.868 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.868 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23503 [Leader]" +49:22.868 [INFO] TestAgent.server: cluster leadership acquired +49:22.868 [INFO] TestAgent.server: New leader elected: payload=Node-9ec42b39-0ab7-76c9-c29b-1a2be7fad93b +49:22.868 [INFO] TestAgent.server: initializing acls +49:22.868 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:22.868 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:22.869 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:22.869 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:22.869 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:22.869 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:22.869 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9ec42b39-0ab7-76c9-c29b-1a2be7fad93b +49:22.869 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9ec42b39-0ab7-76c9-c29b-1a2be7fad93b.dc1 +49:22.870 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.870 [INFO] TestAgent.server: Updating LAN server: server="Node-9ec42b39-0ab7-76c9-c29b-1a2be7fad93b (Addr: tcp/127.0.0.1:23503) (DC: dc1)" +49:22.870 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9ec42b39-0ab7-76c9-c29b-1a2be7fad93b.dc1 area=wan +49:22.871 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.871 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.871 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.871 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.871 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.871 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9ec42b39-0ab7-76c9-c29b-1a2be7fad93b +49:22.871 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9ec42b39-0ab7-76c9-c29b-1a2be7fad93b +49:22.872 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.872 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9ec42b39-0ab7-76c9-c29b-1a2be7fad93b +49:22.997 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:22.997 [INFO] TestAgent: Synced node info +49:22.997 [DEBUG] TestAgent: Node info in sync +49:23.115 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9ec42b39-0ab7-76c9-c29b-1a2be7fad93b +49:23.116 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9ec42b39-0ab7-76c9-c29b-1a2be7fad93b +49:23.116 [DEBUG] TestAgent.server: User event: event=foo +49:23.116 [DEBUG] TestAgent: new event: event_name=foo event_id=39b33b45-92e4-0ee6-f421-d0d11562c1b1 +49:23.116 [INFO] TestAgent: Requesting shutdown +49:23.116 [INFO] TestAgent.server: shutting down server +49:23.116 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.116 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:23.116 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:23.116 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.116 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.116 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.116 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.116 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.116 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.116 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:23.116 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:23.116 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:23.116 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.116 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.116 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:23.117Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.117Z [INFO] agent.router.manager: shutting down +49:23.117 [INFO] TestAgent: consul server down +49:23.117 [INFO] TestAgent: shutdown complete +49:23.117 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23498 network=tcp +49:23.117 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23498 network=udp +49:23.117 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23499 network=tcp +49:23.617 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.617 [INFO] TestAgent: Endpoints down +--- PASS: TestEventList_ACLFilter (0.81s) + --- PASS: TestEventList_ACLFilter/no_token (0.00s) + --- PASS: TestEventList_ACLFilter/root_token (0.00s) +=== CONT TestDNS_AddressLookupIPV6 +2022-12-04T05:49:23.067Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.079Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.081 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:50108990-fa27-acfe-0b37-7dae45633322 Address:127.0.0.1:23538}]" +49:23.081 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23538 [Follower]" leader= +49:23.082 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-50108990-fa27-acfe-0b37-7dae45633322.dc1 127.0.0.1 +49:23.082 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-50108990-fa27-acfe-0b37-7dae45633322 127.0.0.1 +2022-12-04T05:49:23.082Z [INFO] agent.router: Initializing LAN area manager +49:23.082 [INFO] TestAgent.server: Adding LAN server: server="Node-50108990-fa27-acfe-0b37-7dae45633322 (Addr: tcp/127.0.0.1:23538) (DC: dc1)" +49:23.082 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-50108990-fa27-acfe-0b37-7dae45633322.dc1 area=wan +49:23.082 [DEBUG] TestAgent.dns: recursor enabled +49:23.082 [DEBUG] TestAgent.dns: recursor enabled +49:23.082 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23554 network=udp +49:23.082 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23554 network=tcp +49:23.083 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23555 network=tcp +49:23.083 [INFO] TestAgent: started state syncer +49:23.083 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23539 network=tcp +49:23.124 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.124 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23538 [Candidate]" term=2 +49:23.124 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.124 [DEBUG] TestAgent.server.raft: vote granted: from=50108990-fa27-acfe-0b37-7dae45633322 term=2 tally=1 +49:23.124 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.124 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23538 [Leader]" +49:23.124 [INFO] TestAgent.server: cluster leadership acquired +49:23.124 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23538 +49:23.124 [INFO] TestAgent.server: New leader elected: payload=Node-50108990-fa27-acfe-0b37-7dae45633322 +49:23.125 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.126 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.126 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.126 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.126 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.126 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.126 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-50108990-fa27-acfe-0b37-7dae45633322 +49:23.126 [INFO] TestAgent.server: member joined, marking health alive: member=Node-50108990-fa27-acfe-0b37-7dae45633322 +49:23.127 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.147 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{apple.com. 255 1}" rtt=275.773µs recursor=127.0.0.1:56693 +49:23.147 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=539.967µs client=127.0.0.1:50181 client_network=udp +49:23.147 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{apple.com. 255 1}" rtt=134.232µs recursor=127.0.0.1:56693 +49:23.147 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=267.497µs client=127.0.0.1:50181 client_network=udp +49:23.147 [INFO] TestAgent: Requesting shutdown +49:23.147 [INFO] TestAgent.server: shutting down server +49:23.147 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.147 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.147 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.147 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.147 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.147 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.147 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.147 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:23.147 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:23.147 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.147 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.147 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:23.147Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.147Z [INFO] agent.router.manager: shutting down +49:23.147 [INFO] TestAgent: consul server down +49:23.147 [INFO] TestAgent: shutdown complete +49:23.147 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23554 network=tcp +49:23.147 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23554 network=udp +49:23.148 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23555 network=tcp +49:23.648 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.648 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_Compression_Recurse (0.59s) === CONT TestDNS_AltDomains_SOA -2021-10-31T23:13:08.734Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:08.821Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:08.838 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c63b9f74-43ba-6de2-9743-94b62ea0b05e Address:127.0.0.1:16582}]" -13:08.869 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c63b9f74-43ba-6de2-9743-94b62ea0b05e.dc1 127.0.0.1 -13:08.870 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c63b9f74-43ba-6de2-9743-94b62ea0b05e 127.0.0.1 -2021-10-31T23:13:08.870Z [INFO] agent.router: Initializing LAN area manager -13:08.871 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16577 network=udp -13:08.871 [INFO] TestAgent.server: Adding LAN server: server="Node-c63b9f74-43ba-6de2-9743-94b62ea0b05e (Addr: tcp/127.0.0.1:16582) (DC: dc1)" -13:08.871 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c63b9f74-43ba-6de2-9743-94b62ea0b05e.dc1 area=wan -13:08.871 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16577 network=tcp -13:08.872 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16578 network=tcp -13:08.872 [INFO] TestAgent: started state syncer -13:08.872 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16583 network=tcp -13:08.872 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16582 [Follower]" leader= -13:08.921 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:08.922 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16582 [Candidate]" term=2 -13:08.922 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:08.922 [DEBUG] TestAgent.server.raft: vote granted: from=c63b9f74-43ba-6de2-9743-94b62ea0b05e term=2 tally=1 -13:08.922 [INFO] TestAgent.server.raft: election won: tally=1 -13:08.922 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16582 [Leader]" -13:08.922 [INFO] TestAgent.server: cluster leadership acquired -13:08.922 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16582 -13:08.922 [INFO] TestAgent.server: New leader elected: payload=Node-c63b9f74-43ba-6de2-9743-94b62ea0b05e -13:08.923 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:08.924 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:08.924 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:08.924 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:08.924 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:08.924 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:08.924 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c63b9f74-43ba-6de2-9743-94b62ea0b05e -13:08.924 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c63b9f74-43ba-6de2-9743-94b62ea0b05e -13:08.925 [INFO] TestAgent.server: federation state anti-entropy synced -13:09.086 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:09.086 [INFO] TestAgent: Synced node info -13:09.086 [DEBUG] TestAgent: Node info in sync -13:09.615 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:09.615 [DEBUG] TestAgent: Node info in sync -13:10.339 [INFO] TestAgent: Requesting shutdown -13:10.340 [INFO] TestAgent.server: shutting down server -13:10.340 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:10.340 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:10.340 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:10.340 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:10.340 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.340 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:10.340 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:10.340 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:10.340 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:10.340 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:10.340 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:10.340Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.340Z [INFO] agent.router.manager: shutting down -13:10.340 [INFO] TestAgent: consul server down -13:10.340 [INFO] TestAgent: shutdown complete -13:10.340 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16577 network=tcp -13:10.340 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16577 network=udp -13:10.340 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16578 network=tcp -13:10.841 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.841 [INFO] TestAgent: Endpoints down -=== RUN TestTxnEndpoint_Bad_Size_Net/allowed -2021-10-31T23:13:10.092Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:10.155Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:10.157 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2a71019e-3041-bd82-00e6-2d1f04ede867 Address:127.0.0.1:16763}]" -13:10.166 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16763 [Follower]" leader= -13:10.166 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2a71019e-3041-bd82-00e6-2d1f04ede867.dc1 127.0.0.1 -13:10.167 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2a71019e-3041-bd82-00e6-2d1f04ede867 127.0.0.1 -2021-10-31T23:13:10.167Z [INFO] agent.router: Initializing LAN area manager -13:10.167 [DEBUG] TestAgent.dns: recursor enabled -13:10.167 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16758 network=udp -13:10.167 [INFO] TestAgent.server: Adding LAN server: server="Node-2a71019e-3041-bd82-00e6-2d1f04ede867 (Addr: tcp/127.0.0.1:16763) (DC: dc1)" -13:10.167 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2a71019e-3041-bd82-00e6-2d1f04ede867.dc1 area=wan -13:10.168 [DEBUG] TestAgent.dns: recursor enabled -13:10.168 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16758 network=tcp -13:10.168 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16759 network=tcp -13:10.168 [INFO] TestAgent: started state syncer -13:10.168 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16750 network=tcp -13:10.205 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:10.205 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16763 [Candidate]" term=2 -13:10.206 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:10.206 [DEBUG] TestAgent.server.raft: vote granted: from=2a71019e-3041-bd82-00e6-2d1f04ede867 term=2 tally=1 -13:10.206 [INFO] TestAgent.server.raft: election won: tally=1 -13:10.206 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16763 [Leader]" -13:10.206 [INFO] TestAgent.server: cluster leadership acquired -13:10.206 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16763 -13:10.206 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:10.208 [INFO] TestAgent.server: New leader elected: payload=Node-2a71019e-3041-bd82-00e6-2d1f04ede867 -13:10.208 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:10.208 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:10.208 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:10.208 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:10.208 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:10.208 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2a71019e-3041-bd82-00e6-2d1f04ede867 -13:10.208 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2a71019e-3041-bd82-00e6-2d1f04ede867 -13:10.208 [INFO] TestAgent.server: federation state anti-entropy synced -13:10.350 [INFO] TestAgent: Requesting shutdown -13:10.351 [INFO] TestAgent.server: shutting down server -13:10.351 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.351 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:10.351 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:10.351 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:10.351Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.351Z [INFO] agent.router.manager: shutting down -13:10.351 [INFO] TestAgent: consul server down -13:10.351 [INFO] TestAgent: shutdown complete -13:10.351 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16758 network=tcp -13:10.351 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16758 network=udp -13:10.351 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16759 network=tcp -13:10.857 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.858 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ConfigReload (0.79s) -[WARN] freeport: 10 out of 66 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:23.044Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.054Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.056 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:359cca84-cc4e-81f7-36d0-baeede586357 Address:127.0.0.1:23552}]" +49:23.056 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23552 [Follower]" leader= +49:23.056 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-359cca84-cc4e-81f7-36d0-baeede586357.dc1 127.0.0.1 +49:23.057 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-359cca84-cc4e-81f7-36d0-baeede586357 127.0.0.1 +2022-12-04T05:49:23.057Z [INFO] agent.router: Initializing LAN area manager +49:23.057 [INFO] TestAgent.server: Adding LAN server: server="Node-359cca84-cc4e-81f7-36d0-baeede586357 (Addr: tcp/127.0.0.1:23552) (DC: dc1)" +49:23.057 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-359cca84-cc4e-81f7-36d0-baeede586357.dc1 area=wan +49:23.057 [DEBUG] TestAgent.dns: recursor enabled +49:23.057 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23561 network=udp +49:23.057 [DEBUG] TestAgent.dns: recursor enabled +49:23.057 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23561 network=tcp +49:23.058 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23562 network=tcp +49:23.058 [INFO] TestAgent: started state syncer +49:23.058 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23553 network=tcp +49:23.101 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.101 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23552 [Candidate]" term=2 +49:23.101 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.101 [DEBUG] TestAgent.server.raft: vote granted: from=359cca84-cc4e-81f7-36d0-baeede586357 term=2 tally=1 +49:23.101 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.101 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23552 [Leader]" +49:23.101 [INFO] TestAgent.server: cluster leadership acquired +49:23.101 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23552 +49:23.101 [INFO] TestAgent.server: New leader elected: payload=Node-359cca84-cc4e-81f7-36d0-baeede586357 +49:23.105 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.110 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.110 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.110 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.110 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.110 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.110 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-359cca84-cc4e-81f7-36d0-baeede586357 +49:23.110 [INFO] TestAgent.server: member joined, marking health alive: member=Node-359cca84-cc4e-81f7-36d0-baeede586357 +49:23.111 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.194 [INFO] TestAgent: Requesting shutdown +49:23.194 [INFO] TestAgent.server: shutting down server +49:23.194 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.194 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.194 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:23.194 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +2022-12-04T05:49:23.195Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.195Z [INFO] agent.router.manager: shutting down +49:23.195 [INFO] TestAgent: consul server down +49:23.195 [INFO] TestAgent: shutdown complete +49:23.195 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23561 network=tcp +49:23.195 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23561 network=udp +49:23.195 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23562 network=tcp +49:23.695 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.695 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ConfigReload (0.67s) +=== CONT TestDNS_AltDomains_Service +2022-12-04T05:49:22.863Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.878Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.880 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f4fffdd3-cfb9-da6f-b28e-859e085bb354 Address:127.0.0.1:23454}]" +49:22.880 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23454 [Follower]" leader= +49:22.883 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f4fffdd3-cfb9-da6f-b28e-859e085bb354.dc1 127.0.0.1 +49:22.884 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f4fffdd3-cfb9-da6f-b28e-859e085bb354 127.0.0.1 +2022-12-04T05:49:22.884Z [INFO] agent.router: Initializing LAN area manager +49:22.884 [INFO] TestAgent.server: Adding LAN server: server="Node-f4fffdd3-cfb9-da6f-b28e-859e085bb354 (Addr: tcp/127.0.0.1:23454) (DC: dc1)" +49:22.884 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f4fffdd3-cfb9-da6f-b28e-859e085bb354.dc1 area=wan +49:22.884 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23491 network=udp +49:22.884 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23491 network=tcp +49:22.884 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23492 network=tcp +49:22.884 [INFO] TestAgent: started state syncer +49:22.885 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23455 network=tcp +49:22.937 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.937 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23454 [Candidate]" term=2 +49:22.937 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.937 [DEBUG] TestAgent.server.raft: vote granted: from=f4fffdd3-cfb9-da6f-b28e-859e085bb354 term=2 tally=1 +49:22.937 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.937 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23454 [Leader]" +49:22.937 [INFO] TestAgent.server: cluster leadership acquired +49:22.938 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23454 +49:22.938 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.940 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.940 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.940 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.940 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.940 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.940 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f4fffdd3-cfb9-da6f-b28e-859e085bb354 +49:22.940 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f4fffdd3-cfb9-da6f-b28e-859e085bb354 +49:22.942 [INFO] TestAgent.server: New leader elected: payload=Node-f4fffdd3-cfb9-da6f-b28e-859e085bb354 +49:22.943 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.223 [DEBUG] TestAgent.server: User event: event=test +49:23.223 [DEBUG] TestAgent.server: User event: event=foo +49:23.223 [DEBUG] TestAgent: new event: event_name=test event_id=8a77b3c3-810c-b080-dc80-6576c738fe9f +49:23.223 [DEBUG] TestAgent: new event: event_name=foo event_id=add6dbc4-5cbe-51f4-dc26-6619d2424fc6 +49:23.247 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:23.247 [INFO] TestAgent: Synced node info +49:23.248 [INFO] TestAgent: Requesting shutdown +49:23.248 [INFO] TestAgent.server: shutting down server +49:23.248 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.248 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.248 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.248 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.248 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.248 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.248 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.248 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.248 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.248 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:23.248 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:23.248Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.248Z [INFO] agent.router.manager: shutting down +49:23.248 [INFO] TestAgent: consul server down +49:23.248 [INFO] TestAgent: shutdown complete +49:23.248 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23491 network=tcp +49:23.248 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23491 network=udp +49:23.248 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23492 network=tcp +49:23.751 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.751 [INFO] TestAgent: Endpoints down +--- PASS: TestEventList_Filter (0.90s) +=== CONT TestDNS_AddressLookup +2022-12-04T05:49:22.858Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.867Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.868 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:93430e7f-264a-08e6-6b0f-22e320dff0ac Address:127.0.0.1:23482}]" +49:22.868 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23482 [Follower]" leader= +49:22.869 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-93430e7f-264a-08e6-6b0f-22e320dff0ac.dc1 127.0.0.1 +49:22.869 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-93430e7f-264a-08e6-6b0f-22e320dff0ac 127.0.0.1 +2022-12-04T05:49:22.869Z [INFO] agent.router: Initializing LAN area manager +49:22.869 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23505 network=udp +49:22.869 [INFO] TestAgent.server: Adding LAN server: server="Node-93430e7f-264a-08e6-6b0f-22e320dff0ac (Addr: tcp/127.0.0.1:23482) (DC: dc1)" +49:22.870 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-93430e7f-264a-08e6-6b0f-22e320dff0ac.dc1 area=wan +49:22.870 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23505 network=tcp +49:22.870 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23506 network=tcp +49:22.870 [INFO] TestAgent: started state syncer +49:22.870 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23483 network=tcp +49:22.924 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.924 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23482 [Candidate]" term=2 +49:22.924 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.924 [DEBUG] TestAgent.server.raft: vote granted: from=93430e7f-264a-08e6-6b0f-22e320dff0ac term=2 tally=1 +49:22.924 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.924 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23482 [Leader]" +49:22.924 [INFO] TestAgent.server: cluster leadership acquired +49:22.926 [INFO] TestAgent.server: New leader elected: payload=Node-93430e7f-264a-08e6-6b0f-22e320dff0ac +49:22.926 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23482 +49:22.927 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.929 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.929 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.929 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.929 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.929 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.929 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-93430e7f-264a-08e6-6b0f-22e320dff0ac +49:22.929 [INFO] TestAgent.server: member joined, marking health alive: member=Node-93430e7f-264a-08e6-6b0f-22e320dff0ac +49:22.929 [INFO] TestAgent.server: federation state anti-entropy synced +49:22.977 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:22.977 [INFO] TestAgent: Synced node info +49:23.238 [DEBUG] TestAgent.server: User event: event=foo +49:23.238 [DEBUG] TestAgent.server: User event: event=bar +49:23.238 [DEBUG] TestAgent.server: User event: event=foo +49:23.238 [DEBUG] TestAgent: new event: event_name=foo event_id=d274f4b0-357f-9a7b-b8b0-3e3b296ff7ad +49:23.238 [DEBUG] TestAgent: new event: event_name=bar event_id=9048084f-bef9-87ee-fd14-cd667c5815f1 +49:23.238 [DEBUG] TestAgent: new event: event_name=foo event_id=aaee8e5a-4152-65ed-2276-1f8cf413582d +49:23.238 [DEBUG] TestAgent.server: User event: event=foo +49:23.238 [DEBUG] TestAgent.server: User event: event=bar +49:23.238 [DEBUG] TestAgent: new event: event_name=foo event_id=41e6e6fa-32a0-4d0c-b051-b7d308d0be96 +49:23.238 [DEBUG] TestAgent: new event: event_name=bar event_id=2c037c18-7061-a299-c701-bf1831f0abfe +49:23.263 [INFO] TestAgent: Requesting shutdown +49:23.263 [INFO] TestAgent.server: shutting down server +49:23.263 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.263 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.263 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.263 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.263 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.263 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.263 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:23.263 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.263 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.263 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.263 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:23.264Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.264Z [INFO] agent.router.manager: shutting down +49:23.264 [INFO] TestAgent: consul server down +49:23.264 [INFO] TestAgent: shutdown complete +49:23.264 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23505 network=tcp +49:23.264 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23505 network=udp +49:23.264 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23506 network=tcp +49:23.764 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.764 [INFO] TestAgent: Endpoints down +--- PASS: TestEventList_EventBufOrder (0.92s) +=== CONT TestDNS_NonExistingLookupEmptyAorAAAA +2022-12-04T05:49:22.920Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.939Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.940 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6802a97c-d278-be3f-5406-f9f6f7ed7dcf Address:127.0.0.1:23468}]" +49:22.940 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6802a97c-d278-be3f-5406-f9f6f7ed7dcf.dc1 127.0.0.1 +49:22.941 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6802a97c-d278-be3f-5406-f9f6f7ed7dcf 127.0.0.1 +2022-12-04T05:49:22.941Z [INFO] agent.router: Initializing LAN area manager +49:22.941 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23456 network=udp +49:22.941 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23468 [Follower]" leader= +49:22.941 [INFO] TestAgent.server: Adding LAN server: server="Node-6802a97c-d278-be3f-5406-f9f6f7ed7dcf (Addr: tcp/127.0.0.1:23468) (DC: dc1)" +49:22.941 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6802a97c-d278-be3f-5406-f9f6f7ed7dcf.dc1 area=wan +49:22.941 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23456 network=tcp +49:22.942 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23457 network=tcp +49:22.942 [INFO] TestAgent: started state syncer +49:22.942 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23469 network=tcp +49:22.992 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:22.992 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23468 [Candidate]" term=2 +49:22.992 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:22.992 [DEBUG] TestAgent.server.raft: vote granted: from=6802a97c-d278-be3f-5406-f9f6f7ed7dcf term=2 tally=1 +49:22.992 [INFO] TestAgent.server.raft: election won: tally=1 +49:22.992 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23468 [Leader]" +49:22.993 [INFO] TestAgent.server: cluster leadership acquired +49:22.993 [INFO] TestAgent.server: New leader elected: payload=Node-6802a97c-d278-be3f-5406-f9f6f7ed7dcf +49:22.993 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23468 +49:22.993 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:22.995 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:22.995 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:22.995 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:22.995 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:22.995 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:22.995 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6802a97c-d278-be3f-5406-f9f6f7ed7dcf +49:22.995 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6802a97c-d278-be3f-5406-f9f6f7ed7dcf +49:22.995 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.034 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:23.034 [INFO] TestAgent: Synced node info +49:23.266 [DEBUG] TestAgent.server: User event: event=test +49:23.266 [DEBUG] TestAgent: new event: event_name=test event_id=b60bf7c0-8f3c-a32b-fd25-479c1543a07f +49:23.291 [INFO] TestAgent: Requesting shutdown +49:23.291 [INFO] TestAgent.server: shutting down server +49:23.291 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.291 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.291 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.291 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.291 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.291 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.291 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:23.291 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.291 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:23.292Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.292Z [INFO] agent.router.manager: shutting down +49:23.292 [INFO] TestAgent: consul server down +49:23.292 [INFO] TestAgent: shutdown complete +49:23.292 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23456 network=tcp +49:23.292 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23456 network=udp +49:23.292 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23457 network=tcp +49:23.792 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.792 [INFO] TestAgent: Endpoints down +--- PASS: TestEventList (0.90s) === CONT TestDNS_ServiceLookup_FilterACL === RUN TestDNS_ServiceLookup_FilterACL/ACLToken_==_root -2021-10-31T23:13:10.055Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:10.079Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:10.081 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5328b70d-5ebc-6520-dbb8-104c7b57650e Address:127.0.0.1:16749}]" -13:10.081 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16749 [Follower]" leader= -13:10.090 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5328b70d-5ebc-6520-dbb8-104c7b57650e.dc1 127.0.0.1 -13:10.090 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5328b70d-5ebc-6520-dbb8-104c7b57650e 127.0.0.1 -2021-10-31T23:13:10.090Z [INFO] agent.router: Initializing LAN area manager -13:10.091 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16744 network=udp -13:10.091 [INFO] TestAgent.server: Adding LAN server: server="Node-5328b70d-5ebc-6520-dbb8-104c7b57650e (Addr: tcp/127.0.0.1:16749) (DC: dc1)" -13:10.091 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5328b70d-5ebc-6520-dbb8-104c7b57650e.dc1 area=wan -13:10.091 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16744 network=tcp -13:10.091 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16745 network=tcp -13:10.091 [INFO] TestAgent: started state syncer -13:10.092 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16736 network=tcp -13:10.149 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:10.149 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16749 [Candidate]" term=2 -13:10.150 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:10.150 [DEBUG] TestAgent.server.raft: vote granted: from=5328b70d-5ebc-6520-dbb8-104c7b57650e term=2 tally=1 -13:10.150 [INFO] TestAgent.server.raft: election won: tally=1 -13:10.150 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16749 [Leader]" -13:10.150 [INFO] TestAgent.server: cluster leadership acquired -13:10.150 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16749 -13:10.150 [INFO] TestAgent.server: New leader elected: payload=Node-5328b70d-5ebc-6520-dbb8-104c7b57650e -13:10.151 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:10.152 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:10.152 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:10.153 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:10.153 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:10.153 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:10.153 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5328b70d-5ebc-6520-dbb8-104c7b57650e -13:10.153 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5328b70d-5ebc-6520-dbb8-104c7b57650e -13:10.154 [INFO] TestAgent.server: federation state anti-entropy synced -13:10.262 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:10.367 [INFO] TestAgent: Requesting shutdown -13:10.367 [INFO] TestAgent.server: shutting down server -13:10.367 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.367 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:10.367 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:10.367 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:10.367Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.367Z [INFO] agent.router.manager: shutting down -13:10.367 [INFO] TestAgent: consul server down -13:10.367 [INFO] TestAgent: shutdown complete -13:10.367 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16744 network=tcp -13:10.367 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16744 network=udp -13:10.367 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16745 network=tcp -13:10.870 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.870 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ReloadConfig_DuringQuery (0.90s) -=== CONT TestDNS_AltDomains_Service -2021-10-31T23:13:09.917Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:10.016Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:10.023 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9a3d4c55-56b2-81e4-c134-420b42da0699 Address:127.0.0.1:16721}]" -13:10.024 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16721 [Follower]" leader= -13:10.024 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9a3d4c55-56b2-81e4-c134-420b42da0699.dc1 127.0.0.1 -13:10.024 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9a3d4c55-56b2-81e4-c134-420b42da0699 127.0.0.1 -2021-10-31T23:13:10.024Z [INFO] agent.router: Initializing LAN area manager -13:10.025 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16716 network=udp -13:10.025 [INFO] TestAgent.server: Adding LAN server: server="Node-9a3d4c55-56b2-81e4-c134-420b42da0699 (Addr: tcp/127.0.0.1:16721) (DC: dc1)" -13:10.025 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9a3d4c55-56b2-81e4-c134-420b42da0699.dc1 area=wan -13:10.025 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16716 network=tcp -13:10.062 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16717 network=tcp -13:10.062 [INFO] TestAgent: started state syncer -13:10.062 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16722 network=tcp -13:10.068 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:10.068 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16721 [Candidate]" term=2 -13:10.068 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:10.068 [DEBUG] TestAgent.server.raft: vote granted: from=9a3d4c55-56b2-81e4-c134-420b42da0699 term=2 tally=1 -13:10.068 [INFO] TestAgent.server.raft: election won: tally=1 -13:10.068 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16721 [Leader]" -13:10.068 [INFO] TestAgent.server: cluster leadership acquired -13:10.068 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16721 -13:10.069 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:10.069 [INFO] TestAgent.server: New leader elected: payload=Node-9a3d4c55-56b2-81e4-c134-420b42da0699 -13:10.071 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:10.071 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:10.071 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:10.071 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:10.071 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:10.071 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9a3d4c55-56b2-81e4-c134-420b42da0699 -13:10.071 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9a3d4c55-56b2-81e4-c134-420b42da0699 -13:10.072 [INFO] TestAgent.server: federation state anti-entropy synced -13:10.294 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:10.294 [INFO] TestAgent: Synced node info -13:10.382 [INFO] TestAgent: Requesting shutdown -13:10.382 [INFO] TestAgent.server: shutting down server -13:10.382 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:10.382 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:10.382 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:10.382 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:10.382 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.382 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:10.382 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:10.382 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:10.382 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:10.382 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:10.382 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:10.382Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.382Z [INFO] agent.router.manager: shutting down -13:10.382 [INFO] TestAgent: consul server down -13:10.382 [INFO] TestAgent: shutdown complete -13:10.382 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16716 network=tcp -13:10.382 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16716 network=udp -13:10.382 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16717 network=tcp -13:10.883 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.883 [INFO] TestAgent: Endpoints down ---- PASS: TestEventFire (1.08s) -=== CONT TestDNS_NonExistentDC_RPC -2021-10-31T23:13:10.021Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:10.091Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:10.093 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:74b4a22a-4ccf-16c5-76cd-1f32fff2848c Address:127.0.0.1:16728}]" -13:10.093 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16728 [Follower]" leader= -13:10.110 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-74b4a22a-4ccf-16c5-76cd-1f32fff2848c.dc1 127.0.0.1 -13:10.111 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-74b4a22a-4ccf-16c5-76cd-1f32fff2848c 127.0.0.1 -2021-10-31T23:13:10.111Z [INFO] agent.router: Initializing LAN area manager -13:10.111 [INFO] TestAgent.server: Adding LAN server: server="Node-74b4a22a-4ccf-16c5-76cd-1f32fff2848c (Addr: tcp/127.0.0.1:16728) (DC: dc1)" -13:10.111 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-74b4a22a-4ccf-16c5-76cd-1f32fff2848c.dc1 area=wan -13:10.111 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16723 network=udp -13:10.111 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16723 network=tcp -13:10.112 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16724 network=tcp -13:10.112 [INFO] TestAgent: started state syncer -13:10.112 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16708 network=tcp -13:10.166 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:10.166 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16728 [Candidate]" term=2 -13:10.166 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:10.166 [DEBUG] TestAgent.server.raft: vote granted: from=74b4a22a-4ccf-16c5-76cd-1f32fff2848c term=2 tally=1 -13:10.166 [INFO] TestAgent.server.raft: election won: tally=1 -13:10.166 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16728 [Leader]" -13:10.166 [INFO] TestAgent.server: cluster leadership acquired -13:10.166 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16728 -13:10.167 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:10.168 [INFO] TestAgent.server: New leader elected: payload=Node-74b4a22a-4ccf-16c5-76cd-1f32fff2848c -13:10.169 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:10.169 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:10.169 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:10.169 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:10.169 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:10.169 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-74b4a22a-4ccf-16c5-76cd-1f32fff2848c -13:10.169 [INFO] TestAgent.server: member joined, marking health alive: member=Node-74b4a22a-4ccf-16c5-76cd-1f32fff2848c -13:10.170 [INFO] TestAgent.server: federation state anti-entropy synced -13:10.383 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=89.062µs client=127.0.0.1:46867 client_network=udp -13:10.383 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=38.025µs client=127.0.0.1:46867 client_network=udp -13:10.383 [INFO] TestAgent: Requesting shutdown -13:10.383 [INFO] TestAgent.server: shutting down server -13:10.383 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:10.383 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:10.383 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:10.383 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:10.383 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.383 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:10.383 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:10.383 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:10.383 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:10.383 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:10.383 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:10.383 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:10.383Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.383Z [INFO] agent.router.manager: shutting down -13:10.384 [INFO] TestAgent: consul server down -13:10.384 [INFO] TestAgent: shutdown complete -13:10.384 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16723 network=tcp -13:10.384 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16723 network=udp -13:10.384 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16724 network=tcp -13:10.884 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.884 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_Compression_ReverseLookup (0.99s) -=== CONT TestDNS_ServiceLookup_SRV_RFC_TCP_Default -2021-10-31T23:13:10.212Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:10.262Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:10.264 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:718c751e-3731-8c5b-ba09-dcc9ca0010f6 Address:127.0.0.1:16756}]" -13:10.265 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-718c751e-3731-8c5b-ba09-dcc9ca0010f6.dc1 127.0.0.1 -13:10.265 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-718c751e-3731-8c5b-ba09-dcc9ca0010f6 127.0.0.1 -2021-10-31T23:13:10.265Z [INFO] agent.router: Initializing LAN area manager -13:10.265 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16751 network=udp -13:10.265 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16756 [Follower]" leader= -13:10.266 [INFO] TestAgent.server: Adding LAN server: server="Node-718c751e-3731-8c5b-ba09-dcc9ca0010f6 (Addr: tcp/127.0.0.1:16756) (DC: dc1)" -13:10.266 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-718c751e-3731-8c5b-ba09-dcc9ca0010f6.dc1 area=wan -13:10.266 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16751 network=tcp -13:10.266 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16752 network=tcp -13:10.266 [INFO] TestAgent: started state syncer -13:10.266 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16778 network=tcp -13:10.326 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:10.326 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16756 [Candidate]" term=2 -13:10.326 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:10.326 [DEBUG] TestAgent.server.raft: vote granted: from=718c751e-3731-8c5b-ba09-dcc9ca0010f6 term=2 tally=1 -13:10.326 [INFO] TestAgent.server.raft: election won: tally=1 -13:10.326 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16756 [Leader]" -13:10.326 [INFO] TestAgent.server: cluster leadership acquired -13:10.327 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16756 -13:10.328 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:10.330 [INFO] TestAgent.server: New leader elected: payload=Node-718c751e-3731-8c5b-ba09-dcc9ca0010f6 -13:10.334 [INFO] TestAgent: Synced node info -13:10.351 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:10.351 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:10.351 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:10.351 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:10.351 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:10.351 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-718c751e-3731-8c5b-ba09-dcc9ca0010f6 -13:10.351 [INFO] TestAgent.server: member joined, marking health alive: member=Node-718c751e-3731-8c5b-ba09-dcc9ca0010f6 -13:10.352 [INFO] TestAgent.server: federation state anti-entropy synced -13:10.394 [INFO] TestAgent: Requesting shutdown -13:10.394 [INFO] TestAgent.server: shutting down server -13:10.394 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:10.394 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:10.394 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:10.394 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:10.394 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.394 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:10.394 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:10.394 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:10.394 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:10.394 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:10.394 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:10.394Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.394Z [INFO] agent.router.manager: shutting down -13:10.394 [INFO] TestAgent: consul server down -13:10.394 [INFO] TestAgent: shutdown complete -13:10.394 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16751 network=tcp -13:10.394 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16751 network=udp -13:10.394 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16752 network=tcp -13:10.895 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.895 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_InmemKeyrings (3.45s) - --- PASS: TestAgent_InmemKeyrings/no_keys (1.18s) - --- PASS: TestAgent_InmemKeyrings/server_with_keys (0.88s) - --- PASS: TestAgent_InmemKeyrings/client_with_keys (0.63s) - --- PASS: TestAgent_InmemKeyrings/ignore_files (0.77s) -=== CONT TestDNS_NonExistentDC_Server -2021-10-31T23:13:10.107Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:10.163Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:10.165 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:df4da357-b773-3b41-e561-15efd428fbde Address:127.0.0.1:16742}]" -13:10.165 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16742 [Follower]" leader= -13:10.178 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-df4da357-b773-3b41-e561-15efd428fbde.dc1 127.0.0.1 -13:10.178 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-df4da357-b773-3b41-e561-15efd428fbde 127.0.0.1 -2021-10-31T23:13:10.178Z [INFO] agent.router: Initializing LAN area manager -13:10.179 [INFO] TestAgent.server: Adding LAN server: server="Node-df4da357-b773-3b41-e561-15efd428fbde (Addr: tcp/127.0.0.1:16742) (DC: dc1)" -13:10.179 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16737 network=udp -13:10.179 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-df4da357-b773-3b41-e561-15efd428fbde.dc1 area=wan -13:10.179 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16737 network=tcp -13:10.179 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16738 network=tcp -13:10.179 [INFO] TestAgent: started state syncer -13:10.179 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16757 network=tcp -13:10.217 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:10.217 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16742 [Candidate]" term=2 -13:10.218 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:10.218 [DEBUG] TestAgent.server.raft: vote granted: from=df4da357-b773-3b41-e561-15efd428fbde term=2 tally=1 -13:10.218 [INFO] TestAgent.server.raft: election won: tally=1 -13:10.218 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16742 [Leader]" -13:10.218 [INFO] TestAgent.server: cluster leadership acquired -13:10.218 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16742 -13:10.219 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:10.220 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:10.220 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:10.220 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:10.220 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:10.220 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:10.220 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-df4da357-b773-3b41-e561-15efd428fbde -13:10.220 [INFO] TestAgent.server: member joined, marking health alive: member=Node-df4da357-b773-3b41-e561-15efd428fbde -13:10.221 [INFO] TestAgent.server: New leader elected: payload=Node-df4da357-b773-3b41-e561-15efd428fbde -13:10.221 [INFO] TestAgent.server: federation state anti-entropy synced -13:10.270 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:10.270 [INFO] TestAgent: Synced node info -13:10.403 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=96.269µs client=127.0.0.1:47542 client_network=udp -13:10.403 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=41.551µs client=127.0.0.1:47542 client_network=udp -13:10.403 [DEBUG] TestAgent.dns: request served from client: name=ad215bfa-aa89-bb1c-bb21-8b4484f75893.query.consul. type=SRV class=IN latency=68.968µs client=127.0.0.1:38510 client_network=udp -13:10.403 [DEBUG] TestAgent.dns: request served from client: name=ad215bfa-aa89-bb1c-bb21-8b4484f75893.query.consul. type=SRV class=IN latency=46.193µs client=127.0.0.1:38510 client_network=udp -13:10.403 [INFO] TestAgent: Requesting shutdown -13:10.403 [INFO] TestAgent.server: shutting down server -13:10.403 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:10.403 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:10.403 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:10.403 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:10.403 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.403 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:10.403 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:10.403 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:10.403 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:10.403 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:10.403 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:10.403Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.404Z [INFO] agent.router.manager: shutting down -13:10.404 [INFO] TestAgent: consul server down -13:10.404 [INFO] TestAgent: shutdown complete -13:10.404 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16737 network=tcp -13:10.404 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16737 network=udp -13:10.404 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16738 network=tcp -13:10.905 [INFO] TestAgent: Waiting for endpoints to shut down -13:10.945 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_Compression_Query (0.89s) +2022-12-04T05:49:22.953Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:22.966Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:22.968 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7f10f109-56fe-91fe-0f2a-3764131e16c3 Address:127.0.0.1:23510}]" +49:22.968 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23510 [Follower]" leader= +49:22.968 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7f10f109-56fe-91fe-0f2a-3764131e16c3.dc1 127.0.0.1 +49:22.969 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7f10f109-56fe-91fe-0f2a-3764131e16c3 127.0.0.1 +2022-12-04T05:49:22.969Z [INFO] agent.router: Initializing LAN area manager +49:22.969 [INFO] TestAgent.server: Adding LAN server: server="Node-7f10f109-56fe-91fe-0f2a-3764131e16c3 (Addr: tcp/127.0.0.1:23510) (DC: dc1)" +49:22.969 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7f10f109-56fe-91fe-0f2a-3764131e16c3.dc1 area=wan +49:22.969 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23470 network=udp +49:22.969 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23470 network=tcp +49:22.970 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23471 network=tcp +49:22.970 [INFO] TestAgent: started state syncer +49:22.970 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23511 network=tcp +49:23.018 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.018 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23510 [Candidate]" term=2 +49:23.019 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.019 [DEBUG] TestAgent.server.raft: vote granted: from=7f10f109-56fe-91fe-0f2a-3764131e16c3 term=2 tally=1 +49:23.019 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.019 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23510 [Leader]" +49:23.019 [INFO] TestAgent.server: cluster leadership acquired +49:23.019 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23510 +49:23.019 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.020 [INFO] TestAgent.server: New leader elected: payload=Node-7f10f109-56fe-91fe-0f2a-3764131e16c3 +49:23.020 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.020 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.020 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.020 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.020 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.020 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7f10f109-56fe-91fe-0f2a-3764131e16c3 +49:23.020 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7f10f109-56fe-91fe-0f2a-3764131e16c3 +49:23.021 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.157 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:23.158 [INFO] TestAgent: Synced node info +49:23.158 [DEBUG] TestAgent: Node info in sync +49:23.294 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=176.827µs client=127.0.0.1:34012 client_network=udp +49:23.294 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=131.496µs client=127.0.0.1:34012 client_network=udp +49:23.295 [DEBUG] TestAgent.dns: request served from client: name=21a0e773-18d3-e5ca-f3dd-4671ad5ab8c2.query.consul. type=SRV class=IN latency=104.802µs client=127.0.0.1:48228 client_network=udp +49:23.295 [DEBUG] TestAgent.dns: request served from client: name=21a0e773-18d3-e5ca-f3dd-4671ad5ab8c2.query.consul. type=SRV class=IN latency=112.781µs client=127.0.0.1:48228 client_network=udp +49:23.295 [INFO] TestAgent: Requesting shutdown +49:23.295 [INFO] TestAgent.server: shutting down server +49:23.295 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.295 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.295 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.295 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.295 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.295 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.295 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.295 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.295 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:23.295 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:23.295 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +2022-12-04T05:49:23.295Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.295Z [INFO] agent.router.manager: shutting down +49:23.295 [INFO] TestAgent: consul server down +49:23.295 [INFO] TestAgent: shutdown complete +49:23.295 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23470 network=tcp +49:23.295 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23470 network=udp +49:23.295 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23471 network=tcp +49:23.796 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.796 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_Compression_Query (0.86s) +=== CONT TestDNS_NonExistingLookup +[WARN] freeport: 10 out of 73 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:23.033Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.045Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.047 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e4769433-0c6e-e846-22bf-453555b68415 Address:127.0.0.1:23531}]" +49:23.047 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23531 [Follower]" leader= +49:23.047 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e4769433-0c6e-e846-22bf-453555b68415.dc1 127.0.0.1 +49:23.048 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e4769433-0c6e-e846-22bf-453555b68415 127.0.0.1 +2022-12-04T05:49:23.048Z [INFO] agent.router: Initializing LAN area manager +49:23.048 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23526 network=udp +49:23.048 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e4769433-0c6e-e846-22bf-453555b68415.dc1 area=wan +49:23.048 [INFO] TestAgent.server: Adding LAN server: server="Node-e4769433-0c6e-e846-22bf-453555b68415 (Addr: tcp/127.0.0.1:23531) (DC: dc1)" +49:23.048 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23526 network=tcp +49:23.049 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23527 network=tcp +49:23.049 [INFO] TestAgent: started state syncer +49:23.049 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23532 network=tcp +49:23.115 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.115 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23531 [Candidate]" term=2 +49:23.116 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.116 [DEBUG] TestAgent.server.raft: vote granted: from=e4769433-0c6e-e846-22bf-453555b68415 term=2 tally=1 +49:23.116 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.116 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23531 [Leader]" +49:23.116 [INFO] TestAgent.server: cluster leadership acquired +49:23.116 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23531 +49:23.116 [INFO] TestAgent.server: New leader elected: payload=Node-e4769433-0c6e-e846-22bf-453555b68415 +49:23.118 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.121 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.121 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.121 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.121 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.121 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.121 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e4769433-0c6e-e846-22bf-453555b68415 +49:23.121 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e4769433-0c6e-e846-22bf-453555b68415 +49:23.121 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.187 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:23.188 [INFO] TestAgent: Synced node info +49:23.188 [DEBUG] TestAgent: Node info in sync +49:23.309 [INFO] TestAgent: Requesting shutdown +49:23.309 [INFO] TestAgent.server: shutting down server +49:23.309 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.309 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.309 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.309 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.310 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.310 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.310 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.310 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.310 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.310 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:23.310 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:23.310Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.310Z [INFO] agent.router.manager: shutting down +49:23.310 [INFO] TestAgent: consul server down +49:23.310 [INFO] TestAgent: shutdown complete +49:23.310 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23526 network=tcp +49:23.310 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23526 network=udp +49:23.310 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23527 network=tcp +49:23.824 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.824 [INFO] TestAgent: Endpoints down +--- PASS: TestEventFire (0.80s) +=== CONT TestDNS_ServiceLookup_TTL +2022-12-04T05:49:23.042Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.054Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.056 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b329937c-e891-c185-4fb2-5eccb4f90e50 Address:127.0.0.1:23559}]" +49:23.056 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23559 [Follower]" leader= +49:23.057 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b329937c-e891-c185-4fb2-5eccb4f90e50.dc1 127.0.0.1 +49:23.057 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b329937c-e891-c185-4fb2-5eccb4f90e50 127.0.0.1 +2022-12-04T05:49:23.057Z [INFO] agent.router: Initializing LAN area manager +49:23.057 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b329937c-e891-c185-4fb2-5eccb4f90e50.dc1 area=wan +49:23.057 [INFO] TestAgent.server: Adding LAN server: server="Node-b329937c-e891-c185-4fb2-5eccb4f90e50 (Addr: tcp/127.0.0.1:23559) (DC: dc1)" +49:23.057 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23533 network=udp +49:23.058 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23533 network=tcp +49:23.058 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23534 network=tcp +49:23.058 [INFO] TestAgent: started state syncer +49:23.058 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23560 network=tcp +49:23.101 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.101 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23559 [Candidate]" term=2 +49:23.101 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.101 [DEBUG] TestAgent.server.raft: vote granted: from=b329937c-e891-c185-4fb2-5eccb4f90e50 term=2 tally=1 +49:23.101 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.101 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23559 [Leader]" +49:23.101 [INFO] TestAgent.server: cluster leadership acquired +49:23.101 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23559 +49:23.101 [INFO] TestAgent.server: New leader elected: payload=Node-b329937c-e891-c185-4fb2-5eccb4f90e50 +49:23.106 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.109 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.109 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.109 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.109 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.109 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.109 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b329937c-e891-c185-4fb2-5eccb4f90e50 +49:23.109 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b329937c-e891-c185-4fb2-5eccb4f90e50 +49:23.111 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.241 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:23.342 [INFO] TestAgent: Requesting shutdown +49:23.342 [INFO] TestAgent.server: shutting down server +49:23.342 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.342 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.342 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:23.342 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:23.342Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.342Z [INFO] agent.router.manager: shutting down +49:23.342 [INFO] TestAgent: consul server down +49:23.342 [INFO] TestAgent: shutdown complete +49:23.342 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23533 network=tcp +49:23.342 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23533 network=udp +49:23.342 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23534 network=tcp +49:23.844 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.844 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ReloadConfig_DuringQuery (0.82s) === CONT TestDNS_NodeLookup_TTL -[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:10.558Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:10.603Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:10.605 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d8b7bde4-a429-b980-fc4f-d99d1a0a6d09 Address:127.0.0.1:16770}]" -13:10.605 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d8b7bde4-a429-b980-fc4f-d99d1a0a6d09.dc1 127.0.0.1 -13:10.605 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16770 [Follower]" leader= -13:10.613 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d8b7bde4-a429-b980-fc4f-d99d1a0a6d09 127.0.0.1 -2021-10-31T23:13:10.613Z [INFO] agent.router: Initializing LAN area manager -13:10.614 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16765 network=udp -13:10.614 [INFO] TestAgent.server: Adding LAN server: server="Node-d8b7bde4-a429-b980-fc4f-d99d1a0a6d09 (Addr: tcp/127.0.0.1:16770) (DC: dc1)" -13:10.614 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d8b7bde4-a429-b980-fc4f-d99d1a0a6d09.dc1 area=wan -13:10.614 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16765 network=tcp -13:10.615 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16766 network=tcp -13:10.615 [INFO] TestAgent: started state syncer -13:10.615 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16785 network=tcp -13:10.657 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:10.657 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16770 [Candidate]" term=2 -13:10.658 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:10.658 [DEBUG] TestAgent.server.raft: vote granted: from=d8b7bde4-a429-b980-fc4f-d99d1a0a6d09 term=2 tally=1 -13:10.658 [INFO] TestAgent.server.raft: election won: tally=1 -13:10.658 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16770 [Leader]" -13:10.658 [INFO] TestAgent.server: cluster leadership acquired -13:10.658 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16770 -13:10.659 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:10.660 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:10.660 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:10.660 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:10.660 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:10.660 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:10.660 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d8b7bde4-a429-b980-fc4f-d99d1a0a6d09 -13:10.660 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d8b7bde4-a429-b980-fc4f-d99d1a0a6d09 -13:10.661 [INFO] TestAgent.server: New leader elected: payload=Node-d8b7bde4-a429-b980-fc4f-d99d1a0a6d09 -13:10.661 [INFO] TestAgent.server: federation state anti-entropy synced -13:10.690 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:10.690 [DEBUG] TestAgent.dns: request served from client: name=foo.query.consul. type=SRV class=IN latency=76.644µs client=127.0.0.1:42830 client_network=udp -13:10.690 [INFO] TestAgent: Requesting shutdown -13:10.690 [INFO] TestAgent.server: shutting down server -13:10.690 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:10.690 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:10.690 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:10.690 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:10.690 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.690 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:10.690 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:10.690 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:10.691 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:10.691 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:10.691 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:10.691 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:10.691Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.691Z [INFO] agent.router.manager: shutting down -13:10.691 [INFO] TestAgent: consul server down -13:10.691 [INFO] TestAgent: shutdown complete -13:10.691 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16765 network=tcp -13:10.691 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16765 network=udp -13:10.691 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16766 network=tcp -13:11.193 [INFO] TestAgent: Waiting for endpoints to shut down -13:11.193 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_PreparedQuery_AgentSource (0.68s) -=== CONT TestDNS_AddressLookupIPV6 -2021-10-31T23:13:10.496Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:10.551Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:10.552 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6b94ba57-0ccc-b054-4fdc-8a574668f078 Address:127.0.0.1:16784}]" -13:10.553 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6b94ba57-0ccc-b054-4fdc-8a574668f078.dc1 127.0.0.1 -13:10.553 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16784 [Follower]" leader= -13:10.562 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6b94ba57-0ccc-b054-4fdc-8a574668f078 127.0.0.1 -2021-10-31T23:13:10.562Z [INFO] agent.router: Initializing LAN area manager -13:10.562 [DEBUG] TestAgent.dns: recursor enabled -13:10.562 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16779 network=udp -13:10.562 [INFO] TestAgent.server: Adding LAN server: server="Node-6b94ba57-0ccc-b054-4fdc-8a574668f078 (Addr: tcp/127.0.0.1:16784) (DC: dc1)" -13:10.562 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6b94ba57-0ccc-b054-4fdc-8a574668f078.dc1 area=wan -13:10.563 [DEBUG] TestAgent.dns: recursor enabled -13:10.563 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16779 network=tcp -13:10.563 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16780 network=tcp -13:10.563 [INFO] TestAgent: started state syncer -13:10.563 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16764 network=tcp -13:10.602 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:10.602 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16784 [Candidate]" term=2 -13:10.603 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:10.603 [DEBUG] TestAgent.server.raft: vote granted: from=6b94ba57-0ccc-b054-4fdc-8a574668f078 term=2 tally=1 -13:10.603 [INFO] TestAgent.server.raft: election won: tally=1 -13:10.603 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16784 [Leader]" -13:10.603 [INFO] TestAgent.server: cluster leadership acquired -13:10.603 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16784 -13:10.604 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:10.605 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:10.605 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:10.605 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:10.605 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:10.605 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:10.605 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6b94ba57-0ccc-b054-4fdc-8a574668f078 -13:10.605 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6b94ba57-0ccc-b054-4fdc-8a574668f078 -13:10.606 [INFO] TestAgent.server: New leader elected: payload=Node-6b94ba57-0ccc-b054-4fdc-8a574668f078 -13:10.610 [INFO] TestAgent.server: federation state anti-entropy synced -13:10.746 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{apple.com. 255 1}" rtt=92.84µs recursor=127.0.0.1:52707 -13:10.746 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=232.723µs client=127.0.0.1:42223 client_network=udp -13:10.746 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{apple.com. 255 1}" rtt=76.086µs recursor=127.0.0.1:52707 -13:10.746 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=151.159µs client=127.0.0.1:42223 client_network=udp -13:10.746 [INFO] TestAgent: Requesting shutdown -13:10.746 [INFO] TestAgent.server: shutting down server -13:10.746 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:10.746 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:10.746 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:10.746 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:10.746 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.746 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:10.746 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:10.746 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:10.746 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:10.746 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:10.746 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:10.747 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:10.747Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.747Z [INFO] agent.router.manager: shutting down -13:10.747 [INFO] TestAgent: consul server down -13:10.747 [INFO] TestAgent: shutdown complete -13:10.747 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16779 network=tcp -13:10.747 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16779 network=udp -13:10.747 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16780 network=tcp -13:11.248 [INFO] TestAgent: Waiting for endpoints to shut down -13:11.248 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_Compression_Recurse (0.80s) -=== CONT TestDNS_ServiceLookup_SRV_RFC -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:10.742Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:10.778Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:10.780 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3de72a64-7756-c877-c3e4-26e50a8db4bf Address:127.0.0.1:16805}]" -13:10.780 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:10.781 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:10.781Z [INFO] agent.router: Initializing LAN area manager -13:10.781 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16800 network=udp -13:10.781 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16805 [Follower]" leader= -13:10.781 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16805) (DC: dc1)" -13:10.781 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:10.782 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16800 network=tcp -13:10.782 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16801 network=tcp -13:10.782 [INFO] TestAgent: started state syncer -13:10.782 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16806 network=tcp -13:10.824 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:10.824 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16805 [Candidate]" term=2 -13:10.824 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:10.824 [DEBUG] TestAgent.server.raft: vote granted: from=3de72a64-7756-c877-c3e4-26e50a8db4bf term=2 tally=1 -13:10.824 [INFO] TestAgent.server.raft: election won: tally=1 -13:10.824 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16805 [Leader]" -13:10.825 [INFO] TestAgent.server: cluster leadership acquired -13:10.825 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16805 -13:10.825 [INFO] TestAgent.server: New leader elected: payload=test-node -13:10.826 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:10.827 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:10.827 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:10.827 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:10.827 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:10.827 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:10.827 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:10.827 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:10.827 [INFO] TestAgent.server: federation state anti-entropy synced -13:10.943 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.consul. type=A class=IN latency=210.146µs client=127.0.0.1:43183 client_network=udp -13:10.943 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.test.consul. type=A class=IN latency=52.619µs client=127.0.0.1:41370 client_network=udp -13:10.943 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.dc1.consul. type=A class=IN latency=48.202µs client=127.0.0.1:37139 client_network=udp -13:10.944 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.dc1.test.consul. type=A class=IN latency=49.992µs client=127.0.0.1:52522 client_network=udp -13:10.944 [INFO] TestAgent: Requesting shutdown -13:10.944 [INFO] TestAgent.server: shutting down server -13:10.944 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:10.944 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:10.944 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:10.944 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:10.944 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.944 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:10.944 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:10.944 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:10.944 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:10.944 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:10.944 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:10.944 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:10.944Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.944Z [INFO] agent.router.manager: shutting down -13:10.944 [INFO] TestAgent: consul server down -13:10.944 [INFO] TestAgent: shutdown complete -13:10.944 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16800 network=tcp -13:10.944 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16800 network=udp -13:10.944 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16801 network=tcp -13:11.445 [INFO] TestAgent: Waiting for endpoints to shut down -13:11.445 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_AltDomains_Overlap (0.74s) -=== CONT TestDNS_AddressLookup -2021-10-31T23:13:10.716Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:10.755Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:10.757 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:114879d0-94a1-d8c0-5fd9-6912aa2ad72c Address:127.0.0.1:16777}]" -13:10.757 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16777 [Follower]" leader= -13:10.766 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-114879d0-94a1-d8c0-5fd9-6912aa2ad72c.dc1 127.0.0.1 -13:10.766 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-114879d0-94a1-d8c0-5fd9-6912aa2ad72c 127.0.0.1 -2021-10-31T23:13:10.766Z [INFO] agent.router: Initializing LAN area manager -13:10.767 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16772 network=udp -13:10.767 [INFO] TestAgent.server: Adding LAN server: server="Node-114879d0-94a1-d8c0-5fd9-6912aa2ad72c (Addr: tcp/127.0.0.1:16777) (DC: dc1)" -13:10.767 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-114879d0-94a1-d8c0-5fd9-6912aa2ad72c.dc1 area=wan -13:10.767 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16772 network=tcp -13:10.767 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16773 network=tcp -13:10.768 [INFO] TestAgent: started state syncer -13:10.768 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16792 network=tcp -13:10.797 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:10.798 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16777 [Candidate]" term=2 -13:10.798 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:10.798 [DEBUG] TestAgent.server.raft: vote granted: from=114879d0-94a1-d8c0-5fd9-6912aa2ad72c term=2 tally=1 -13:10.798 [INFO] TestAgent.server.raft: election won: tally=1 -13:10.798 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16777 [Leader]" -13:10.798 [INFO] TestAgent.server: cluster leadership acquired -13:10.798 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16777 -13:10.798 [INFO] TestAgent.server: New leader elected: payload=Node-114879d0-94a1-d8c0-5fd9-6912aa2ad72c -13:10.799 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:10.800 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:10.800 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:10.800 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:10.800 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:10.801 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:10.801 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-114879d0-94a1-d8c0-5fd9-6912aa2ad72c -13:10.801 [INFO] TestAgent.server: member joined, marking health alive: member=Node-114879d0-94a1-d8c0-5fd9-6912aa2ad72c -13:10.806 [INFO] TestAgent.server: federation state anti-entropy synced -13:10.814 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:10.814 [INFO] TestAgent: Synced node info -13:10.950 [DEBUG] TestAgent.dns: request served from client: name=webv4.service.consul. type=AAAA class=IN latency=244.032µs client=127.0.0.1:37171 client_network=udp -13:10.954 [DEBUG] TestAgent.dns: request served from client: name=webv4.query.consul. type=AAAA class=IN latency=174.747µs client=127.0.0.1:36443 client_network=udp -13:10.954 [DEBUG] TestAgent.dns: request served from client: name=webv6.service.consul. type=A class=IN latency=86.793µs client=127.0.0.1:40401 client_network=udp -13:10.954 [DEBUG] TestAgent.dns: request served from client: name=webv6.query.consul. type=A class=IN latency=83.555µs client=127.0.0.1:42342 client_network=udp -13:10.954 [INFO] TestAgent: Requesting shutdown -13:10.955 [INFO] TestAgent.server: shutting down server -13:10.955 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:10.955 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:10.955 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:10.955 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:10.955 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:10.955 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:10.955 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:10.955 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:10.955 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:10.955 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:10.955 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:10.955Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:10.955Z [INFO] agent.router.manager: shutting down -13:10.955 [INFO] TestAgent: consul server down -13:10.955 [INFO] TestAgent: shutdown complete -13:10.955 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16772 network=tcp -13:10.955 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16772 network=udp -13:10.955 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16773 network=tcp -13:11.456 [INFO] TestAgent: Waiting for endpoints to shut down -13:11.456 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NonExistingLookupEmptyAorAAAA (0.78s) +2022-12-04T05:49:23.138Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.146Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.148 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:abfe3e32-3147-3527-3591-3c638e4e0152 Address:127.0.0.1:23545}]" +49:23.148 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23545 [Follower]" leader= +49:23.148 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-abfe3e32-3147-3527-3591-3c638e4e0152.dc1 127.0.0.1 +49:23.149 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-abfe3e32-3147-3527-3591-3c638e4e0152 127.0.0.1 +2022-12-04T05:49:23.149Z [INFO] agent.router: Initializing LAN area manager +49:23.149 [INFO] TestAgent.server: Adding LAN server: server="Node-abfe3e32-3147-3527-3591-3c638e4e0152 (Addr: tcp/127.0.0.1:23545) (DC: dc1)" +49:23.149 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-abfe3e32-3147-3527-3591-3c638e4e0152.dc1 area=wan +49:23.149 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23540 network=udp +49:23.149 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23540 network=tcp +49:23.150 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23541 network=tcp +49:23.150 [INFO] TestAgent: started state syncer +49:23.150 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23546 network=tcp +49:23.217 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.217 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23545 [Candidate]" term=2 +49:23.217 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.217 [DEBUG] TestAgent.server.raft: vote granted: from=abfe3e32-3147-3527-3591-3c638e4e0152 term=2 tally=1 +49:23.217 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.217 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23545 [Leader]" +49:23.218 [INFO] TestAgent.server: cluster leadership acquired +49:23.218 [INFO] TestAgent.server: New leader elected: payload=Node-abfe3e32-3147-3527-3591-3c638e4e0152 +49:23.218 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23545 +49:23.218 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.220 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.220 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.220 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.220 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.220 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.220 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-abfe3e32-3147-3527-3591-3c638e4e0152 +49:23.220 [INFO] TestAgent.server: member joined, marking health alive: member=Node-abfe3e32-3147-3527-3591-3c638e4e0152 +49:23.220 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.375 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:23.376 [INFO] TestAgent: Synced node info +49:23.376 [DEBUG] TestAgent: Node info in sync +49:23.379 [WARN] TestAgent: check has interval below minimum: check=http minimum_interval=1s +49:23.379 [WARN] TestAgent: check has interval below minimum: check=grpc minimum_interval=1s +49:23.379 [INFO] TestAgent: Requesting shutdown +49:23.379 [INFO] TestAgent.server: shutting down server +49:23.379 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.379 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.379 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.379 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.379 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.379 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.379 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.379 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.379 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.379 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:23.379 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:23.380Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.380Z [INFO] agent.router.manager: shutting down +49:23.380 [INFO] TestAgent: consul server down +49:23.380 [INFO] TestAgent: shutdown complete +49:23.380 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23540 network=tcp +49:23.380 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23540 network=udp +49:23.380 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23541 network=tcp +49:23.880 [INFO] TestAgent: Waiting for endpoints to shut down +49:23.880 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RerouteExistingHTTPChecks (0.75s) +=== CONT TestDNS_ServiceLookup_SRV_RFC_TCP_Default +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:23.292Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.302Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.303 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:24c931c5-2ed7-913b-2ef7-2dc8d0fe6283 Address:127.0.0.1:23580}]" +49:23.303 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23580 [Follower]" leader= +49:23.304 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-24c931c5-2ed7-913b-2ef7-2dc8d0fe6283.dc1 127.0.0.1 +49:23.304 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-24c931c5-2ed7-913b-2ef7-2dc8d0fe6283 127.0.0.1 +2022-12-04T05:49:23.304Z [INFO] agent.router: Initializing LAN area manager +49:23.304 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-24c931c5-2ed7-913b-2ef7-2dc8d0fe6283.dc1 area=wan +49:23.304 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23547 network=udp +49:23.304 [INFO] TestAgent.server: Adding LAN server: server="Node-24c931c5-2ed7-913b-2ef7-2dc8d0fe6283 (Addr: tcp/127.0.0.1:23580) (DC: dc1)" +49:23.304 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23547 network=tcp +49:23.305 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23548 network=tcp +49:23.305 [INFO] TestAgent: started state syncer +49:23.305 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23581 network=tcp +49:23.372 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.372 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23580 [Candidate]" term=2 +49:23.372 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.372 [DEBUG] TestAgent.server.raft: vote granted: from=24c931c5-2ed7-913b-2ef7-2dc8d0fe6283 term=2 tally=1 +49:23.372 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.372 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23580 [Leader]" +49:23.372 [INFO] TestAgent.server: cluster leadership acquired +49:23.372 [INFO] TestAgent.server: New leader elected: payload=Node-24c931c5-2ed7-913b-2ef7-2dc8d0fe6283 +49:23.372 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23580 +49:23.373 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.374 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.374 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.374 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.374 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.374 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.374 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-24c931c5-2ed7-913b-2ef7-2dc8d0fe6283 +49:23.374 [INFO] TestAgent.server: member joined, marking health alive: member=Node-24c931c5-2ed7-913b-2ef7-2dc8d0fe6283 +49:23.375 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.529 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:23.529 [INFO] TestAgent: Synced node info +49:23.529 [DEBUG] TestAgent: Node info in sync +49:23.590 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=189.985µs client=127.0.0.1:42754 client_network=udp +49:23.590 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=60.122µs client=127.0.0.1:42754 client_network=udp +49:23.590 [INFO] TestAgent: Requesting shutdown +49:23.590 [INFO] TestAgent.server: shutting down server +49:23.590 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.590 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.590 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.590 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.590 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.590 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.590 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.590 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.590 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.590 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:23.590 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:23.591Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.591Z [INFO] agent.router.manager: shutting down +49:23.591 [INFO] TestAgent: consul server down +49:23.591 [INFO] TestAgent: shutdown complete +49:23.591 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23547 network=tcp +49:23.591 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23547 network=udp +49:23.591 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23548 network=tcp +49:24.091 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.091 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_Compression_ReverseLookup (0.81s) === CONT TestDNS_ServiceLookup_ServiceAddress_CNAME -2021-10-31T23:13:10.871Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:10.943Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:10.945 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:87b9ecc3-883a-2c26-b3ca-74ad841f8a5a Address:127.0.0.1:16819}]" -13:10.945 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16819 [Follower]" leader= -13:10.947 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:10.948 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:10.948Z [INFO] agent.router: Initializing LAN area manager -13:10.948 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16814 network=udp -13:10.948 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16819) (DC: dc1)" -13:10.948 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:10.948 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16814 network=tcp -13:10.949 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16815 network=tcp -13:10.949 [INFO] TestAgent: started state syncer -13:10.949 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16820 network=tcp -13:10.997 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:10.997 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16819 [Candidate]" term=2 -13:10.998 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:10.998 [DEBUG] TestAgent.server.raft: vote granted: from=87b9ecc3-883a-2c26-b3ca-74ad841f8a5a term=2 tally=1 -13:10.998 [INFO] TestAgent.server.raft: election won: tally=1 -13:10.998 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16819 [Leader]" -13:10.998 [INFO] TestAgent.server: cluster leadership acquired -13:10.998 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16819 -13:10.999 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.000 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.000 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.000 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.000 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.000 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.000 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:11.000 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:11.001 [INFO] TestAgent.server: New leader elected: payload=test-node -13:11.001 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.002 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.consul. type=SOA class=IN latency=113.96µs client=127.0.0.1:35034 client_network=udp -13:11.006 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.test-domain. type=SOA class=IN latency=269.739µs client=127.0.0.1:45911 client_network=udp -13:11.009 [INFO] TestAgent: Requesting shutdown -13:11.010 [INFO] TestAgent.server: shutting down server -13:11.010 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:11.010 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:11.010 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:11.010 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:11.010 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:11.010 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:11.010 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:11.010 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:11.010 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:11.010 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:11.010 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:11.010 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -2021-10-31T23:13:11.010Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:11.010Z [INFO] agent.router.manager: shutting down -13:11.010 [INFO] TestAgent: consul server down -13:11.010 [INFO] TestAgent: shutdown complete -13:11.010 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16814 network=tcp -13:11.010 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16814 network=udp -13:11.010 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16815 network=tcp -13:11.513 [INFO] TestAgent: Waiting for endpoints to shut down -13:11.513 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_AltDomains_SOA (0.72s) +=== RUN TestDNS_ServiceLookup_TTL/db.service.consul. +=== RUN TestDNS_ServiceLookup_TTL/dblb.service.consul. +=== RUN TestDNS_ServiceLookup_TTL/dk.service.consul. +=== RUN TestDNS_ServiceLookup_TTL/api.service.consul. +2022-12-04T05:49:23.384Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.393Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.395 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c324c30d-944e-896c-c91d-b98c2c7efc9a Address:127.0.0.1:23587}]" +49:23.395 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23587 [Follower]" leader= +49:23.395 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c324c30d-944e-896c-c91d-b98c2c7efc9a.dc1 127.0.0.1 +49:23.396 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c324c30d-944e-896c-c91d-b98c2c7efc9a 127.0.0.1 +2022-12-04T05:49:23.396Z [INFO] agent.router: Initializing LAN area manager +49:23.396 [INFO] TestAgent.server: Adding LAN server: server="Node-c324c30d-944e-896c-c91d-b98c2c7efc9a (Addr: tcp/127.0.0.1:23587) (DC: dc1)" +49:23.396 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c324c30d-944e-896c-c91d-b98c2c7efc9a.dc1 area=wan +49:23.396 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23610 network=udp +49:23.396 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23610 network=tcp +49:23.397 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23611 network=tcp +49:23.397 [INFO] TestAgent: started state syncer +49:23.397 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23588 network=tcp +49:23.441 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.441 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23587 [Candidate]" term=2 +49:23.441 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.441 [DEBUG] TestAgent.server.raft: vote granted: from=c324c30d-944e-896c-c91d-b98c2c7efc9a term=2 tally=1 +49:23.441 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.441 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23587 [Leader]" +49:23.441 [INFO] TestAgent.server: cluster leadership acquired +49:23.441 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23587 +49:23.441 [INFO] TestAgent.server: New leader elected: payload=Node-c324c30d-944e-896c-c91d-b98c2c7efc9a +49:23.442 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.443 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.443 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.443 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.443 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.443 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.443 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c324c30d-944e-896c-c91d-b98c2c7efc9a +49:23.443 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c324c30d-944e-896c-c91d-b98c2c7efc9a +49:23.444 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.667 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:23.668 [INFO] TestAgent: Synced node info +49:23.737 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:23.737 [WARN] TestAgent.dns: Query results too stale, re-requesting +49:23.737 [DEBUG] TestAgent.dns: request served from client: name=nope.query.consul. type=SRV class=IN latency=160.969µs client=127.0.0.1:47027 client_network=udp +49:23.737 [INFO] TestAgent: Requesting shutdown +49:23.737 [INFO] TestAgent.server: shutting down server +49:23.737 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.737 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.737 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.737 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.737 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.737 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.737 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.737 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.737 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.737 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:23.737 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:23.738Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.738Z [INFO] agent.router.manager: shutting down +49:23.738 [INFO] TestAgent: consul server down +49:23.738 [INFO] TestAgent: shutdown complete +49:23.738 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23610 network=tcp +49:23.738 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23610 network=udp +49:23.738 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23611 network=tcp +49:24.238 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.238 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_PreparedQuery_AllowStale (0.87s) +=== CONT TestDNS_ServiceLookup_SRV_RFC +2022-12-04T05:49:23.641Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.652Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.654 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bed3320c-8777-d24f-77db-2e807d097cfc Address:127.0.0.1:23629}]" +49:23.654 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23629 [Follower]" leader= +49:23.654 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bed3320c-8777-d24f-77db-2e807d097cfc.dc1 127.0.0.1 +49:23.655 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bed3320c-8777-d24f-77db-2e807d097cfc 127.0.0.1 +2022-12-04T05:49:23.655Z [INFO] agent.router: Initializing LAN area manager +49:23.655 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bed3320c-8777-d24f-77db-2e807d097cfc.dc1 area=wan +49:23.655 [INFO] TestAgent.server: Adding LAN server: server="Node-bed3320c-8777-d24f-77db-2e807d097cfc (Addr: tcp/127.0.0.1:23629) (DC: dc1)" +49:23.655 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23575 network=udp +49:23.655 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23575 network=tcp +49:23.655 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23576 network=tcp +49:23.655 [INFO] TestAgent: started state syncer +49:23.656 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23630 network=tcp +49:23.714 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.714 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23629 [Candidate]" term=2 +49:23.714 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.714 [DEBUG] TestAgent.server.raft: vote granted: from=bed3320c-8777-d24f-77db-2e807d097cfc term=2 tally=1 +49:23.714 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.714 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23629 [Leader]" +49:23.715 [INFO] TestAgent.server: cluster leadership acquired +49:23.715 [INFO] TestAgent.server: New leader elected: payload=Node-bed3320c-8777-d24f-77db-2e807d097cfc +49:23.715 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23629 +49:23.716 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.717 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.717 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.717 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.717 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.717 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.717 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bed3320c-8777-d24f-77db-2e807d097cfc +49:23.717 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bed3320c-8777-d24f-77db-2e807d097cfc +49:23.717 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.788 [DEBUG] TestAgent.dns: request served from client: name=2607002040050808000000000000200e.addr.consul. type=SRV class=IN latency=39.583µs client=127.0.0.1:46090 client_network=udp +49:23.788 [DEBUG] TestAgent.dns: request served from client: name=2607112040051808ffffffffffff200e.addr.consul. type=SRV class=IN latency=19.94µs client=127.0.0.1:48674 client_network=udp +49:23.788 [INFO] TestAgent: Requesting shutdown +49:23.788 [INFO] TestAgent.server: shutting down server +49:23.788 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.788 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.788 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.788 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.788 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.788 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.788 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.788 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:23.788 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.788 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.788 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:23.788 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:23.788Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.789Z [INFO] agent.router.manager: shutting down +49:23.789 [INFO] TestAgent: consul server down +49:23.789 [INFO] TestAgent: shutdown complete +49:23.789 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23575 network=tcp +49:23.789 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23575 network=udp +49:23.789 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23576 network=tcp +49:24.289 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.289 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_AddressLookupIPV6 (0.67s) +=== CONT TestDNS_PreparedQuery_Failover +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:23.379Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.389Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.390 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1ebde856-d19d-1f30-5391-32dcc87ff22c Address:127.0.0.1:23601}]" +49:23.390 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23601 [Follower]" leader= +49:23.390 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1ebde856-d19d-1f30-5391-32dcc87ff22c.dc1 127.0.0.1 +49:23.391 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1ebde856-d19d-1f30-5391-32dcc87ff22c 127.0.0.1 +2022-12-04T05:49:23.391Z [INFO] agent.router: Initializing LAN area manager +49:23.391 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1ebde856-d19d-1f30-5391-32dcc87ff22c.dc1 area=wan +49:23.391 [INFO] TestAgent.server: Adding LAN server: server="Node-1ebde856-d19d-1f30-5391-32dcc87ff22c (Addr: tcp/127.0.0.1:23601) (DC: dc1)" +49:23.391 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23582 network=udp +49:23.391 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23582 network=tcp +49:23.392 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23583 network=tcp +49:23.392 [INFO] TestAgent: started state syncer +49:23.392 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23602 network=tcp +49:23.430 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.430 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23601 [Candidate]" term=2 +49:23.430 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.430 [DEBUG] TestAgent.server.raft: vote granted: from=1ebde856-d19d-1f30-5391-32dcc87ff22c term=2 tally=1 +49:23.430 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.430 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23601 [Leader]" +49:23.430 [INFO] TestAgent.server: cluster leadership acquired +49:23.430 [INFO] TestAgent.server: New leader elected: payload=Node-1ebde856-d19d-1f30-5391-32dcc87ff22c +49:23.430 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23601 +49:23.431 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.432 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.432 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.432 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.432 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.432 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.432 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1ebde856-d19d-1f30-5391-32dcc87ff22c +49:23.432 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1ebde856-d19d-1f30-5391-32dcc87ff22c +49:23.433 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.638 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:23.638 [INFO] TestAgent: Synced node info +49:23.638 [DEBUG] TestAgent: Node info in sync +49:23.800 [WARN] TestAgent.dns: QName invalid: qname= +49:23.800 [DEBUG] TestAgent.dns: request served from client: name=consul. type=SRV class=IN latency=103.539µs client=127.0.0.1:48764 client_network=udp +49:23.804 [WARN] TestAgent.dns: QName invalid: qname=node. +49:23.804 [DEBUG] TestAgent.dns: request served from client: name=node.consul. type=SRV class=IN latency=51.835µs client=127.0.0.1:35811 client_network=udp +49:23.808 [WARN] TestAgent.dns: QName invalid: qname=service. +49:23.808 [DEBUG] TestAgent.dns: request served from client: name=service.consul. type=SRV class=IN latency=53.973µs client=127.0.0.1:49968 client_network=udp +49:23.820 [WARN] TestAgent.dns: QName invalid: qname=query. +49:23.820 [DEBUG] TestAgent.dns: request served from client: name=query.consul. type=SRV class=IN latency=108.368µs client=127.0.0.1:50240 client_network=udp +49:23.820 [WARN] TestAgent.dns: QName invalid: qname=foo.node.dc1.extra.more. +49:23.820 [DEBUG] TestAgent.dns: request served from client: name=foo.node.dc1.extra.more.consul. type=SRV class=IN latency=38.896µs client=127.0.0.1:60628 client_network=udp +49:23.821 [WARN] TestAgent.dns: QName invalid: qname=foo.service.dc1.extra.more. +49:23.821 [DEBUG] TestAgent.dns: request served from client: name=foo.service.dc1.extra.more.consul. type=SRV class=IN latency=30.707µs client=127.0.0.1:46725 client_network=udp +49:23.821 [WARN] TestAgent.dns: QName invalid: qname=foo.query.dc1.extra.more. +49:23.821 [DEBUG] TestAgent.dns: request served from client: name=foo.query.dc1.extra.more.consul. type=SRV class=IN latency=53.36µs client=127.0.0.1:50412 client_network=udp +49:23.821 [INFO] TestAgent: Requesting shutdown +49:23.821 [INFO] TestAgent.server: shutting down server +49:23.821 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.821 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.821 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.821 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.821 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.821 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.821 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.821 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.821 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.821 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:23.821 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:23.821Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.821Z [INFO] agent.router.manager: shutting down +49:23.822 [INFO] TestAgent: consul server down +49:23.822 [INFO] TestAgent: shutdown complete +49:23.822 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23582 network=tcp +49:23.822 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23582 network=udp +49:23.822 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23583 network=tcp +49:24.324 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.324 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_InvalidQueries (0.96s) +=== CONT TestDNS_NonExistentDC_RPC +2022-12-04T05:49:23.802Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:23.802Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.837Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:23.837Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.838 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4b36f1c5-ec86-c5c4-1451-70e8760802b1 Address:127.0.0.1:23622}]" +49:23.839 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4b36f1c5-ec86-c5c4-1451-70e8760802b1.dc1 127.0.0.1 +49:23.839 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4b36f1c5-ec86-c5c4-1451-70e8760802b1 127.0.0.1 +2022-12-04T05:49:23.839Z [INFO] agent.router: Initializing LAN area manager +49:23.839 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23622 [Follower]" leader= +49:23.839 [INFO] TestAgent.server: Adding LAN server: server="Node-4b36f1c5-ec86-c5c4-1451-70e8760802b1 (Addr: tcp/127.0.0.1:23622) (DC: dc1)" +49:23.839 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23659 network=udp +49:23.839 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4b36f1c5-ec86-c5c4-1451-70e8760802b1.dc1 area=wan +49:23.840 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23659 network=tcp +49:23.840 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23660 network=tcp +49:23.840 [INFO] TestAgent: started state syncer +49:23.840 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23623 network=tcp +49:23.890 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:23.890 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.890 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23622 [Candidate]" term=2 +49:23.890 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.890 [DEBUG] TestAgent.server.raft: vote granted: from=4b36f1c5-ec86-c5c4-1451-70e8760802b1 term=2 tally=1 +49:23.890 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.890 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23622 [Leader]" +49:23.891 [INFO] TestAgent.server: cluster leadership acquired +49:23.891 [INFO] TestAgent.server: initializing acls +49:23.891 [INFO] TestAgent.server: New leader elected: payload=Node-4b36f1c5-ec86-c5c4-1451-70e8760802b1 +49:23.891 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:23.891 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:23.891 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:23.891 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:23.891 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:23.891 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:23.892 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4b36f1c5-ec86-c5c4-1451-70e8760802b1 +49:23.892 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4b36f1c5-ec86-c5c4-1451-70e8760802b1.dc1 +49:23.892 [INFO] TestAgent.server: Updating LAN server: server="Node-4b36f1c5-ec86-c5c4-1451-70e8760802b1 (Addr: tcp/127.0.0.1:23622) (DC: dc1)" +49:23.892 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4b36f1c5-ec86-c5c4-1451-70e8760802b1.dc1 area=wan +49:23.892 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.893 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.893 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.893 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.893 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.893 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.893 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4b36f1c5-ec86-c5c4-1451-70e8760802b1 +49:23.893 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4b36f1c5-ec86-c5c4-1451-70e8760802b1 +49:23.894 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4b36f1c5-ec86-c5c4-1451-70e8760802b1 +49:23.894 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.901 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4b36f1c5-ec86-c5c4-1451-70e8760802b1 +49:23.902 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4b36f1c5-ec86-c5c4-1451-70e8760802b1 +49:23.903 [DEBUG] TestAgent.dns: request served from client: name=foo.service.consul. type=A class=IN latency=172.739µs client=127.0.0.1:58935 client_network=udp +49:23.903 [INFO] TestAgent: Requesting shutdown +49:23.903 [INFO] TestAgent.server: shutting down server +49:23.903 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:23.903 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.903 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.903 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.903 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.903 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:23.903 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.903 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.903 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.903 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:23.903 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:23.903 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.903 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.903 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:23.903 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:23.903 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:23.903Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.903Z [INFO] agent.router.manager: shutting down +49:23.903 [INFO] TestAgent: consul server down +49:23.903 [INFO] TestAgent: shutdown complete +49:23.903 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23659 network=tcp +49:23.903 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23659 network=udp +49:23.903 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23660 network=tcp +49:24.404 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.404 [INFO] TestAgent: Endpoints down +=== RUN TestDNS_ServiceLookup_FilterACL/ACLToken_==_anonymous +2022-12-04T05:49:23.807Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.836Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.837 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d163e772-3504-3dff-9369-83e955303c4d Address:127.0.0.1:23657}]" +49:23.838 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d163e772-3504-3dff-9369-83e955303c4d.dc1 127.0.0.1 +49:23.838 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d163e772-3504-3dff-9369-83e955303c4d 127.0.0.1 +2022-12-04T05:49:23.838Z [INFO] agent.router: Initializing LAN area manager +49:23.838 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23638 network=udp +49:23.838 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23657 [Follower]" leader= +49:23.838 [INFO] TestAgent.server: Adding LAN server: server="Node-d163e772-3504-3dff-9369-83e955303c4d (Addr: tcp/127.0.0.1:23657) (DC: dc1)" +49:23.838 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d163e772-3504-3dff-9369-83e955303c4d.dc1 area=wan +49:23.839 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23638 network=tcp +49:23.839 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23639 network=tcp +49:23.839 [INFO] TestAgent: started state syncer +49:23.839 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23658 network=tcp +49:23.903 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.903 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23657 [Candidate]" term=2 +49:23.903 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.903 [DEBUG] TestAgent.server.raft: vote granted: from=d163e772-3504-3dff-9369-83e955303c4d term=2 tally=1 +49:23.903 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.903 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23657 [Leader]" +49:23.903 [INFO] TestAgent.server: cluster leadership acquired +49:23.903 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23657 +49:23.903 [INFO] TestAgent.server: New leader elected: payload=Node-d163e772-3504-3dff-9369-83e955303c4d +49:23.904 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.905 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.905 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.905 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.905 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.905 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.905 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d163e772-3504-3dff-9369-83e955303c4d +49:23.905 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d163e772-3504-3dff-9369-83e955303c4d +49:23.905 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.937 [WARN] TestAgent.dns: QName invalid: qname=nonexisting. +49:23.937 [DEBUG] TestAgent.dns: request served from client: name=nonexisting.consul. type=ANY class=IN latency=109.165µs client=127.0.0.1:47589 client_network=udp +49:23.937 [INFO] TestAgent: Requesting shutdown +49:23.937 [INFO] TestAgent.server: shutting down server +49:23.937 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.937 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.937 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.937 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.937 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.937 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:23.937 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.937 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.937 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.937 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:23.937 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.937 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:23.937Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.937Z [INFO] agent.router.manager: shutting down +49:23.938 [INFO] TestAgent: consul server down +49:23.938 [INFO] TestAgent: shutdown complete +49:23.938 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23638 network=tcp +49:23.938 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23638 network=udp +49:23.938 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23639 network=tcp +49:24.439 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.439 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NonExistingLookup (0.64s) === CONT TestDNS_ServiceLookup_AnswerLimits === RUN TestDNS_ServiceLookup_AnswerLimits/A_lookup_{0_0_0_0_0_0_0_0_0_0_0} === PAUSE TestDNS_ServiceLookup_AnswerLimits/A_lookup_{0_0_0_0_0_0_0_0_0_0_0} @@ -27727,782 +33891,715 @@ === PAUSE TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} === RUN TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} === PAUSE TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} -=== CONT TestDNS_TCP_and_UDP_Truncate -2021-10-31T23:13:10.961Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:10.998Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:11.000 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:093e397b-1ac1-9ca1-1101-c731ade2cbab Address:127.0.0.1:16840}]" -13:11.001 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-093e397b-1ac1-9ca1-1101-c731ade2cbab.dc1 127.0.0.1 -13:11.001 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16840 [Follower]" leader= -13:11.009 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-093e397b-1ac1-9ca1-1101-c731ade2cbab 127.0.0.1 -2021-10-31T23:13:11.009Z [INFO] agent.router: Initializing LAN area manager -13:11.010 [INFO] TestAgent.server: Adding LAN server: server="Node-093e397b-1ac1-9ca1-1101-c731ade2cbab (Addr: tcp/127.0.0.1:16840) (DC: dc1)" -13:11.010 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-093e397b-1ac1-9ca1-1101-c731ade2cbab.dc1 area=wan -13:11.010 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16835 network=udp -13:11.010 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16835 network=tcp -13:11.011 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16836 network=tcp -13:11.011 [INFO] TestAgent: started state syncer -13:11.011 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16841 network=tcp -13:11.049 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:11.059 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16840 [Candidate]" term=2 -13:11.059 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:11.059 [DEBUG] TestAgent.server.raft: vote granted: from=093e397b-1ac1-9ca1-1101-c731ade2cbab term=2 tally=1 -13:11.059 [INFO] TestAgent.server.raft: election won: tally=1 -13:11.059 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16840 [Leader]" -13:11.059 [INFO] TestAgent.server: cluster leadership acquired -13:11.059 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16840 -13:11.060 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.083 [INFO] TestAgent.server: New leader elected: payload=Node-093e397b-1ac1-9ca1-1101-c731ade2cbab -13:11.087 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.087 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.087 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.087 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.087 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.087 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-093e397b-1ac1-9ca1-1101-c731ade2cbab -13:11.087 [INFO] TestAgent.server: member joined, marking health alive: member=Node-093e397b-1ac1-9ca1-1101-c731ade2cbab -13:11.088 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.089 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=127.197µs client=127.0.0.1:43712 client_network=udp -13:11.089 [DEBUG] TestAgent.dns: request served from client: name=db.service.test-domain. type=SRV class=IN latency=68.81µs client=127.0.0.1:43355 client_network=udp -13:11.090 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc1.consul. type=SRV class=IN latency=73.781µs client=127.0.0.1:36205 client_network=udp -13:11.090 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc1.test-domain. type=SRV class=IN latency=91.914µs client=127.0.0.1:42565 client_network=udp -13:11.090 [INFO] TestAgent: Requesting shutdown -13:11.090 [INFO] TestAgent.server: shutting down server -13:11.090 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:11.090 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:11.090 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:11.090 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:11.090 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:11.090 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:11.090 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:11.090 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:11.090 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:11.090 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:11.090 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:11.090 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:11.090Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:11.090Z [INFO] agent.router.manager: shutting down -13:11.090 [INFO] TestAgent: consul server down -13:11.090 [INFO] TestAgent: shutdown complete -13:11.090 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16835 network=tcp -13:11.090 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16835 network=udp -13:11.090 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16836 network=tcp -13:11.591 [INFO] TestAgent: Waiting for endpoints to shut down -13:11.591 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_AltDomains_Service (0.72s) -=== CONT TestDNS_ServiceLookup_LargeResponses -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:10.820Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:10.930Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:10.932 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4641c3a2-edd2-ee46-c1cf-8f225c7942a2 Address:127.0.0.1:16812}]" -13:10.932 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4641c3a2-edd2-ee46-c1cf-8f225c7942a2.dc1 127.0.0.1 -13:10.933 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4641c3a2-edd2-ee46-c1cf-8f225c7942a2 127.0.0.1 -2021-10-31T23:13:10.933Z [INFO] agent.router: Initializing LAN area manager -13:10.933 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16807 network=udp -13:10.933 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16812 [Follower]" leader= -13:10.934 [INFO] TestAgent.server: Adding LAN server: server="Node-4641c3a2-edd2-ee46-c1cf-8f225c7942a2 (Addr: tcp/127.0.0.1:16812) (DC: dc1)" -13:10.934 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4641c3a2-edd2-ee46-c1cf-8f225c7942a2.dc1 area=wan -13:10.934 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16807 network=tcp -13:10.934 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16808 network=tcp -13:10.934 [INFO] TestAgent: started state syncer -13:10.934 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16813 network=tcp -13:10.981 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:10.981 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16812 [Candidate]" term=2 -13:10.982 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:10.982 [DEBUG] TestAgent.server.raft: vote granted: from=4641c3a2-edd2-ee46-c1cf-8f225c7942a2 term=2 tally=1 -13:10.982 [INFO] TestAgent.server.raft: election won: tally=1 -13:10.982 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16812 [Leader]" -13:10.982 [INFO] TestAgent.server: cluster leadership acquired -13:10.982 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16812 -13:10.983 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:10.984 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:10.984 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:10.984 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:10.984 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:10.984 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:10.984 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4641c3a2-edd2-ee46-c1cf-8f225c7942a2 -13:10.984 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4641c3a2-edd2-ee46-c1cf-8f225c7942a2 -13:10.985 [INFO] TestAgent.server: New leader elected: payload=Node-4641c3a2-edd2-ee46-c1cf-8f225c7942a2 -13:10.985 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.119 [WARN] TestAgent.dns: QName invalid: qname=nonexisting. -13:11.119 [DEBUG] TestAgent.dns: request served from client: name=nonexisting.consul. type=ANY class=IN latency=83.314µs client=127.0.0.1:47808 client_network=udp -13:11.119 [INFO] TestAgent: Requesting shutdown -13:11.119 [INFO] TestAgent.server: shutting down server -13:11.119 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:11.119 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:11.119 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:11.119 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:11.119 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:11.119 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:11.119 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:11.119 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:11.119 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:11.119 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:11.119 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:11.119 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:11.119Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:11.119Z [INFO] agent.router.manager: shutting down -13:11.120 [INFO] TestAgent: consul server down -13:11.120 [INFO] TestAgent: shutdown complete -13:11.120 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16807 network=tcp -13:11.120 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16807 network=udp -13:11.120 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16808 network=tcp -13:11.620 [INFO] TestAgent: Waiting for endpoints to shut down -13:11.620 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NonExistingLookup (0.85s) === CONT TestBinarySearch -2021-10-31T23:13:10.697Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:10.737Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:10.743 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4301ea79-8210-2d21-864d-18d8f6c8e4f5 Address:127.0.0.1:16791}]" -13:10.743 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16791 [Follower]" leader= -13:10.744 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4301ea79-8210-2d21-864d-18d8f6c8e4f5.dc1 127.0.0.1 -13:10.744 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4301ea79-8210-2d21-864d-18d8f6c8e4f5 127.0.0.1 -2021-10-31T23:13:10.744Z [INFO] agent.router: Initializing LAN area manager -13:10.744 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16786 network=udp -13:10.744 [INFO] TestAgent.server: Adding LAN server: server="Node-4301ea79-8210-2d21-864d-18d8f6c8e4f5 (Addr: tcp/127.0.0.1:16791) (DC: dc1)" -13:10.744 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4301ea79-8210-2d21-864d-18d8f6c8e4f5.dc1 area=wan -13:10.745 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16786 network=tcp -13:10.755 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16787 network=tcp -13:10.756 [INFO] TestAgent: started state syncer -13:10.756 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16771 network=tcp -13:10.793 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:10.793 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16791 [Candidate]" term=2 -13:10.794 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:10.794 [DEBUG] TestAgent.server.raft: vote granted: from=4301ea79-8210-2d21-864d-18d8f6c8e4f5 term=2 tally=1 -13:10.794 [INFO] TestAgent.server.raft: election won: tally=1 -13:10.794 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16791 [Leader]" -13:10.794 [INFO] TestAgent.server: cluster leadership acquired -13:10.794 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16791 -13:10.795 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:10.796 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:10.796 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:10.796 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:10.796 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:10.796 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:10.796 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4301ea79-8210-2d21-864d-18d8f6c8e4f5 -13:10.796 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4301ea79-8210-2d21-864d-18d8f6c8e4f5 -13:10.797 [INFO] TestAgent.server: New leader elected: payload=Node-4301ea79-8210-2d21-864d-18d8f6c8e4f5 -13:10.797 [INFO] TestAgent.server: federation state anti-entropy synced -13:10.875 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:10.875 [INFO] TestAgent: Synced node info -13:10.875 [DEBUG] TestAgent: Node info in sync -13:11.133 [WARN] TestAgent.dns: QName invalid: qname= -13:11.134 [DEBUG] TestAgent.dns: request served from client: name=consul. type=SRV class=IN latency=122.082µs client=127.0.0.1:45282 client_network=udp -13:11.134 [WARN] TestAgent.dns: QName invalid: qname=node. -13:11.134 [DEBUG] TestAgent.dns: request served from client: name=node.consul. type=SRV class=IN latency=27.931µs client=127.0.0.1:47915 client_network=udp -13:11.134 [WARN] TestAgent.dns: QName invalid: qname=service. -13:11.134 [DEBUG] TestAgent.dns: request served from client: name=service.consul. type=SRV class=IN latency=29.084µs client=127.0.0.1:54533 client_network=udp -13:11.134 [WARN] TestAgent.dns: QName invalid: qname=query. -13:11.134 [DEBUG] TestAgent.dns: request served from client: name=query.consul. type=SRV class=IN latency=37.217µs client=127.0.0.1:56310 client_network=udp -13:11.134 [WARN] TestAgent.dns: QName invalid: qname=foo.node.dc1.extra.more. -13:11.134 [DEBUG] TestAgent.dns: request served from client: name=foo.node.dc1.extra.more.consul. type=SRV class=IN latency=40.613µs client=127.0.0.1:48753 client_network=udp -13:11.134 [WARN] TestAgent.dns: QName invalid: qname=foo.service.dc1.extra.more. -13:11.134 [DEBUG] TestAgent.dns: request served from client: name=foo.service.dc1.extra.more.consul. type=SRV class=IN latency=33.346µs client=127.0.0.1:56038 client_network=udp -13:11.135 [WARN] TestAgent.dns: QName invalid: qname=foo.query.dc1.extra.more. -13:11.135 [DEBUG] TestAgent.dns: request served from client: name=foo.query.dc1.extra.more.consul. type=SRV class=IN latency=32.143µs client=127.0.0.1:45531 client_network=udp -13:11.135 [INFO] TestAgent: Requesting shutdown -13:11.135 [INFO] TestAgent.server: shutting down server -13:11.135 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:11.135 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:11.135 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:11.135 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:11.135 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:11.135 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:11.135 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:11.135 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:11.135 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:11.135 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:11.135 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:11.135Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:11.135Z [INFO] agent.router.manager: shutting down -13:11.135 [INFO] TestAgent: consul server down -13:11.135 [INFO] TestAgent: shutdown complete -13:11.135 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16786 network=tcp -13:11.135 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16786 network=udp -13:11.135 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16787 network=tcp -13:11.637 [INFO] TestAgent: Waiting for endpoints to shut down -13:11.637 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_InvalidQueries (0.98s) -=== CONT TestDNS_ServiceLookup_Truncate === RUN TestBinarySearch/binarySearch_12 === RUN TestBinarySearch/binarySearch_256 -2021-10-31T23:13:10.742Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:10.788Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:10.802 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d6b902f5-d39a-a6a9-5986-3dafcd9cb8ce Address:127.0.0.1:16798}]" -13:10.803 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d6b902f5-d39a-a6a9-5986-3dafcd9cb8ce.dc1 127.0.0.1 -13:10.803 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d6b902f5-d39a-a6a9-5986-3dafcd9cb8ce 127.0.0.1 -2021-10-31T23:13:10.803Z [INFO] agent.router: Initializing LAN area manager -13:10.803 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16793 network=udp -13:10.803 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16798 [Follower]" leader= -13:10.804 [INFO] TestAgent.server: Adding LAN server: server="Node-d6b902f5-d39a-a6a9-5986-3dafcd9cb8ce (Addr: tcp/127.0.0.1:16798) (DC: dc1)" -13:10.804 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d6b902f5-d39a-a6a9-5986-3dafcd9cb8ce.dc1 area=wan -13:10.804 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16793 network=tcp -13:10.804 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16794 network=tcp -13:10.804 [INFO] TestAgent: started state syncer -13:10.804 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16799 network=tcp -13:10.862 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:10.862 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16798 [Candidate]" term=2 -13:10.862 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:10.862 [DEBUG] TestAgent.server.raft: vote granted: from=d6b902f5-d39a-a6a9-5986-3dafcd9cb8ce term=2 tally=1 -13:10.862 [INFO] TestAgent.server.raft: election won: tally=1 -13:10.862 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16798 [Leader]" -13:10.863 [INFO] TestAgent.server: cluster leadership acquired -13:10.863 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16798 -13:10.865 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:10.865 [INFO] TestAgent.server: New leader elected: payload=Node-d6b902f5-d39a-a6a9-5986-3dafcd9cb8ce -13:10.876 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:10.876 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:10.876 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:10.876 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:10.876 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:10.876 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d6b902f5-d39a-a6a9-5986-3dafcd9cb8ce -13:10.876 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d6b902f5-d39a-a6a9-5986-3dafcd9cb8ce -13:10.877 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.105 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:11.106 [INFO] TestAgent: Synced node info -13:11.186 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:11.186 [WARN] TestAgent.dns: Query results too stale, re-requesting -13:11.186 [DEBUG] TestAgent.dns: request served from client: name=nope.query.consul. type=SRV class=IN latency=77.266µs client=127.0.0.1:58630 client_network=udp -13:11.186 [INFO] TestAgent: Requesting shutdown -13:11.186 [INFO] TestAgent.server: shutting down server -13:11.186 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:11.186 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:11.186 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:11.186 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:11.186 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:11.186 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:11.186 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:11.186 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:11.186 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:11.186 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:11.187 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:11.187Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:11.187Z [INFO] agent.router.manager: shutting down -13:11.187 [INFO] TestAgent: consul server down -13:11.187 [INFO] TestAgent: shutdown complete -13:11.187 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16793 network=tcp -13:11.187 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16793 network=udp -13:11.187 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16794 network=tcp -13:11.687 [INFO] TestAgent: Waiting for endpoints to shut down -13:11.687 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_PreparedQuery_AllowStale (1.00s) -=== CONT TestDNS_PreparedQuery_Failover -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=255)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=255)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=255)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=255)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=true === RUN TestBinarySearch/binarySearch_512 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=1)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=1)_compressed=false -2021-10-31T23:13:10.978Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:11.086Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:11.088 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4c21773c-c2f7-c23c-e253-9a9237e7c81b Address:127.0.0.1:16861}]" -13:11.088 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4c21773c-c2f7-c23c-e253-9a9237e7c81b.dc1 127.0.0.1 -13:11.089 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16861 [Follower]" leader= -13:11.089 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4c21773c-c2f7-c23c-e253-9a9237e7c81b 127.0.0.1 -2021-10-31T23:13:11.089Z [INFO] agent.router: Initializing LAN area manager -13:11.089 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16856 network=udp -13:11.090 [INFO] TestAgent.server: Adding LAN server: server="Node-4c21773c-c2f7-c23c-e253-9a9237e7c81b (Addr: tcp/127.0.0.1:16861) (DC: dc1)" -13:11.090 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4c21773c-c2f7-c23c-e253-9a9237e7c81b.dc1 area=wan -13:11.090 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16856 network=tcp -13:11.090 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16857 network=tcp -13:11.090 [INFO] TestAgent: started state syncer -13:11.091 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16848 network=tcp -13:11.153 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:11.153 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16861 [Candidate]" term=2 -13:11.153 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:11.153 [DEBUG] TestAgent.server.raft: vote granted: from=4c21773c-c2f7-c23c-e253-9a9237e7c81b term=2 tally=1 -13:11.153 [INFO] TestAgent.server.raft: election won: tally=1 -13:11.153 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16861 [Leader]" -13:11.153 [INFO] TestAgent.server: cluster leadership acquired -13:11.153 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16861 -13:11.153 [INFO] TestAgent.server: New leader elected: payload=Node-4c21773c-c2f7-c23c-e253-9a9237e7c81b -13:11.154 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.155 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.155 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.155 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.155 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.155 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.156 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4c21773c-c2f7-c23c-e253-9a9237e7c81b -13:11.156 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4c21773c-c2f7-c23c-e253-9a9237e7c81b -13:11.156 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.214 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:11.214 [INFO] TestAgent: Synced node info -13:11.214 [DEBUG] TestAgent: Node info in sync -13:11.266 [DEBUG] TestAgent.dns: request served from client: name=_db._tcp.service.dc1.consul. type=SRV class=IN latency=217.691µs client=127.0.0.1:37119 client_network=udp -13:11.266 [DEBUG] TestAgent.dns: request served from client: name=_db._tcp.service.consul. type=SRV class=IN latency=80.634µs client=127.0.0.1:49041 client_network=udp -13:11.267 [DEBUG] TestAgent.dns: request served from client: name=_db._tcp.dc1.consul. type=SRV class=IN latency=69.786µs client=127.0.0.1:35766 client_network=udp -13:11.267 [DEBUG] TestAgent.dns: request served from client: name=_db._tcp.consul. type=SRV class=IN latency=69.774µs client=127.0.0.1:49012 client_network=udp -13:11.267 [INFO] TestAgent: Requesting shutdown -13:11.267 [INFO] TestAgent.server: shutting down server -13:11.267 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:11.267 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:11.267 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:11.267 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:11.267 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:11.267 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:11.267 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:11.267 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:11.267 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:11.267 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:11.267 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:11.267Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:11.267Z [INFO] agent.router.manager: shutting down -13:11.267 [INFO] TestAgent: consul server down -13:11.267 [INFO] TestAgent: shutdown complete -13:11.267 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16856 network=tcp -13:11.267 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16856 network=udp -13:11.267 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16857 network=tcp -13:11.769 [INFO] TestAgent: Waiting for endpoints to shut down -13:11.769 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_SRV_RFC_TCP_Default (0.89s) -=== CONT TestDNS_ServiceLookup_Randomize -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=1)_compressed=true === RUN TestBinarySearch/binarySearch_8192 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=1)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=33)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=33)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=33)_compressed=true -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=33)_compressed=false -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=true#01 === RUN TestBinarySearch/binarySearch_65535 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=255)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=255)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=255)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=255)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=1)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=1)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=1)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=1)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=33)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=33)_compressed=false#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=33)_compressed=true#01 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=33)_compressed=false#01 +2022-12-04T05:49:23.712Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.724Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.726 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7e970517-1b0b-a1f1-add8-9f8d1d8976a7 Address:127.0.0.1:23615}]" +49:23.726 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23615 [Follower]" leader= +49:23.726 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7e970517-1b0b-a1f1-add8-9f8d1d8976a7.dc1 127.0.0.1 +49:23.727 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7e970517-1b0b-a1f1-add8-9f8d1d8976a7 127.0.0.1 +2022-12-04T05:49:23.727Z [INFO] agent.router: Initializing LAN area manager +49:23.727 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23631 network=udp +49:23.727 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7e970517-1b0b-a1f1-add8-9f8d1d8976a7.dc1 area=wan +49:23.727 [INFO] TestAgent.server: Adding LAN server: server="Node-7e970517-1b0b-a1f1-add8-9f8d1d8976a7 (Addr: tcp/127.0.0.1:23615) (DC: dc1)" +49:23.727 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23631 network=tcp +49:23.727 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23632 network=tcp +49:23.728 [INFO] TestAgent: started state syncer +49:23.728 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23616 network=tcp +49:23.777 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.777 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23615 [Candidate]" term=2 +49:23.777 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.777 [DEBUG] TestAgent.server.raft: vote granted: from=7e970517-1b0b-a1f1-add8-9f8d1d8976a7 term=2 tally=1 +49:23.777 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.777 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23615 [Leader]" +49:23.777 [INFO] TestAgent.server: cluster leadership acquired +49:23.777 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23615 +49:23.778 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.778 [INFO] TestAgent.server: New leader elected: payload=Node-7e970517-1b0b-a1f1-add8-9f8d1d8976a7 +49:23.779 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.779 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.779 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.779 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.779 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.779 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7e970517-1b0b-a1f1-add8-9f8d1d8976a7 +49:23.779 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7e970517-1b0b-a1f1-add8-9f8d1d8976a7 +49:23.780 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.974 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=218.504µs client=127.0.0.1:50423 client_network=udp +49:23.974 [DEBUG] TestAgent.dns: request served from client: name=db.service.test-domain. type=SRV class=IN latency=69.783µs client=127.0.0.1:41967 client_network=udp +49:23.974 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc1.consul. type=SRV class=IN latency=132.337µs client=127.0.0.1:45504 client_network=udp +49:23.974 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc1.test-domain. type=SRV class=IN latency=67.241µs client=127.0.0.1:59667 client_network=udp +49:23.975 [INFO] TestAgent: Requesting shutdown +49:23.975 [INFO] TestAgent.server: shutting down server +49:23.975 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.975 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.975 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.975 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.975 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.975 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.975 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.975 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:23.975 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.975 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:23.975 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.975 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:23.975Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.975Z [INFO] agent.router.manager: shutting down +49:23.975 [INFO] TestAgent: consul server down +49:23.975 [INFO] TestAgent: shutdown complete +49:23.975 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23631 network=tcp +49:23.975 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23631 network=udp +49:23.975 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23632 network=tcp +49:24.475 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.475 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_AltDomains_Service (0.78s) +=== CONT TestDNS_PreparedQuery_TTL +2022-12-04T05:49:23.565Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.575Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.576 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ddb8b4f7-71b7-beff-34c7-5e84028cb7e8 Address:127.0.0.1:23566}]" +49:23.576 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23566 [Follower]" leader= +49:23.576 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:23.577 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:23.577Z [INFO] agent.router: Initializing LAN area manager +49:23.577 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23589 network=udp +49:23.577 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23566) (DC: dc1)" +49:23.577 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:23.577 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23589 network=tcp +49:23.578 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23590 network=tcp +49:23.578 [INFO] TestAgent: started state syncer +49:23.578 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23567 network=tcp +49:23.631 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.631 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23566 [Candidate]" term=2 +49:23.632 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.632 [DEBUG] TestAgent.server.raft: vote granted: from=ddb8b4f7-71b7-beff-34c7-5e84028cb7e8 term=2 tally=1 +49:23.632 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.632 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23566 [Leader]" +49:23.632 [INFO] TestAgent.server: cluster leadership acquired +49:23.638 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23566 +49:23.638 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.640 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.640 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.640 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.640 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.640 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.640 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:23.640 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:23.640 [INFO] TestAgent.server: New leader elected: payload=test-node +49:23.640 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.670 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:23.670 [INFO] TestAgent: Synced node info +49:23.670 [DEBUG] TestAgent: Node info in sync +49:23.975 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.consul. type=A class=IN latency=115.585µs client=127.0.0.1:36191 client_network=udp +49:23.976 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.test.consul. type=A class=IN latency=64.875µs client=127.0.0.1:56347 client_network=udp +49:23.976 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.dc1.consul. type=A class=IN latency=91.34µs client=127.0.0.1:44572 client_network=udp +49:23.976 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.dc1.test.consul. type=A class=IN latency=76.073µs client=127.0.0.1:57289 client_network=udp +49:23.976 [INFO] TestAgent: Requesting shutdown +49:23.976 [INFO] TestAgent.server: shutting down server +49:23.976 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:23.976 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:23.976 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:23.976 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:23.976 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:23.976 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:23.976 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:23.976 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:23.976 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:23.976 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:23.976 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:23.976Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:23.977Z [INFO] agent.router.manager: shutting down +49:23.977 [INFO] TestAgent: consul server down +49:23.977 [INFO] TestAgent: shutdown complete +49:23.977 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23589 network=tcp +49:23.977 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23589 network=udp +49:23.977 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23590 network=tcp +49:24.477 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.477 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_AltDomains_Overlap (0.92s) +=== CONT TestDNS_ServiceLookup_Randomize === RUN TestBinarySearch/binarySearch_12#01 -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestBinarySearch/binarySearch_256#01 === RUN TestBinarySearch/binarySearch_512#01 -2021-10-31T23:13:11.228Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:11.272Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:11.286 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b946e2c6-f2b8-5309-2fa4-c33681551c5f Address:127.0.0.1:16875}]" -13:11.286 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16875 [Follower]" leader= -13:11.287 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b946e2c6-f2b8-5309-2fa4-c33681551c5f.dc1 127.0.0.1 -13:11.287 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b946e2c6-f2b8-5309-2fa4-c33681551c5f 127.0.0.1 -2021-10-31T23:13:11.287Z [INFO] agent.router: Initializing LAN area manager -13:11.288 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16870 network=udp -13:11.288 [INFO] TestAgent.server: Adding LAN server: server="Node-b946e2c6-f2b8-5309-2fa4-c33681551c5f (Addr: tcp/127.0.0.1:16875) (DC: dc1)" -13:11.288 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b946e2c6-f2b8-5309-2fa4-c33681551c5f.dc1 area=wan -13:11.288 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16870 network=tcp -13:11.288 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16871 network=tcp -13:11.288 [INFO] TestAgent: started state syncer -13:11.289 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16876 network=tcp -13:11.326 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:11.326 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16875 [Candidate]" term=2 -13:11.327 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:11.327 [DEBUG] TestAgent.server.raft: vote granted: from=b946e2c6-f2b8-5309-2fa4-c33681551c5f term=2 tally=1 -13:11.327 [INFO] TestAgent.server.raft: election won: tally=1 -13:11.327 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16875 [Leader]" -13:11.327 [INFO] TestAgent.server: cluster leadership acquired -13:11.327 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16875 -13:11.329 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.330 [INFO] TestAgent.server: New leader elected: payload=Node-b946e2c6-f2b8-5309-2fa4-c33681551c5f -13:11.344 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.344 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.344 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.344 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.344 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.344 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b946e2c6-f2b8-5309-2fa4-c33681551c5f -13:11.344 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b946e2c6-f2b8-5309-2fa4-c33681551c5f -13:11.347 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.378 [DEBUG] TestAgent.dns: request served from client: name=2607112040051808ffffffffffff200e.addr.consul. type=SRV class=IN latency=131.82µs client=127.0.0.1:59675 client_network=udp -13:11.381 [DEBUG] TestAgent.dns: request served from client: name=2607002040050808000000000000200e.addr.consul. type=SRV class=IN latency=41.748µs client=127.0.0.1:36133 client_network=udp -13:11.381 [INFO] TestAgent: Requesting shutdown -13:11.381 [INFO] TestAgent.server: shutting down server -13:11.381 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:11.381 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:11.381 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:11.381 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:11.381 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:11.397 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:11.397 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:11.398 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:11.398 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:11.398 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:11.398 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:11.398 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:11.398Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:11.398Z [INFO] agent.router.manager: shutting down -13:11.398 [INFO] TestAgent: consul server down -13:11.398 [INFO] TestAgent: shutdown complete -13:11.398 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16870 network=tcp -13:11.398 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16870 network=udp -13:11.398 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16871 network=tcp -13:11.910 [INFO] TestAgent: Waiting for endpoints to shut down -13:11.910 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_AddressLookupIPV6 (0.72s) -=== CONT TestDNS_PreparedQuery_TTL -2021-10-31T23:13:10.978Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:11.060Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:11.087 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0809e9ac-7e0d-f281-145e-678a0998640e Address:127.0.0.1:16854}]" -13:11.087 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0809e9ac-7e0d-f281-145e-678a0998640e.dc1 127.0.0.1 -13:11.088 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0809e9ac-7e0d-f281-145e-678a0998640e 127.0.0.1 -2021-10-31T23:13:11.088Z [INFO] agent.router: Initializing LAN area manager -13:11.088 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16849 network=udp -13:11.088 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16854 [Follower]" leader= -13:11.088 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0809e9ac-7e0d-f281-145e-678a0998640e.dc1 area=wan -13:11.088 [INFO] TestAgent.server: Adding LAN server: server="Node-0809e9ac-7e0d-f281-145e-678a0998640e (Addr: tcp/127.0.0.1:16854) (DC: dc1)" -13:11.088 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16849 network=tcp -13:11.089 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16850 network=tcp -13:11.089 [INFO] TestAgent: started state syncer -13:11.089 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16862 network=tcp -13:11.137 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:11.137 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16854 [Candidate]" term=2 -13:11.138 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:11.138 [DEBUG] TestAgent.server.raft: vote granted: from=0809e9ac-7e0d-f281-145e-678a0998640e term=2 tally=1 -13:11.138 [INFO] TestAgent.server.raft: election won: tally=1 -13:11.138 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16854 [Leader]" -13:11.138 [INFO] TestAgent.server: cluster leadership acquired -13:11.138 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16854 -13:11.139 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.140 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.140 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.140 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.140 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.140 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.140 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0809e9ac-7e0d-f281-145e-678a0998640e -13:11.140 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0809e9ac-7e0d-f281-145e-678a0998640e -13:11.140 [INFO] TestAgent.server: New leader elected: payload=Node-0809e9ac-7e0d-f281-145e-678a0998640e -13:11.141 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.229 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:11.229 [INFO] TestAgent: Synced node info -13:11.229 [DEBUG] TestAgent: Node info in sync -13:11.418 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc2 method=Health.ServiceNodes -13:11.418 [ERROR] TestAgent.dns: rpc error: error="No path to datacenter" -13:11.418 [DEBUG] TestAgent.dns: request served from client: name=consul.service.dc2.consul. type=ANY class=IN latency=106.65µs client=127.0.0.1:50395 client_network=udp -13:11.418 [INFO] TestAgent: Requesting shutdown -13:11.418 [INFO] TestAgent.server: shutting down server -13:11.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:11.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:11.418 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:11.418 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:11.418 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:11.419 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:11.419 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:11.419 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:11.419 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:11.419 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:11.419 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:11.419Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:11.419Z [INFO] agent.router.manager: shutting down -13:11.419 [INFO] TestAgent: consul server down -13:11.419 [INFO] TestAgent: shutdown complete -13:11.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16849 network=tcp -13:11.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16849 network=udp -13:11.419 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16850 network=tcp -13:11.922 [INFO] TestAgent: Waiting for endpoints to shut down -13:11.922 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NonExistentDC_Server (0.99s) -=== CONT TestDNS_ServiceLookup_OnlyPassing === RUN TestBinarySearch/binarySearch_8192#01 -2021-10-31T23:13:10.970Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:10.970Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:11.004Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:11.004Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:11.011 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:01506ded-ec89-2da3-b04d-e1552ad789a3 Address:127.0.0.1:16833}]" -13:11.011 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16833 [Follower]" leader= -13:11.011 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-01506ded-ec89-2da3-b04d-e1552ad789a3.dc1 127.0.0.1 -13:11.011 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-01506ded-ec89-2da3-b04d-e1552ad789a3 127.0.0.1 -2021-10-31T23:13:11.011Z [INFO] agent.router: Initializing LAN area manager -13:11.012 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16828 network=udp -13:11.012 [INFO] TestAgent.server: Adding LAN server: server="Node-01506ded-ec89-2da3-b04d-e1552ad789a3 (Addr: tcp/127.0.0.1:16833) (DC: dc1)" -13:11.012 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-01506ded-ec89-2da3-b04d-e1552ad789a3.dc1 area=wan -13:11.012 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16828 network=tcp -13:11.013 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16829 network=tcp -13:11.013 [INFO] TestAgent: started state syncer -13:11.013 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16834 network=tcp -13:11.082 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:11.082 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16833 [Candidate]" term=2 -13:11.082 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:11.082 [DEBUG] TestAgent.server.raft: vote granted: from=01506ded-ec89-2da3-b04d-e1552ad789a3 term=2 tally=1 -13:11.082 [INFO] TestAgent.server.raft: election won: tally=1 -13:11.082 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16833 [Leader]" -13:11.088 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:11.088 [INFO] TestAgent.server: cluster leadership acquired -13:11.089 [INFO] TestAgent.server: initializing acls -13:11.089 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:11.089 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:11.089 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:11.089 [INFO] TestAgent.server: New leader elected: payload=Node-01506ded-ec89-2da3-b04d-e1552ad789a3 -13:11.098 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:11.098 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:11.098 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:11.098 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-01506ded-ec89-2da3-b04d-e1552ad789a3 -13:11.098 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-01506ded-ec89-2da3-b04d-e1552ad789a3.dc1 -13:11.098 [INFO] TestAgent.server: Updating LAN server: server="Node-01506ded-ec89-2da3-b04d-e1552ad789a3 (Addr: tcp/127.0.0.1:16833) (DC: dc1)" -13:11.098 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-01506ded-ec89-2da3-b04d-e1552ad789a3.dc1 area=wan -13:11.099 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.100 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.100 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.100 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.100 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.100 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.100 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-01506ded-ec89-2da3-b04d-e1552ad789a3 -13:11.100 [INFO] TestAgent.server: member joined, marking health alive: member=Node-01506ded-ec89-2da3-b04d-e1552ad789a3 -13:11.101 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-01506ded-ec89-2da3-b04d-e1552ad789a3 -13:11.101 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.275 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:11.275 [INFO] TestAgent: Synced node info -13:11.438 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-01506ded-ec89-2da3-b04d-e1552ad789a3 -13:11.438 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-01506ded-ec89-2da3-b04d-e1552ad789a3 -13:11.442 [DEBUG] TestAgent.dns: request served from client: name=foo.service.consul. type=A class=IN latency=331.731µs client=127.0.0.1:40931 client_network=udp -13:11.442 [INFO] TestAgent: Requesting shutdown -13:11.442 [INFO] TestAgent.server: shutting down server -13:11.442 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:11.442 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:11.442 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:11.442 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:11.442 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:11.442 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:11.442 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:11.442 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:11.442 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:11.443 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:11.443 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:11.443 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:11.443 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:11.443 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:11.443 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -2021-10-31T23:13:11.443Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:11.443Z [INFO] agent.router.manager: shutting down -13:11.443 [INFO] TestAgent: consul server down -13:11.443 [INFO] TestAgent: shutdown complete -13:11.443 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16828 network=tcp -13:11.443 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16828 network=udp -13:11.443 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16829 network=tcp -13:11.943 [INFO] TestAgent: Waiting for endpoints to shut down -13:11.986 [INFO] TestAgent: Endpoints down -=== RUN TestDNS_ServiceLookup_FilterACL/ACLToken_==_anonymous +2022-12-04T05:49:23.645Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.658Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.660 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6b21b798-8ab0-c814-a5b7-32dabce63332 Address:127.0.0.1:23573}]" +49:23.660 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23573 [Follower]" leader= +49:23.660 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6b21b798-8ab0-c814-a5b7-32dabce63332.dc1 127.0.0.1 +49:23.661 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6b21b798-8ab0-c814-a5b7-32dabce63332 127.0.0.1 +2022-12-04T05:49:23.661Z [INFO] agent.router: Initializing LAN area manager +49:23.661 [INFO] TestAgent.server: Adding LAN server: server="Node-6b21b798-8ab0-c814-a5b7-32dabce63332 (Addr: tcp/127.0.0.1:23573) (DC: dc1)" +49:23.661 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6b21b798-8ab0-c814-a5b7-32dabce63332.dc1 area=wan +49:23.661 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23596 network=udp +49:23.661 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23596 network=tcp +49:23.661 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23597 network=tcp +49:23.661 [INFO] TestAgent: started state syncer +49:23.661 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23574 network=tcp +49:23.727 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.727 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23573 [Candidate]" term=2 +49:23.727 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.727 [DEBUG] TestAgent.server.raft: vote granted: from=6b21b798-8ab0-c814-a5b7-32dabce63332 term=2 tally=1 +49:23.727 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.727 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23573 [Leader]" +49:23.727 [INFO] TestAgent.server: cluster leadership acquired +49:23.727 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23573 +49:23.728 [INFO] TestAgent.server: New leader elected: payload=Node-6b21b798-8ab0-c814-a5b7-32dabce63332 +49:23.728 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.730 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.730 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.730 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.730 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.730 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.730 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6b21b798-8ab0-c814-a5b7-32dabce63332 +49:23.730 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6b21b798-8ab0-c814-a5b7-32dabce63332 +49:23.731 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.820 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:23.821 [INFO] TestAgent: Synced node info +49:24.012 [WARN] TestAgent.server: endpoint injected; this should only be used for testing +49:24.013 [DEBUG] TestAgent.dns: request served from client: name=foo.query.consul. type=SRV class=IN latency=140.066µs client=127.0.0.1:45213 client_network=udp +49:24.013 [INFO] TestAgent: Requesting shutdown +49:24.013 [INFO] TestAgent.server: shutting down server +49:24.013 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.013 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.013 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.013 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.013 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.013 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.013 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.013 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.013 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:24.013 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.013 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:49:24.013Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.013Z [INFO] agent.router.manager: shutting down +49:24.013 [INFO] TestAgent: consul server down +49:24.013 [INFO] TestAgent: shutdown complete +49:24.013 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23596 network=tcp +49:24.013 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23596 network=udp +49:24.013 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23597 network=tcp +49:24.516 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.516 [INFO] TestAgent: Endpoints down === RUN TestBinarySearch/binarySearch_65535#01 ---- PASS: TestBinarySearch (0.41s) - --- PASS: TestBinarySearch/binarySearch_12 (0.04s) - --- PASS: TestBinarySearch/binarySearch_256 (0.07s) - --- PASS: TestBinarySearch/binarySearch_512 (0.02s) - --- PASS: TestBinarySearch/binarySearch_8192 (0.03s) - --- PASS: TestBinarySearch/binarySearch_65535 (0.04s) - --- PASS: TestBinarySearch/binarySearch_12#01 (0.02s) - --- PASS: TestBinarySearch/binarySearch_256#01 (0.02s) - --- PASS: TestBinarySearch/binarySearch_512#01 (0.05s) - --- PASS: TestBinarySearch/binarySearch_8192#01 (0.01s) - --- PASS: TestBinarySearch/binarySearch_65535#01 (0.04s) +=== CONT TestDNS_Recurse +--- PASS: TestDNS_PreparedQuery_AgentSource (0.90s) +2022-12-04T05:49:23.665Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.675Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.676 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4a9b99d7-2c09-d2c5-59a8-9a51b24271f3 Address:127.0.0.1:23643}]" +49:23.676 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23643 [Follower]" leader= +49:23.677 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:23.677 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:23.677Z [INFO] agent.router: Initializing LAN area manager +49:23.677 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:23.677 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23643) (DC: dc1)" +49:23.677 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23617 network=udp +49:23.678 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23617 network=tcp +49:23.678 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23618 network=tcp +49:23.678 [INFO] TestAgent: started state syncer +49:23.678 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23644 network=tcp +49:23.738 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.738 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23643 [Candidate]" term=2 +49:23.738 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.738 [DEBUG] TestAgent.server.raft: vote granted: from=4a9b99d7-2c09-d2c5-59a8-9a51b24271f3 term=2 tally=1 +49:23.738 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.738 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23643 [Leader]" +49:23.739 [INFO] TestAgent.server: cluster leadership acquired +49:23.739 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23643 +49:23.739 [INFO] TestAgent.server: New leader elected: payload=test-node +49:23.739 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.741 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.741 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.741 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.741 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.741 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.741 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:23.741 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:23.741 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.760 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:23.760 [INFO] TestAgent: Synced node info +49:24.020 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.consul. type=SOA class=IN latency=195.111µs client=127.0.0.1:50965 client_network=udp +49:24.021 [DEBUG] TestAgent.dns: request served from client: name=test-node.node.test-domain. type=SOA class=IN latency=206.917µs client=127.0.0.1:50472 client_network=udp +49:24.021 [INFO] TestAgent: Requesting shutdown +49:24.021 [INFO] TestAgent.server: shutting down server +49:24.021 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.021 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.021 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.021 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.021 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.021 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.021 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.021 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.021 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.021 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.021 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.021Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.021Z [INFO] agent.router.manager: shutting down +49:24.021 [INFO] TestAgent: consul server down +49:24.021 [INFO] TestAgent: shutdown complete +49:24.021 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23617 network=tcp +49:24.021 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23617 network=udp +49:24.021 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23618 network=tcp +49:24.521 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.521 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_AltDomains_SOA (0.87s) +=== CONT TestDNS_ServiceLookup_OnlyPassing +2022-12-04T05:49:23.605Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.631Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.639 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fd8955c2-efdd-e573-5b69-fc3c5427e7ac Address:127.0.0.1:23594}]" +49:23.640 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fd8955c2-efdd-e573-5b69-fc3c5427e7ac.dc1 127.0.0.1 +49:23.640 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23594 [Follower]" leader= +49:23.640 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fd8955c2-efdd-e573-5b69-fc3c5427e7ac 127.0.0.1 +2022-12-04T05:49:23.640Z [INFO] agent.router: Initializing LAN area manager +49:23.641 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23568 network=udp +49:23.641 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fd8955c2-efdd-e573-5b69-fc3c5427e7ac.dc1 area=wan +49:23.641 [INFO] TestAgent.server: Adding LAN server: server="Node-fd8955c2-efdd-e573-5b69-fc3c5427e7ac (Addr: tcp/127.0.0.1:23594) (DC: dc1)" +49:23.641 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23568 network=tcp +49:23.641 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23569 network=tcp +49:23.641 [INFO] TestAgent: started state syncer +49:23.641 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23595 network=tcp +49:23.700 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.700 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23594 [Candidate]" term=2 +49:23.700 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.700 [DEBUG] TestAgent.server.raft: vote granted: from=fd8955c2-efdd-e573-5b69-fc3c5427e7ac term=2 tally=1 +49:23.700 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.700 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23594 [Leader]" +49:23.700 [INFO] TestAgent.server: cluster leadership acquired +49:23.700 [INFO] TestAgent.server: New leader elected: payload=Node-fd8955c2-efdd-e573-5b69-fc3c5427e7ac +49:23.700 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23594 +49:23.701 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.702 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.702 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.702 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.702 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.702 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.702 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fd8955c2-efdd-e573-5b69-fc3c5427e7ac +49:23.702 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fd8955c2-efdd-e573-5b69-fc3c5427e7ac +49:23.703 [INFO] TestAgent.server: federation state anti-entropy synced +49:23.838 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:23.838 [INFO] TestAgent: Synced node info +49:23.838 [DEBUG] TestAgent: Node info in sync +49:24.024 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc2 method=Health.ServiceNodes +49:24.024 [ERROR] TestAgent.dns: rpc error: error="No path to datacenter" +49:24.024 [DEBUG] TestAgent.dns: request served from client: name=consul.service.dc2.consul. type=ANY class=IN latency=124.523µs client=127.0.0.1:41296 client_network=udp +49:24.024 [INFO] TestAgent: Requesting shutdown +49:24.024 [INFO] TestAgent.server: shutting down server +49:24.024 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.024 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.024 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.024 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.024 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.024 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.024 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.024 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.024 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.024 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.024 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.025Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.025Z [INFO] agent.router.manager: shutting down +49:24.025 [INFO] TestAgent: consul server down +49:24.025 [INFO] TestAgent: shutdown complete +49:24.025 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23568 network=tcp +49:24.025 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23568 network=udp +49:24.025 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23569 network=tcp +49:24.525 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.525 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NonExistentDC_Server (0.93s) +=== CONT TestDNS_ServiceLookup_Dedup_SRV +--- PASS: TestBinarySearch (0.09s) + --- PASS: TestBinarySearch/binarySearch_12 (0.01s) + --- PASS: TestBinarySearch/binarySearch_256 (0.01s) + --- PASS: TestBinarySearch/binarySearch_512 (0.01s) + --- PASS: TestBinarySearch/binarySearch_8192 (0.01s) + --- PASS: TestBinarySearch/binarySearch_65535 (0.01s) + --- PASS: TestBinarySearch/binarySearch_12#01 (0.01s) + --- PASS: TestBinarySearch/binarySearch_256#01 (0.01s) + --- PASS: TestBinarySearch/binarySearch_512#01 (0.01s) + --- PASS: TestBinarySearch/binarySearch_8192#01 (0.02s) + --- PASS: TestBinarySearch/binarySearch_65535#01 (0.01s) === CONT TestDNS_ServiceLookup_Dedup -2021-10-31T23:13:11.011Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:11.106Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:11.108 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1c7c2662-a278-4607-9d44-3e860c27460b Address:127.0.0.1:16868}]" -13:11.109 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1c7c2662-a278-4607-9d44-3e860c27460b.dc1 127.0.0.1 -13:11.109 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16868 [Follower]" leader= -13:11.109 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1c7c2662-a278-4607-9d44-3e860c27460b 127.0.0.1 -2021-10-31T23:13:11.109Z [INFO] agent.router: Initializing LAN area manager -13:11.109 [DEBUG] TestAgent.dns: recursor enabled -13:11.109 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16863 network=udp -13:11.109 [DEBUG] TestAgent.dns: recursor enabled -13:11.109 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1c7c2662-a278-4607-9d44-3e860c27460b.dc1 area=wan -13:11.110 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16863 network=tcp -13:11.110 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16864 network=tcp -13:11.110 [INFO] TestAgent: started state syncer -13:11.110 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16869 network=tcp -13:11.117 [INFO] TestAgent.server: Adding LAN server: server="Node-1c7c2662-a278-4607-9d44-3e860c27460b (Addr: tcp/127.0.0.1:16868) (DC: dc1)" -13:11.164 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:11.164 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16868 [Candidate]" term=2 -13:11.165 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:11.165 [DEBUG] TestAgent.server.raft: vote granted: from=1c7c2662-a278-4607-9d44-3e860c27460b term=2 tally=1 -13:11.165 [INFO] TestAgent.server.raft: election won: tally=1 -13:11.165 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16868 [Leader]" -13:11.165 [INFO] TestAgent.server: cluster leadership acquired -13:11.165 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16868 -13:11.165 [INFO] TestAgent.server: New leader elected: payload=Node-1c7c2662-a278-4607-9d44-3e860c27460b -13:11.166 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.167 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.167 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.167 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.167 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.167 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.167 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1c7c2662-a278-4607-9d44-3e860c27460b -13:11.167 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1c7c2662-a278-4607-9d44-3e860c27460b -13:11.168 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.213 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:11.214 [INFO] TestAgent: Synced node info -13:11.214 [DEBUG] TestAgent: Node info in sync -13:11.538 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=ANY class=IN latency=174.226µs client=127.0.0.1:55646 client_network=udp -13:11.544 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=ANY class=IN latency=126.849µs client=127.0.0.1:54240 client_network=udp -13:11.545 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=66.788µs -13:11.545 [DEBUG] TestAgent.dns: request served from client: name=google.node.consul. type=ANY class=IN latency=151.761µs client=127.0.0.1:48162 client_network=udp -13:11.545 [INFO] TestAgent: Requesting shutdown -13:11.545 [INFO] TestAgent.server: shutting down server -13:11.545 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:11.545 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:11.545 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:11.545 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:11.545 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:11.545 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:11.545 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:11.545 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:11.545 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:11.545 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:11.545 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:11.545Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:11.545Z [INFO] agent.router.manager: shutting down -13:11.545 [INFO] TestAgent: consul server down -13:11.545 [INFO] TestAgent: shutdown complete -13:11.545 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16863 network=tcp -13:11.545 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16863 network=udp -13:11.545 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16864 network=tcp -13:12.077 [INFO] TestAgent: Waiting for endpoints to shut down -13:12.077 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_TTL (1.13s) +[WARN] freeport: 10 out of 66 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:23.890Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.900Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.901 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f934a09c-bef7-f3d9-a0d4-91661ffa1ace Address:127.0.0.1:23685}]" +49:23.901 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23685 [Follower]" leader= +49:23.902 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f934a09c-bef7-f3d9-a0d4-91661ffa1ace.dc1 127.0.0.1 +49:23.902 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f934a09c-bef7-f3d9-a0d4-91661ffa1ace 127.0.0.1 +2022-12-04T05:49:23.902Z [INFO] agent.router: Initializing LAN area manager +49:23.902 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23680 network=udp +49:23.903 [INFO] TestAgent.server: Adding LAN server: server="Node-f934a09c-bef7-f3d9-a0d4-91661ffa1ace (Addr: tcp/127.0.0.1:23685) (DC: dc1)" +49:23.903 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f934a09c-bef7-f3d9-a0d4-91661ffa1ace.dc1 area=wan +49:23.903 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23680 network=tcp +49:23.903 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23681 network=tcp +49:23.903 [INFO] TestAgent: started state syncer +49:23.903 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23686 network=tcp +49:23.941 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.941 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23685 [Candidate]" term=2 +49:23.941 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.941 [DEBUG] TestAgent.server.raft: vote granted: from=f934a09c-bef7-f3d9-a0d4-91661ffa1ace term=2 tally=1 +49:23.941 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.941 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23685 [Leader]" +49:23.941 [INFO] TestAgent.server: cluster leadership acquired +49:23.941 [INFO] TestAgent.server: New leader elected: payload=Node-f934a09c-bef7-f3d9-a0d4-91661ffa1ace +49:23.941 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23685 +49:23.942 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.943 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.943 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.943 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.943 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.943 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.943 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f934a09c-bef7-f3d9-a0d4-91661ffa1ace +49:23.943 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f934a09c-bef7-f3d9-a0d4-91661ffa1ace +49:23.944 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.082 [DEBUG] TestAgent.dns: request served from client: name=_db._tcp.service.dc1.consul. type=SRV class=IN latency=213.022µs client=127.0.0.1:35762 client_network=udp +49:24.082 [DEBUG] TestAgent.dns: request served from client: name=_db._tcp.service.consul. type=SRV class=IN latency=68.661µs client=127.0.0.1:48136 client_network=udp +49:24.082 [DEBUG] TestAgent.dns: request served from client: name=_db._tcp.dc1.consul. type=SRV class=IN latency=72.916µs client=127.0.0.1:44440 client_network=udp +49:24.082 [DEBUG] TestAgent.dns: request served from client: name=_db._tcp.consul. type=SRV class=IN latency=72.708µs client=127.0.0.1:36011 client_network=udp +49:24.082 [INFO] TestAgent: Requesting shutdown +49:24.082 [INFO] TestAgent.server: shutting down server +49:24.082 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.082 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.082 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.082 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.082 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.082 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.082 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.082 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.082 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:24.083 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.082 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.083 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.083Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.083Z [INFO] agent.router.manager: shutting down +49:24.083 [INFO] TestAgent: consul server down +49:24.083 [INFO] TestAgent: shutdown complete +49:24.083 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23680 network=tcp +49:24.083 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23680 network=udp +49:24.083 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23681 network=tcp +49:24.584 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.584 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_SRV_RFC_TCP_Default (0.70s) +=== CONT TestDNS_ServiceLookup_OnlyFailing +2022-12-04T05:49:23.858Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.870Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.872 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:780d4671-b1b8-1150-6ad8-d0c7b20c0c48 Address:127.0.0.1:23664}]" +49:23.872 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23664 [Follower]" leader= +49:23.872 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-780d4671-b1b8-1150-6ad8-d0c7b20c0c48.dc1 127.0.0.1 +49:23.873 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-780d4671-b1b8-1150-6ad8-d0c7b20c0c48 127.0.0.1 +2022-12-04T05:49:23.873Z [INFO] agent.router: Initializing LAN area manager +49:23.873 [INFO] TestAgent.server: Adding LAN server: server="Node-780d4671-b1b8-1150-6ad8-d0c7b20c0c48 (Addr: tcp/127.0.0.1:23664) (DC: dc1)" +49:23.873 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-780d4671-b1b8-1150-6ad8-d0c7b20c0c48.dc1 area=wan +49:23.873 [DEBUG] TestAgent.dns: recursor enabled +49:23.873 [DEBUG] TestAgent.dns: recursor enabled +49:23.873 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23652 network=udp +49:23.873 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23652 network=tcp +49:23.874 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23653 network=tcp +49:23.874 [INFO] TestAgent: started state syncer +49:23.874 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23665 network=tcp +49:23.919 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.919 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23664 [Candidate]" term=2 +49:23.919 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.919 [DEBUG] TestAgent.server.raft: vote granted: from=780d4671-b1b8-1150-6ad8-d0c7b20c0c48 term=2 tally=1 +49:23.919 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.919 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23664 [Leader]" +49:23.919 [INFO] TestAgent.server: cluster leadership acquired +49:23.919 [INFO] TestAgent.server: New leader elected: payload=Node-780d4671-b1b8-1150-6ad8-d0c7b20c0c48 +49:23.919 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23664 +49:23.920 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.921 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.921 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.921 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.921 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.921 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.921 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-780d4671-b1b8-1150-6ad8-d0c7b20c0c48 +49:23.921 [INFO] TestAgent.server: member joined, marking health alive: member=Node-780d4671-b1b8-1150-6ad8-d0c7b20c0c48 +49:23.922 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.071 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.071 [INFO] TestAgent: Synced node info +49:24.071 [DEBUG] TestAgent: Node info in sync +49:24.129 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=ANY class=IN latency=181.501µs client=127.0.0.1:60263 client_network=udp +49:24.130 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=ANY class=IN latency=73.392µs client=127.0.0.1:42622 client_network=udp +49:24.130 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=149.138µs +49:24.130 [DEBUG] TestAgent.dns: request served from client: name=google.node.consul. type=ANY class=IN latency=336.979µs client=127.0.0.1:39615 client_network=udp +49:24.130 [INFO] TestAgent: Requesting shutdown +49:24.130 [INFO] TestAgent.server: shutting down server +49:24.130 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.130 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.130 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.130 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.130 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.131 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.131 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.131 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.131 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.131 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.131 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.131Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.131Z [INFO] agent.router.manager: shutting down +49:24.131 [INFO] TestAgent: consul server down +49:24.131 [INFO] TestAgent: shutdown complete +49:24.131 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23652 network=tcp +49:24.131 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23652 network=udp +49:24.131 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23653 network=tcp +49:24.631 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.631 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_TTL (0.79s) +=== CONT TestDNS_ServiceLookup_LargeResponses +2022-12-04T05:49:23.777Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.789Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.791 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0ad2598a-633d-e623-a739-7878f7acd280 Address:127.0.0.1:23678}]" +49:23.791 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23678 [Follower]" leader= +49:23.791 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0ad2598a-633d-e623-a739-7878f7acd280.dc1 127.0.0.1 +49:23.792 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0ad2598a-633d-e623-a739-7878f7acd280 127.0.0.1 +2022-12-04T05:49:23.792Z [INFO] agent.router: Initializing LAN area manager +49:23.792 [INFO] TestAgent.server: Adding LAN server: server="Node-0ad2598a-633d-e623-a739-7878f7acd280 (Addr: tcp/127.0.0.1:23678) (DC: dc1)" +49:23.792 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0ad2598a-633d-e623-a739-7878f7acd280.dc1 area=wan +49:23.792 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23624 network=udp +49:23.792 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23624 network=tcp +49:23.793 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23625 network=tcp +49:23.793 [INFO] TestAgent: started state syncer +49:23.793 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23679 network=tcp +49:23.855 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.855 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23678 [Candidate]" term=2 +49:23.856 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.856 [DEBUG] TestAgent.server.raft: vote granted: from=0ad2598a-633d-e623-a739-7878f7acd280 term=2 tally=1 +49:23.856 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.856 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23678 [Leader]" +49:23.856 [INFO] TestAgent.server: cluster leadership acquired +49:23.856 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23678 +49:23.857 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.857 [INFO] TestAgent.server: New leader elected: payload=Node-0ad2598a-633d-e623-a739-7878f7acd280 +49:23.858 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.858 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.858 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.858 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.858 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.858 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0ad2598a-633d-e623-a739-7878f7acd280 +49:23.858 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0ad2598a-633d-e623-a739-7878f7acd280 +49:23.858 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.151 [DEBUG] TestAgent.dns: request served from client: name=webv4.service.consul. type=AAAA class=IN latency=177.096µs client=127.0.0.1:43427 client_network=udp +49:24.151 [DEBUG] TestAgent.dns: request served from client: name=webv4.query.consul. type=AAAA class=IN latency=58.84µs client=127.0.0.1:50267 client_network=udp +49:24.151 [DEBUG] TestAgent.dns: request served from client: name=webv6.service.consul. type=A class=IN latency=51.843µs client=127.0.0.1:56705 client_network=udp +49:24.151 [DEBUG] TestAgent.dns: request served from client: name=webv6.query.consul. type=A class=IN latency=57.192µs client=127.0.0.1:51292 client_network=udp +49:24.151 [INFO] TestAgent: Requesting shutdown +49:24.151 [INFO] TestAgent.server: shutting down server +49:24.151 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.151 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.151 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.152 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.152 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.152 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.152 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.152 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:24.152 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.152 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.152 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.152 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.152Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.152Z [INFO] agent.router.manager: shutting down +49:24.152 [INFO] TestAgent: consul server down +49:24.152 [INFO] TestAgent: shutdown complete +49:24.152 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23624 network=tcp +49:24.152 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23624 network=udp +49:24.152 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23625 network=tcp +49:24.652 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.652 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NonExistingLookupEmptyAorAAAA (0.89s) +=== CONT TestDNS_ServiceLookup_PreparedQueryNamePeriod +2022-12-04T05:49:23.841Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.857Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.859 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4c6c9ce9-131e-3fba-faeb-a3d1b557c4cc Address:127.0.0.1:23636}]" +49:23.859 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23636 [Follower]" leader= +49:23.860 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4c6c9ce9-131e-3fba-faeb-a3d1b557c4cc.dc1 127.0.0.1 +49:23.860 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4c6c9ce9-131e-3fba-faeb-a3d1b557c4cc 127.0.0.1 +2022-12-04T05:49:23.860Z [INFO] agent.router: Initializing LAN area manager +49:23.861 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23666 network=udp +49:23.861 [INFO] TestAgent.server: Adding LAN server: server="Node-4c6c9ce9-131e-3fba-faeb-a3d1b557c4cc (Addr: tcp/127.0.0.1:23636) (DC: dc1)" +49:23.861 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4c6c9ce9-131e-3fba-faeb-a3d1b557c4cc.dc1 area=wan +49:23.861 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23666 network=tcp +49:23.861 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23667 network=tcp +49:23.861 [INFO] TestAgent: started state syncer +49:23.862 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23637 network=tcp +49:23.899 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.899 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23636 [Candidate]" term=2 +49:23.900 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.900 [DEBUG] TestAgent.server.raft: vote granted: from=4c6c9ce9-131e-3fba-faeb-a3d1b557c4cc term=2 tally=1 +49:23.900 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.900 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23636 [Leader]" +49:23.900 [INFO] TestAgent.server: cluster leadership acquired +49:23.900 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23636 +49:23.900 [INFO] TestAgent.server: New leader elected: payload=Node-4c6c9ce9-131e-3fba-faeb-a3d1b557c4cc +49:23.900 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.902 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.902 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.902 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.902 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.902 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.902 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4c6c9ce9-131e-3fba-faeb-a3d1b557c4cc +49:23.902 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4c6c9ce9-131e-3fba-faeb-a3d1b557c4cc +49:23.903 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.067 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.067 [INFO] TestAgent: Synced node info +49:24.067 [DEBUG] TestAgent: Node info in sync +49:24.195 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=242.353µs client=127.0.0.1:36455 client_network=udp +49:24.195 [DEBUG] TestAgent.dns: request served from client: name=dblb.service.consul. type=SRV class=IN latency=69.162µs client=127.0.0.1:56667 client_network=udp +49:24.195 [DEBUG] TestAgent.dns: request served from client: name=dk.service.consul. type=SRV class=IN latency=72.82µs client=127.0.0.1:49844 client_network=udp +49:24.196 [DEBUG] TestAgent.dns: request served from client: name=api.service.consul. type=SRV class=IN latency=216.241µs client=127.0.0.1:37712 client_network=udp +49:24.196 [INFO] TestAgent: Requesting shutdown +49:24.196 [INFO] TestAgent.server: shutting down server +49:24.196 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.196 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.196 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.196 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.196 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.196 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.196 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.196 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.196 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.196 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.196 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.196Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.196Z [INFO] agent.router.manager: shutting down +49:24.196 [INFO] TestAgent: consul server down +49:24.196 [INFO] TestAgent: shutdown complete +49:24.196 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23666 network=tcp +49:24.196 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23666 network=udp +49:24.196 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23667 network=tcp +49:24.696 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.696 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_TTL (0.87s) + --- PASS: TestDNS_ServiceLookup_TTL/db.service.consul. (0.00s) + --- PASS: TestDNS_ServiceLookup_TTL/dblb.service.consul. (0.00s) + --- PASS: TestDNS_ServiceLookup_TTL/dk.service.consul. (0.00s) + --- PASS: TestDNS_ServiceLookup_TTL/api.service.consul. (0.00s) === CONT TestDNS_ServiceLookup_FilterCritical -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:11.471Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:11.516Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:11.532 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9bb42232-c52b-3c3a-b9e6-3339621bfe1d Address:127.0.0.1:16896}]" -13:11.532 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9bb42232-c52b-3c3a-b9e6-3339621bfe1d.dc1 127.0.0.1 -13:11.533 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9bb42232-c52b-3c3a-b9e6-3339621bfe1d 127.0.0.1 -2021-10-31T23:13:11.533Z [INFO] agent.router: Initializing LAN area manager -13:11.533 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16891 network=udp -13:11.533 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16896 [Follower]" leader= -13:11.533 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9bb42232-c52b-3c3a-b9e6-3339621bfe1d.dc1 area=wan -13:11.533 [INFO] TestAgent.server: Adding LAN server: server="Node-9bb42232-c52b-3c3a-b9e6-3339621bfe1d (Addr: tcp/127.0.0.1:16896) (DC: dc1)" -13:11.534 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16891 network=tcp -13:11.534 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16892 network=tcp -13:11.534 [INFO] TestAgent: started state syncer -13:11.534 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16897 network=tcp -13:11.589 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:11.590 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16896 [Candidate]" term=2 -13:11.590 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:11.590 [DEBUG] TestAgent.server.raft: vote granted: from=9bb42232-c52b-3c3a-b9e6-3339621bfe1d term=2 tally=1 -13:11.590 [INFO] TestAgent.server.raft: election won: tally=1 -13:11.590 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16896 [Leader]" -13:11.590 [INFO] TestAgent.server: cluster leadership acquired -13:11.590 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16896 -13:11.591 [INFO] TestAgent.server: New leader elected: payload=Node-9bb42232-c52b-3c3a-b9e6-3339621bfe1d -13:11.591 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.593 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.593 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.593 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.593 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.593 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.593 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9bb42232-c52b-3c3a-b9e6-3339621bfe1d -13:11.593 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9bb42232-c52b-3c3a-b9e6-3339621bfe1d -13:11.594 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.619 [DEBUG] TestAgent.dns: request served from client: name=7f000001.addr.dc1.consul. type=SRV class=IN latency=97.062µs client=127.0.0.1:41314 client_network=udp -13:11.619 [INFO] TestAgent: Requesting shutdown -13:11.619 [INFO] TestAgent.server: shutting down server -13:11.619 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:11.619 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:11.619 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:11.619 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:11.619 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:11.619 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:11.619 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:11.619 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:11.619 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:11.619 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:11.619 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:11.619 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:11.619Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:11.619Z [INFO] agent.router.manager: shutting down -13:11.619 [INFO] TestAgent: consul server down -13:11.619 [INFO] TestAgent: shutdown complete -13:11.619 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16891 network=tcp -13:11.619 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16891 network=udp -13:11.620 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16892 network=tcp -13:12.121 [INFO] TestAgent: Waiting for endpoints to shut down -13:12.166 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_AddressLookup (0.72s) -=== CONT TestDNS_ServiceLookup_TTL -2021-10-31T23:13:11.483Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:11.515Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:11.516 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:da92a5ea-265b-7573-4d52-dd78ec2051ab Address:127.0.0.1:16903}]" -13:11.517 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-da92a5ea-265b-7573-4d52-dd78ec2051ab.dc1 127.0.0.1 -13:11.517 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16903 [Follower]" leader= -13:11.522 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-da92a5ea-265b-7573-4d52-dd78ec2051ab 127.0.0.1 -2021-10-31T23:13:11.522Z [INFO] agent.router: Initializing LAN area manager -13:11.522 [DEBUG] TestAgent.dns: recursor enabled -13:11.522 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16898 network=udp -13:11.522 [INFO] TestAgent.server: Adding LAN server: server="Node-da92a5ea-265b-7573-4d52-dd78ec2051ab (Addr: tcp/127.0.0.1:16903) (DC: dc1)" -13:11.522 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-da92a5ea-265b-7573-4d52-dd78ec2051ab.dc1 area=wan -13:11.522 [DEBUG] TestAgent.dns: recursor enabled -13:11.523 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16898 network=tcp -13:11.523 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16899 network=tcp -13:11.523 [INFO] TestAgent: started state syncer -13:11.523 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16911 network=tcp -13:11.577 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:11.578 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16903 [Candidate]" term=2 -13:11.578 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:11.578 [DEBUG] TestAgent.server.raft: vote granted: from=da92a5ea-265b-7573-4d52-dd78ec2051ab term=2 tally=1 -13:11.578 [INFO] TestAgent.server.raft: election won: tally=1 -13:11.578 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16903 [Leader]" -13:11.578 [INFO] TestAgent.server: cluster leadership acquired -13:11.578 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16903 -13:11.579 [INFO] TestAgent.server: New leader elected: payload=Node-da92a5ea-265b-7573-4d52-dd78ec2051ab -13:11.579 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.580 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.580 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.580 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.580 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.580 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.580 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-da92a5ea-265b-7573-4d52-dd78ec2051ab -13:11.580 [INFO] TestAgent.server: member joined, marking health alive: member=Node-da92a5ea-265b-7573-4d52-dd78ec2051ab -13:11.581 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.687 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=187.676µs -13:11.687 [DEBUG] TestAgent.dns: request served from client: name=search.service.consul. type=ANY class=IN latency=474.258µs client=127.0.0.1:43629 client_network=udp -13:11.687 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=77.018µs -13:11.687 [DEBUG] TestAgent.dns: request served from client: name=019d6445-ee9b-d557-e60e-661cf3c38eac.query.consul. type=ANY class=IN latency=436.581µs client=127.0.0.1:60932 client_network=udp -13:11.687 [INFO] TestAgent: Requesting shutdown -13:11.688 [INFO] TestAgent.server: shutting down server -13:11.688 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:11.688 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:11.688 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:11.688 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:11.688 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:11.688 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:11.688 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:11.688 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:11.689 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:11.689 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:11.689 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:11.689 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:11.689Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:11.689Z [INFO] agent.router.manager: shutting down -13:11.690 [INFO] TestAgent: consul server down -13:11.690 [INFO] TestAgent: shutdown complete -13:11.690 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16898 network=tcp -13:11.690 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16898 network=udp -13:11.690 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16899 network=tcp -13:12.190 [INFO] TestAgent: Waiting for endpoints to shut down -13:12.190 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_ServiceAddress_CNAME (0.73s) -=== CONT TestDNS_RecursorTimeout +2022-12-04T05:49:23.763Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:23.774Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:23.776 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4a7424f3-b5c1-61cc-4b03-06421ce44fec Address:127.0.0.1:23608}]" +49:23.776 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23608 [Follower]" leader= +49:23.777 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4a7424f3-b5c1-61cc-4b03-06421ce44fec.dc1 127.0.0.1 +49:23.777 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4a7424f3-b5c1-61cc-4b03-06421ce44fec 127.0.0.1 +2022-12-04T05:49:23.777Z [INFO] agent.router: Initializing LAN area manager +49:23.777 [INFO] TestAgent.server: Adding LAN server: server="Node-4a7424f3-b5c1-61cc-4b03-06421ce44fec (Addr: tcp/127.0.0.1:23608) (DC: dc1)" +49:23.778 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4a7424f3-b5c1-61cc-4b03-06421ce44fec.dc1 area=wan +49:23.778 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23603 network=tcp +49:23.778 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23603 network=udp +49:23.778 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23604 network=tcp +49:23.778 [INFO] TestAgent: started state syncer +49:23.778 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23609 network=tcp +49:23.829 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:23.829 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23608 [Candidate]" term=2 +49:23.829 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:23.829 [DEBUG] TestAgent.server.raft: vote granted: from=4a7424f3-b5c1-61cc-4b03-06421ce44fec term=2 tally=1 +49:23.829 [INFO] TestAgent.server.raft: election won: tally=1 +49:23.829 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23608 [Leader]" +49:23.830 [INFO] TestAgent.server: cluster leadership acquired +49:23.830 [INFO] TestAgent.server: New leader elected: payload=Node-4a7424f3-b5c1-61cc-4b03-06421ce44fec +49:23.830 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23608 +49:23.834 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:23.835 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:23.835 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:23.835 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:23.835 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:23.835 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:23.835 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4a7424f3-b5c1-61cc-4b03-06421ce44fec +49:23.835 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4a7424f3-b5c1-61cc-4b03-06421ce44fec +49:23.836 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.146 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.146 [INFO] TestAgent: Synced node info +49:24.215 [DEBUG] TestAgent.dns: request served from client: name=7f000001.addr.dc1.consul. type=SRV class=IN latency=115.767µs client=127.0.0.1:42033 client_network=udp +49:24.215 [INFO] TestAgent: Requesting shutdown +49:24.215 [INFO] TestAgent.server: shutting down server +49:24.215 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.215 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.215 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.215 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.215 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.215 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.215 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.215 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.216 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.216 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.216 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.216Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.216Z [INFO] agent.router.manager: shutting down +49:24.216 [INFO] TestAgent: consul server down +49:24.216 [INFO] TestAgent: shutdown complete +49:24.216 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23603 network=tcp +49:24.216 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23603 network=udp +49:24.216 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23604 network=tcp +49:24.716 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.716 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_AddressLookup (0.96s) +=== CONT TestDNS_ServiceLookup_Truncate +2022-12-04T05:49:24.102Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.112Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.113 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8dd2d32b-0387-6165-6862-be418dae2753 Address:127.0.0.1:23650}]" +49:24.113 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23650 [Follower]" leader= +49:24.114 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8dd2d32b-0387-6165-6862-be418dae2753.dc1 127.0.0.1 +49:24.114 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8dd2d32b-0387-6165-6862-be418dae2753 127.0.0.1 +2022-12-04T05:49:24.114Z [INFO] agent.router: Initializing LAN area manager +49:24.114 [DEBUG] TestAgent.dns: recursor enabled +49:24.114 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8dd2d32b-0387-6165-6862-be418dae2753.dc1 area=wan +49:24.114 [INFO] TestAgent.server: Adding LAN server: server="Node-8dd2d32b-0387-6165-6862-be418dae2753 (Addr: tcp/127.0.0.1:23650) (DC: dc1)" +49:24.114 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23645 network=udp +49:24.115 [DEBUG] TestAgent.dns: recursor enabled +49:24.115 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23645 network=tcp +49:24.115 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23646 network=tcp +49:24.115 [INFO] TestAgent: started state syncer +49:24.115 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23651 network=tcp +49:24.170 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.170 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23650 [Candidate]" term=2 +49:24.170 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.170 [DEBUG] TestAgent.server.raft: vote granted: from=8dd2d32b-0387-6165-6862-be418dae2753 term=2 tally=1 +49:24.170 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.170 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23650 [Leader]" +49:24.170 [INFO] TestAgent.server: cluster leadership acquired +49:24.170 [INFO] TestAgent.server: New leader elected: payload=Node-8dd2d32b-0387-6165-6862-be418dae2753 +49:24.171 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23650 +49:24.171 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.173 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.173 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.173 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.173 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.173 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.173 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8dd2d32b-0387-6165-6862-be418dae2753 +49:24.173 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8dd2d32b-0387-6165-6862-be418dae2753 +49:24.173 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.201 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.201 [INFO] TestAgent: Synced node info +49:24.201 [DEBUG] TestAgent: Node info in sync +49:24.223 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=169.901µs +49:24.223 [DEBUG] TestAgent.dns: request served from client: name=search.service.consul. type=ANY class=IN latency=463.494µs client=127.0.0.1:39906 client_network=udp +49:24.223 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=185.212µs +49:24.223 [DEBUG] TestAgent.dns: request served from client: name=0791619a-038f-8fbf-41ff-89746ed66306.query.consul. type=ANY class=IN latency=339.278µs client=127.0.0.1:54733 client_network=udp +49:24.223 [INFO] TestAgent: Requesting shutdown +49:24.223 [INFO] TestAgent.server: shutting down server +49:24.223 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.223 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.223 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.223 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.223 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.223 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.223 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.223 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.223 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.223 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:24.223 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:24.224Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.224Z [INFO] agent.router.manager: shutting down +49:24.224 [INFO] TestAgent: consul server down +49:24.224 [INFO] TestAgent: shutdown complete +49:24.224 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23645 network=tcp +49:24.224 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23645 network=udp +49:24.224 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23646 network=tcp +49:24.724 [INFO] TestAgent: Waiting for endpoints to shut down +49:24.724 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_ServiceAddress_CNAME (0.63s) +=== CONT TestDNS_PreparedQueryNearIP +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestDNS_PreparedQuery_TTL/db.query.consul. === RUN TestDNS_PreparedQuery_TTL/db-ttl.query.consul. === RUN TestDNS_PreparedQuery_TTL/dblb.query.consul. @@ -28511,402 +34608,459 @@ === RUN TestDNS_PreparedQuery_TTL/dk-ttl.query.consul. === RUN TestDNS_PreparedQuery_TTL/api.query.consul. === RUN TestDNS_PreparedQuery_TTL/api-ttl.query.consul. -=== RUN TestDNS_ServiceLookup_TTL/db.service.consul. -=== RUN TestDNS_ServiceLookup_TTL/dblb.service.consul. -=== RUN TestDNS_ServiceLookup_TTL/dk.service.consul. -=== RUN TestDNS_ServiceLookup_TTL/api.service.consul. +Added 3 service nodes +2022-12-04T05:49:24.424Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:24.424Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.440Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:24.440Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.441 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ed6f7806-6aeb-6c22-3347-a7048c5942f4 Address:127.0.0.1:23727}]" +49:24.442 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23727 [Follower]" leader= +49:24.442 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ed6f7806-6aeb-6c22-3347-a7048c5942f4.dc1 127.0.0.1 +49:24.442 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ed6f7806-6aeb-6c22-3347-a7048c5942f4 127.0.0.1 +2022-12-04T05:49:24.442Z [INFO] agent.router: Initializing LAN area manager +49:24.443 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ed6f7806-6aeb-6c22-3347-a7048c5942f4.dc1 area=wan +49:24.443 [INFO] TestAgent.server: Adding LAN server: server="Node-ed6f7806-6aeb-6c22-3347-a7048c5942f4 (Addr: tcp/127.0.0.1:23727) (DC: dc1)" +49:24.443 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23673 network=udp +49:24.443 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23673 network=tcp +49:24.443 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23674 network=tcp +49:24.443 [INFO] TestAgent: started state syncer +49:24.443 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23728 network=tcp +49:24.480 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.480 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23727 [Candidate]" term=2 +49:24.480 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.480 [DEBUG] TestAgent.server.raft: vote granted: from=ed6f7806-6aeb-6c22-3347-a7048c5942f4 term=2 tally=1 +49:24.480 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.480 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23727 [Leader]" +49:24.480 [INFO] TestAgent.server: cluster leadership acquired +49:24.480 [INFO] TestAgent.server: initializing acls +49:24.480 [INFO] TestAgent.server: New leader elected: payload=Node-ed6f7806-6aeb-6c22-3347-a7048c5942f4 +49:24.481 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:24.481 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:24.481 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:24.481 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:24.481 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:24.481 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:24.481 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ed6f7806-6aeb-6c22-3347-a7048c5942f4 +49:24.481 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ed6f7806-6aeb-6c22-3347-a7048c5942f4.dc1 +49:24.481 [INFO] TestAgent.server: Updating LAN server: server="Node-ed6f7806-6aeb-6c22-3347-a7048c5942f4 (Addr: tcp/127.0.0.1:23727) (DC: dc1)" +49:24.482 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ed6f7806-6aeb-6c22-3347-a7048c5942f4.dc1 area=wan +49:24.482 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.483 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.483 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.483 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.483 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.483 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.483 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ed6f7806-6aeb-6c22-3347-a7048c5942f4 +49:24.483 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ed6f7806-6aeb-6c22-3347-a7048c5942f4 +49:24.484 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ed6f7806-6aeb-6c22-3347-a7048c5942f4 +49:24.484 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.542 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ed6f7806-6aeb-6c22-3347-a7048c5942f4 +49:24.542 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ed6f7806-6aeb-6c22-3347-a7048c5942f4 +49:24.544 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=foo +49:24.544 [DEBUG] TestAgent.dns: request served from client: name=foo.service.consul. type=A class=IN latency=223.028µs client=127.0.0.1:34877 client_network=udp +49:24.544 [INFO] TestAgent: Requesting shutdown +49:24.544 [INFO] TestAgent.server: shutting down server +49:24.544 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.544 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.544 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.544 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.544 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:24.544 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:24.544 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.544 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.544 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.544 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.544 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:24.544 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.544 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:24.544 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.544 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:24.544 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.545Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.545Z [INFO] agent.router.manager: shutting down +49:24.545 [INFO] TestAgent: consul server down +49:24.545 [INFO] TestAgent: shutdown complete +49:24.545 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23673 network=tcp +49:24.545 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23673 network=udp +49:24.545 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23674 network=tcp +49:25.046 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.046 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_FilterACL (1.25s) + --- PASS: TestDNS_ServiceLookup_FilterACL/ACLToken_==_root (0.61s) + --- PASS: TestDNS_ServiceLookup_FilterACL/ACLToken_==_anonymous (0.64s) +=== CONT TestDNS_TCP_and_UDP_Truncate +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:24.250Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.262Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.263 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e52aecda-1896-956a-1f26-658a7f265b19 Address:127.0.0.1:23489}]" +49:24.263 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23489 [Follower]" leader= +49:24.264 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e52aecda-1896-956a-1f26-658a7f265b19.dc1 127.0.0.1 +49:24.264 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e52aecda-1896-956a-1f26-658a7f265b19 127.0.0.1 +2022-12-04T05:49:24.264Z [INFO] agent.router: Initializing LAN area manager +49:24.264 [INFO] TestAgent.server: Adding LAN server: server="Node-e52aecda-1896-956a-1f26-658a7f265b19 (Addr: tcp/127.0.0.1:23489) (DC: dc1)" +49:24.265 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e52aecda-1896-956a-1f26-658a7f265b19.dc1 area=wan +49:24.265 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23694 network=udp +49:24.265 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23694 network=tcp +49:24.265 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23695 network=tcp +49:24.265 [INFO] TestAgent: started state syncer +49:24.265 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23490 network=tcp +49:24.308 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.308 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23489 [Candidate]" term=2 +49:24.308 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.308 [DEBUG] TestAgent.server.raft: vote granted: from=e52aecda-1896-956a-1f26-658a7f265b19 term=2 tally=1 +49:24.308 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.308 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23489 [Leader]" +49:24.308 [INFO] TestAgent.server: cluster leadership acquired +49:24.309 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23489 +49:24.310 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.312 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.312 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.312 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.312 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.312 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.312 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e52aecda-1896-956a-1f26-658a7f265b19 +49:24.312 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e52aecda-1896-956a-1f26-658a7f265b19 +49:24.312 [INFO] TestAgent.server: New leader elected: payload=Node-e52aecda-1896-956a-1f26-658a7f265b19 +49:24.312 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.542 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.543 [INFO] TestAgent: Synced node info +49:24.621 [DEBUG] TestAgent.dns: request served from client: name=_db._primary.service.dc1.consul. type=SRV class=IN latency=100.241µs client=127.0.0.1:57799 client_network=udp +49:24.621 [DEBUG] TestAgent.dns: request served from client: name=_db._primary.service.consul. type=SRV class=IN latency=61.952µs client=127.0.0.1:39917 client_network=udp +49:24.621 [DEBUG] TestAgent.dns: request served from client: name=_db._primary.dc1.consul. type=SRV class=IN latency=112.058µs client=127.0.0.1:34195 client_network=udp +49:24.621 [DEBUG] TestAgent.dns: request served from client: name=_db._primary.consul. type=SRV class=IN latency=58.441µs client=127.0.0.1:54576 client_network=udp +49:24.621 [INFO] TestAgent: Requesting shutdown +49:24.621 [INFO] TestAgent.server: shutting down server +49:24.621 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.621 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.621 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.621 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.621 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.622 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.622 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.622 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.622 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.622 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.622 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.622Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.622Z [INFO] agent.router.manager: shutting down +49:24.622 [INFO] TestAgent: consul server down +49:24.622 [INFO] TestAgent: shutdown complete +49:24.622 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23694 network=tcp +49:24.622 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23694 network=udp +49:24.622 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23695 network=tcp +49:25.122 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.122 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_SRV_RFC (0.88s) +=== CONT TestDNS_PreparedQueryNearIPEDNS +2022-12-04T05:49:24.531Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.544Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.546 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4ea0e34f-83a5-a6b1-4ded-6f1a09c65e8c Address:127.0.0.1:23720}]" +49:24.546 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23720 [Follower]" leader= +49:24.547 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4ea0e34f-83a5-a6b1-4ded-6f1a09c65e8c.dc1 127.0.0.1 +49:24.547 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4ea0e34f-83a5-a6b1-4ded-6f1a09c65e8c 127.0.0.1 +2022-12-04T05:49:24.547Z [INFO] agent.router: Initializing LAN area manager +49:24.547 [DEBUG] TestAgent.dns: recursor enabled +49:24.547 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23701 network=udp +49:24.548 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4ea0e34f-83a5-a6b1-4ded-6f1a09c65e8c.dc1 area=wan +49:24.548 [INFO] TestAgent.server: Adding LAN server: server="Node-4ea0e34f-83a5-a6b1-4ded-6f1a09c65e8c (Addr: tcp/127.0.0.1:23720) (DC: dc1)" +49:24.548 [DEBUG] TestAgent.dns: recursor enabled +49:24.548 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23701 network=tcp +49:24.548 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23702 network=tcp +49:24.548 [INFO] TestAgent: started state syncer +49:24.548 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23721 network=tcp +49:24.604 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.604 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23720 [Candidate]" term=2 +49:24.604 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.604 [DEBUG] TestAgent.server.raft: vote granted: from=4ea0e34f-83a5-a6b1-4ded-6f1a09c65e8c term=2 tally=1 +49:24.604 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.604 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23720 [Leader]" +49:24.604 [INFO] TestAgent.server: cluster leadership acquired +49:24.604 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23720 +49:24.605 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.606 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.606 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.606 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.606 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.606 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.606 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4ea0e34f-83a5-a6b1-4ded-6f1a09c65e8c +49:24.606 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4ea0e34f-83a5-a6b1-4ded-6f1a09c65e8c +49:24.606 [INFO] TestAgent.server: New leader elected: payload=Node-4ea0e34f-83a5-a6b1-4ded-6f1a09c65e8c +49:24.606 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.679 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.679 [INFO] TestAgent: Synced node info +49:24.704 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{apple.com. 255 1}" rtt=174.531µs recursor=127.0.0.1:45704 +49:24.704 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=342.502µs client=127.0.0.1:40047 client_network=udp +49:24.704 [INFO] TestAgent: Requesting shutdown +49:24.704 [INFO] TestAgent.server: shutting down server +49:24.704 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.704 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.704 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.704 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.704 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.704 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.704 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.704 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.704 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.704 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.704 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.704Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.704Z [INFO] agent.router.manager: shutting down +49:24.704 [INFO] TestAgent: consul server down +49:24.704 [INFO] TestAgent: shutdown complete +49:24.704 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23701 network=tcp +49:24.704 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23701 network=udp +49:24.704 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23702 network=tcp +49:25.207 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.207 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_Recurse (0.69s) +=== CONT TestDNS_RecursorTimeout [WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:11.295Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:11.381Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:11.399 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6d955fd9-e40f-f390-75fe-dbc04bb15a92 Address:127.0.0.1:16882}]" -13:11.400 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6d955fd9-e40f-f390-75fe-dbc04bb15a92.dc1 127.0.0.1 -13:11.400 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6d955fd9-e40f-f390-75fe-dbc04bb15a92 127.0.0.1 -2021-10-31T23:13:11.400Z [INFO] agent.router: Initializing LAN area manager -13:11.400 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16877 network=udp -13:11.400 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16882 [Follower]" leader= -13:11.401 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6d955fd9-e40f-f390-75fe-dbc04bb15a92.dc1 area=wan -13:11.401 [INFO] TestAgent.server: Adding LAN server: server="Node-6d955fd9-e40f-f390-75fe-dbc04bb15a92 (Addr: tcp/127.0.0.1:16882) (DC: dc1)" -13:11.401 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16877 network=tcp -13:11.401 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16878 network=tcp -13:11.401 [INFO] TestAgent: started state syncer -13:11.401 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16883 network=tcp -13:11.449 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:11.449 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16882 [Candidate]" term=2 -13:11.450 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:11.450 [DEBUG] TestAgent.server.raft: vote granted: from=6d955fd9-e40f-f390-75fe-dbc04bb15a92 term=2 tally=1 -13:11.450 [INFO] TestAgent.server.raft: election won: tally=1 -13:11.450 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16882 [Leader]" -13:11.450 [INFO] TestAgent.server: cluster leadership acquired -13:11.450 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16882 -13:11.451 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.452 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.452 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.452 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.452 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.452 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.453 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6d955fd9-e40f-f390-75fe-dbc04bb15a92 -13:11.453 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6d955fd9-e40f-f390-75fe-dbc04bb15a92 -13:11.453 [INFO] TestAgent.server: New leader elected: payload=Node-6d955fd9-e40f-f390-75fe-dbc04bb15a92 -13:11.456 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.672 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:11.687 [INFO] TestAgent: Synced node info -13:11.687 [DEBUG] TestAgent: Node info in sync -13:11.877 [DEBUG] TestAgent.dns: request served from client: name=_db._primary.service.dc1.consul. type=SRV class=IN latency=127.16µs client=127.0.0.1:57459 client_network=udp -13:11.877 [DEBUG] TestAgent.dns: request served from client: name=_db._primary.service.consul. type=SRV class=IN latency=70.811µs client=127.0.0.1:49933 client_network=udp -13:11.877 [DEBUG] TestAgent.dns: request served from client: name=_db._primary.dc1.consul. type=SRV class=IN latency=79.854µs client=127.0.0.1:47078 client_network=udp -13:11.878 [DEBUG] TestAgent.dns: request served from client: name=_db._primary.consul. type=SRV class=IN latency=221.082µs client=127.0.0.1:50844 client_network=udp -13:11.897 [INFO] TestAgent: Requesting shutdown -13:11.898 [INFO] TestAgent.server: shutting down server -13:11.898 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:11.898 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:11.898 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:11.898 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:11.898 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:11.898 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:11.898 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:11.898 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:11.898 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:11.898 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:11.898 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:11.898Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:11.898Z [INFO] agent.router.manager: shutting down -13:11.898 [INFO] TestAgent: consul server down -13:11.898 [INFO] TestAgent: shutdown complete -13:11.898 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16877 network=tcp -13:11.898 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16877 network=udp -13:11.898 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16878 network=tcp -13:12.401 [INFO] TestAgent: Waiting for endpoints to shut down -13:12.401 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_SRV_RFC (1.15s) -=== CONT TestDNS_ServiceLookup_OnlyFailing -13:11.450 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 -2021-10-31T23:13:11.450Z [INFO] agent.router: Initializing LAN area manager -13:11.450 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16884 network=udp -13:11.450 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16884 network=tcp -13:11.451 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16885 network=tcp -13:11.451 [INFO] TestAgent: started state syncer -13:11.452 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:16845] -13:11.452 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16890 network=tcp -2021-10-31T23:13:11.452Z [WARN] agent.router.manager: No servers available -13:11.452 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -13:11.452 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:16845 -13:11.454 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: test-server 127.0.0.1 -13:11.454 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -13:11.454 [DEBUG] TestAgent: systemd notify failed: error="No socket" -2021-10-31T23:13:11.454Z [WARN] agent.router.manager: No servers available -13:11.454 [INFO] TestAgent.client: adding server: server="test-server (Addr: tcp/127.0.0.1:16847) (DC: dc1)" -13:11.495 [ERROR] TestAgent.client: RPC failed to server: method=Health.ServiceNodes server=127.0.0.1:16847 error="rpc error making call: No path to datacenter" -13:11.495 [ERROR] TestAgent.dns: rpc error: error="rpc error making call: No path to datacenter" -13:11.495 [DEBUG] TestAgent.dns: request served from client: name=consul.service.dc2.consul. type=ANY class=IN latency=288.034µs client=127.0.0.1:33333 client_network=udp -13:11.496 [INFO] TestAgent: Requesting shutdown -13:11.496 [INFO] TestAgent.client: shutting down client -13:11.496 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -13:11.496 [INFO] TestAgent: consul client down -13:11.496 [INFO] TestAgent: shutdown complete -13:11.496 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16884 network=tcp -13:11.496 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16884 network=udp -13:11.496 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16885 network=tcp -13:12.001 [INFO] TestAgent: Waiting for endpoints to shut down -13:12.002 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:10.992Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:11.096Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:11.110 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:66e9a60f-de36-9e7e-d35e-1e12810e8f06 Address:127.0.0.1:16847}]" -13:11.110 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16847 [Follower]" leader= -13:11.111 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-server.dc1 127.0.0.1 -13:11.111 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-server 127.0.0.1 -2021-10-31T23:13:11.111Z [INFO] agent.router: Initializing LAN area manager -13:11.111 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16842 network=udp -13:11.112 [INFO] TestAgent.server: Adding LAN server: server="test-server (Addr: tcp/127.0.0.1:16847) (DC: dc1)" -13:11.112 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-server.dc1 area=wan -13:11.112 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16842 network=tcp -13:11.112 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16843 network=tcp -13:11.112 [INFO] TestAgent: started state syncer -13:11.113 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16855 network=tcp -13:11.172 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:11.172 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16847 [Candidate]" term=2 -13:11.172 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:11.172 [DEBUG] TestAgent.server.raft: vote granted: from=66e9a60f-de36-9e7e-d35e-1e12810e8f06 term=2 tally=1 -13:11.172 [INFO] TestAgent.server.raft: election won: tally=1 -13:11.172 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16847 [Leader]" -13:11.172 [INFO] TestAgent.server: cluster leadership acquired -13:11.173 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16847 -13:11.173 [INFO] TestAgent.server: New leader elected: payload=test-server -13:11.177 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.179 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.179 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.179 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.179 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.179 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.179 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-server -13:11.179 [INFO] TestAgent.server: member joined, marking health alive: member=test-server -13:11.180 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.452 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:59532 -13:11.453 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 -13:11.453 [INFO] TestAgent.server: member joined, marking health alive: member=test-client -13:11.485 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:11.486 [INFO] TestAgent: Synced node info -13:11.495 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc2 method=Health.ServiceNodes -13:12.002 [INFO] TestAgent: Requesting shutdown -13:12.002 [INFO] TestAgent.server: shutting down server -13:12.002 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:12.002 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:12.002 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:12.002 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:12.002 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.002 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:12.002 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:12.002 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:12.002 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:12.002 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:12.002 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.002Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.002Z [INFO] agent.router.manager: shutting down -13:12.002 [INFO] TestAgent: consul server down -13:12.002 [INFO] TestAgent: shutdown complete -13:12.002 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16842 network=tcp -13:12.002 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16842 network=udp -13:12.002 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16843 network=tcp -13:12.505 [INFO] TestAgent: Waiting for endpoints to shut down -13:12.505 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NonExistentDC_RPC (1.62s) -=== CONT TestDNS_ServiceLookup_PreparedQueryNamePeriod -2021-10-31T23:13:11.744Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:11.773Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:11.775 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3d514fd6-7c89-3158-c12e-c1092eb1f7b2 Address:127.0.0.1:16931}]" -13:11.776 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3d514fd6-7c89-3158-c12e-c1092eb1f7b2.dc1 127.0.0.1 -13:11.776 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3d514fd6-7c89-3158-c12e-c1092eb1f7b2 127.0.0.1 -2021-10-31T23:13:11.776Z [INFO] agent.router: Initializing LAN area manager -13:11.776 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16926 network=udp -13:11.776 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16931 [Follower]" leader= -13:11.777 [INFO] TestAgent.server: Adding LAN server: server="Node-3d514fd6-7c89-3158-c12e-c1092eb1f7b2 (Addr: tcp/127.0.0.1:16931) (DC: dc1)" -13:11.777 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3d514fd6-7c89-3158-c12e-c1092eb1f7b2.dc1 area=wan -13:11.777 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16926 network=tcp -13:11.783 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16927 network=tcp -13:11.783 [INFO] TestAgent: started state syncer -13:11.784 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16918 network=tcp -13:11.826 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:11.826 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16931 [Candidate]" term=2 -13:11.826 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:11.826 [DEBUG] TestAgent.server.raft: vote granted: from=3d514fd6-7c89-3158-c12e-c1092eb1f7b2 term=2 tally=1 -13:11.826 [INFO] TestAgent.server.raft: election won: tally=1 -13:11.826 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16931 [Leader]" -13:11.826 [INFO] TestAgent.server: cluster leadership acquired -13:11.826 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16931 -13:11.827 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.829 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.829 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.829 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.829 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.829 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.829 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3d514fd6-7c89-3158-c12e-c1092eb1f7b2 -13:11.829 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3d514fd6-7c89-3158-c12e-c1092eb1f7b2 -13:11.829 [INFO] TestAgent.server: New leader elected: payload=Node-3d514fd6-7c89-3158-c12e-c1092eb1f7b2 -13:11.831 [INFO] TestAgent.server: federation state anti-entropy synced -13:12.011 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.631766ms client=127.0.0.1:37649 client_network=udp -13:12.015 [DEBUG] TestAgent.dns: request served from client: name=4d2bf107-ea2a-8e35-a135-4f87694f71ad.query.consul. type=ANY class=IN latency=1.544617ms client=127.0.0.1:60467 client_network=udp -13:12.015 [INFO] TestAgent: Requesting shutdown -13:12.015 [INFO] TestAgent.server: shutting down server -13:12.015 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:12.015 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:12.015 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:12.015 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:12.015 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.015 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:12.015 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:12.016 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:12.016 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:12.016 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:12.016 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:12.016 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.016Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.016Z [INFO] agent.router.manager: shutting down -13:12.016 [INFO] TestAgent: consul server down -13:12.016 [INFO] TestAgent: shutdown complete -13:12.016 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16926 network=tcp -13:12.016 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16926 network=udp -13:12.016 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16927 network=tcp -13:12.517 [INFO] TestAgent: Waiting for endpoints to shut down -13:12.517 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_Truncate (0.90s) -=== CONT TestDNS_PreparedQueryNearIP -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:11.621Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:11.692Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:11.694 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:911c1c64-5b7d-b079-843d-1cda6abac532 Address:127.0.0.1:16910}]" -13:11.695 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16910 [Follower]" leader= -13:11.695 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-911c1c64-5b7d-b079-843d-1cda6abac532.dc1 127.0.0.1 -13:11.696 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-911c1c64-5b7d-b079-843d-1cda6abac532 127.0.0.1 -2021-10-31T23:13:11.696Z [INFO] agent.router: Initializing LAN area manager -13:11.697 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16905 network=udp -13:11.700 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-911c1c64-5b7d-b079-843d-1cda6abac532.dc1 area=wan -13:11.703 [INFO] TestAgent.server: Adding LAN server: server="Node-911c1c64-5b7d-b079-843d-1cda6abac532 (Addr: tcp/127.0.0.1:16910) (DC: dc1)" -13:11.734 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16905 network=tcp -13:11.734 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16906 network=tcp -13:11.734 [INFO] TestAgent: started state syncer -13:11.737 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16925 network=tcp -13:11.741 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:11.742 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16910 [Candidate]" term=2 -13:11.742 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:11.742 [DEBUG] TestAgent.server.raft: vote granted: from=911c1c64-5b7d-b079-843d-1cda6abac532 term=2 tally=1 -13:11.742 [INFO] TestAgent.server.raft: election won: tally=1 -13:11.742 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16910 [Leader]" -13:11.742 [INFO] TestAgent.server: cluster leadership acquired -13:11.742 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16910 -13:11.743 [INFO] TestAgent.server: New leader elected: payload=Node-911c1c64-5b7d-b079-843d-1cda6abac532 -13:11.743 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.745 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.745 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.745 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.745 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.745 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.745 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-911c1c64-5b7d-b079-843d-1cda6abac532 -13:11.745 [INFO] TestAgent.server: member joined, marking health alive: member=Node-911c1c64-5b7d-b079-843d-1cda6abac532 -13:11.746 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.849 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:11.849 [INFO] TestAgent: Synced node info -13:11.849 [DEBUG] TestAgent: Node info in sync -13:12.170 [DEBUG] TestAgent.dns: request served from client: name=_this-is-a-very-very-very-very-very-long-name-for-a-service._primary.service.consul. type=SRV class=IN latency=330.805µs client=127.0.0.1:42653 client_network=udp -13:12.170 [DEBUG] TestAgent.dns: request served from client: name=this-is-a-very-very-very-very-very-long-name-for-a-service.query.consul. type=SRV class=IN latency=153.25µs client=127.0.0.1:53007 client_network=udp -13:12.170 [INFO] TestAgent: Requesting shutdown -13:12.170 [INFO] TestAgent.server: shutting down server -13:12.170 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:12.170 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:12.170 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:12.170 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:12.170 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.171 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:12.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:12.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:12.171 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:12.171 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:12.171 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.171Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.171Z [INFO] agent.router.manager: shutting down -13:12.171 [INFO] TestAgent: consul server down -13:12.171 [INFO] TestAgent: shutdown complete -13:12.171 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16905 network=tcp -13:12.171 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16905 network=udp -13:12.171 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16906 network=tcp -13:12.677 [INFO] TestAgent: Waiting for endpoints to shut down -13:12.677 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_LargeResponses (1.09s) +2022-12-04T05:49:24.708Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.719Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.721 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2c41a7c0-e577-3198-09e5-783320f0f817 Address:127.0.0.1:23706}]" +49:24.721 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23706 [Follower]" leader= +49:24.721 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2c41a7c0-e577-3198-09e5-783320f0f817.dc1 127.0.0.1 +49:24.722 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2c41a7c0-e577-3198-09e5-783320f0f817 127.0.0.1 +2022-12-04T05:49:24.722Z [INFO] agent.router: Initializing LAN area manager +49:24.722 [INFO] TestAgent.server: Adding LAN server: server="Node-2c41a7c0-e577-3198-09e5-783320f0f817 (Addr: tcp/127.0.0.1:23706) (DC: dc1)" +49:24.722 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2c41a7c0-e577-3198-09e5-783320f0f817.dc1 area=wan +49:24.722 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23826 network=udp +49:24.722 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23826 network=tcp +49:24.722 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23827 network=tcp +49:24.723 [INFO] TestAgent: started state syncer +49:24.723 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23707 network=tcp +49:24.774 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.774 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23706 [Candidate]" term=2 +49:24.774 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.774 [DEBUG] TestAgent.server.raft: vote granted: from=2c41a7c0-e577-3198-09e5-783320f0f817 term=2 tally=1 +49:24.774 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.774 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23706 [Leader]" +49:24.774 [INFO] TestAgent.server: cluster leadership acquired +49:24.774 [INFO] TestAgent.server: New leader elected: payload=Node-2c41a7c0-e577-3198-09e5-783320f0f817 +49:24.774 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23706 +49:24.775 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.777 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.777 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.777 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.777 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.777 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.777 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2c41a7c0-e577-3198-09e5-783320f0f817 +49:24.777 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2c41a7c0-e577-3198-09e5-783320f0f817 +49:24.777 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.832 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=231.059µs client=127.0.0.1:47475 client_network=udp +49:24.833 [DEBUG] TestAgent.dns: request served from client: name=ce0ecee9-bfe8-0235-92eb-1ad833f01a8a.query.consul. type=ANY class=IN latency=115.648µs client=127.0.0.1:54439 client_network=udp +49:24.833 [INFO] TestAgent: Requesting shutdown +49:24.833 [INFO] TestAgent.server: shutting down server +49:24.833 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.833 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.833 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.833 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.833 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.833 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.833 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.833 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.833 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:24.833 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.833 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.833 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.833Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.833Z [INFO] agent.router.manager: shutting down +49:24.833 [INFO] TestAgent: consul server down +49:24.833 [INFO] TestAgent: shutdown complete +49:24.833 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23826 network=tcp +49:24.833 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23826 network=udp +49:24.833 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23827 network=tcp +49:25.334 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.334 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_FilterCritical (0.64s) +=== CONT TestDNS_ServiceLookup_TagPeriod +2022-12-04T05:49:24.537Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.546Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.548 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2250ea4f-2fdc-67e5-07d6-7b0b67bec374 Address:127.0.0.1:23762}]" +49:24.548 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23762 [Follower]" leader= +49:24.548 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2250ea4f-2fdc-67e5-07d6-7b0b67bec374.dc1 127.0.0.1 +49:24.549 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2250ea4f-2fdc-67e5-07d6-7b0b67bec374 127.0.0.1 +2022-12-04T05:49:24.549Z [INFO] agent.router: Initializing LAN area manager +49:24.549 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23715 network=udp +49:24.549 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2250ea4f-2fdc-67e5-07d6-7b0b67bec374.dc1 area=wan +49:24.549 [INFO] TestAgent.server: Adding LAN server: server="Node-2250ea4f-2fdc-67e5-07d6-7b0b67bec374 (Addr: tcp/127.0.0.1:23762) (DC: dc1)" +49:24.549 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23715 network=tcp +49:24.549 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23716 network=tcp +49:24.549 [INFO] TestAgent: started state syncer +49:24.550 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23763 network=tcp +49:24.601 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.601 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23762 [Candidate]" term=2 +49:24.601 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.601 [DEBUG] TestAgent.server.raft: vote granted: from=2250ea4f-2fdc-67e5-07d6-7b0b67bec374 term=2 tally=1 +49:24.601 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.601 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23762 [Leader]" +49:24.601 [INFO] TestAgent.server: cluster leadership acquired +49:24.601 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23762 +49:24.602 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.603 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.603 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.603 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.603 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.603 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.603 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2250ea4f-2fdc-67e5-07d6-7b0b67bec374 +49:24.603 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2250ea4f-2fdc-67e5-07d6-7b0b67bec374 +49:24.603 [INFO] TestAgent.server: New leader elected: payload=Node-2250ea4f-2fdc-67e5-07d6-7b0b67bec374 +49:24.604 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.805 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.805 [INFO] TestAgent: Synced node info +49:24.848 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=164.453µs client=127.0.0.1:58828 client_network=udp +49:24.848 [DEBUG] TestAgent.dns: request served from client: name=1c6bab9a-20d2-8c05-e485-ce7eb4a3edc2.query.consul. type=SRV class=IN latency=202.619µs client=127.0.0.1:56710 client_network=udp +49:24.848 [INFO] TestAgent: Requesting shutdown +49:24.848 [INFO] TestAgent.server: shutting down server +49:24.848 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.848 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.848 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.848 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.848 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.848 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.848 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.848 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.848 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.848 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.849 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.849Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.849Z [INFO] agent.router.manager: shutting down +49:24.849 [INFO] TestAgent: consul server down +49:24.849 [INFO] TestAgent: shutdown complete +49:24.849 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23715 network=tcp +49:24.849 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23715 network=udp +49:24.849 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23716 network=tcp +49:25.352 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.352 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_Dedup_SRV (0.83s) +=== CONT TestDNS_Recurse_Truncation +2022-12-04T05:49:24.599Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.613Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.614 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:23a47290-a437-eefc-8367-70329fa11541 Address:127.0.0.1:23775}]" +49:24.614 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23775 [Follower]" leader= +49:24.615 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-23a47290-a437-eefc-8367-70329fa11541.dc1 127.0.0.1 +49:24.615 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-23a47290-a437-eefc-8367-70329fa11541 127.0.0.1 +2022-12-04T05:49:24.615Z [INFO] agent.router: Initializing LAN area manager +49:24.615 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-23a47290-a437-eefc-8367-70329fa11541.dc1 area=wan +49:24.615 [INFO] TestAgent.server: Adding LAN server: server="Node-23a47290-a437-eefc-8367-70329fa11541 (Addr: tcp/127.0.0.1:23775) (DC: dc1)" +49:24.615 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23750 network=udp +49:24.615 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23750 network=tcp +49:24.616 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23751 network=tcp +49:24.616 [INFO] TestAgent: started state syncer +49:24.616 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23776 network=tcp +49:24.672 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.672 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23775 [Candidate]" term=2 +49:24.672 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.672 [DEBUG] TestAgent.server.raft: vote granted: from=23a47290-a437-eefc-8367-70329fa11541 term=2 tally=1 +49:24.672 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.672 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23775 [Leader]" +49:24.672 [INFO] TestAgent.server: cluster leadership acquired +49:24.672 [INFO] TestAgent.server: New leader elected: payload=Node-23a47290-a437-eefc-8367-70329fa11541 +49:24.672 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23775 +49:24.673 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.674 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.674 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.674 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.674 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.674 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.674 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-23a47290-a437-eefc-8367-70329fa11541 +49:24.674 [INFO] TestAgent.server: member joined, marking health alive: member=Node-23a47290-a437-eefc-8367-70329fa11541 +49:24.675 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.771 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.771 [INFO] TestAgent: Synced node info +49:24.771 [DEBUG] TestAgent: Node info in sync +49:24.862 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=235.889µs client=127.0.0.1:45541 client_network=udp +49:24.863 [DEBUG] TestAgent.dns: request served from client: name=621aa6a3-d8ff-5630-aa22-2db762cedf78.query.consul. type=ANY class=IN latency=216.592µs client=127.0.0.1:60692 client_network=udp +49:24.863 [INFO] TestAgent: Requesting shutdown +49:24.863 [INFO] TestAgent.server: shutting down server +49:24.863 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.863 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.863 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.863 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.863 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.863 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.863 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.863 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.863 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.863 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.863 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.863Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.863Z [INFO] agent.router.manager: shutting down +49:24.863 [INFO] TestAgent: consul server down +49:24.863 [INFO] TestAgent: shutdown complete +49:24.863 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23750 network=tcp +49:24.863 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23750 network=udp +49:24.863 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23751 network=tcp +49:25.364 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.364 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_OnlyFailing (0.78s) === CONT TestDNS_CaseInsensitiveServiceLookup -2021-10-31T23:13:12.002Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:12.027Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:12.029 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef7bafa1-417a-6ee2-814c-614dae1361bf Address:127.0.0.1:16949}]" -13:12.029 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16949 [Follower]" leader= -13:12.038 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef7bafa1-417a-6ee2-814c-614dae1361bf.dc1 127.0.0.1 -13:12.078 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef7bafa1-417a-6ee2-814c-614dae1361bf 127.0.0.1 -2021-10-31T23:13:12.078Z [INFO] agent.router: Initializing LAN area manager -13:12.078 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16958 network=udp -13:12.079 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16958 network=tcp -13:12.079 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef7bafa1-417a-6ee2-814c-614dae1361bf.dc1 area=wan -13:12.080 [INFO] TestAgent.server: Adding LAN server: server="Node-ef7bafa1-417a-6ee2-814c-614dae1361bf (Addr: tcp/127.0.0.1:16949) (DC: dc1)" -13:12.081 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16959 network=tcp -13:12.081 [INFO] TestAgent: started state syncer -13:12.081 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16950 network=tcp -13:12.081 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:12.081 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16949 [Candidate]" term=2 -13:12.091 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:12.091 [DEBUG] TestAgent.server.raft: vote granted: from=ef7bafa1-417a-6ee2-814c-614dae1361bf term=2 tally=1 -13:12.091 [INFO] TestAgent.server.raft: election won: tally=1 -13:12.091 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16949 [Leader]" -13:12.092 [INFO] TestAgent.server: cluster leadership acquired -13:12.147 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16949 -13:12.147 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:12.148 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:12.149 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:12.149 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:12.149 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:12.149 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:12.149 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef7bafa1-417a-6ee2-814c-614dae1361bf -13:12.149 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef7bafa1-417a-6ee2-814c-614dae1361bf -13:12.150 [INFO] TestAgent.server: federation state anti-entropy synced -13:12.150 [INFO] TestAgent.server: New leader elected: payload=Node-ef7bafa1-417a-6ee2-814c-614dae1361bf -13:12.206 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=251.156µs client=127.0.0.1:39635 client_network=udp -13:12.214 [DEBUG] TestAgent.dns: request served from client: name=fa9f4188-cf03-64e5-e57b-4ffef09f8097.query.consul. type=ANY class=IN latency=247.885µs client=127.0.0.1:46677 client_network=udp -13:12.214 [INFO] TestAgent: Requesting shutdown -13:12.215 [INFO] TestAgent.server: shutting down server -13:12.215 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.215 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:12.215 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:12.215 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.215Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.215Z [INFO] agent.router.manager: shutting down -13:12.215 [INFO] TestAgent: consul server down -13:12.215 [INFO] TestAgent: shutdown complete -13:12.215 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16958 network=tcp -13:12.215 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16958 network=udp -13:12.215 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16959 network=tcp -13:12.715 [INFO] TestAgent: Waiting for endpoints to shut down -13:12.715 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_OnlyPassing (0.79s) -=== CONT TestDNS_PreparedQueryNearIPEDNS -2021-10-31T23:13:11.992Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:12.027Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:12.029 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fa5b2e8f-dc62-2b12-986f-c04a1b6039b2 Address:127.0.0.1:16956}]" -13:12.034 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16956 [Follower]" leader= -13:12.034 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fa5b2e8f-dc62-2b12-986f-c04a1b6039b2.dc1 127.0.0.1 -13:12.035 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fa5b2e8f-dc62-2b12-986f-c04a1b6039b2 127.0.0.1 -2021-10-31T23:13:12.035Z [INFO] agent.router: Initializing LAN area manager -13:12.035 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16937 network=udp -13:12.035 [INFO] TestAgent.server: Adding LAN server: server="Node-fa5b2e8f-dc62-2b12-986f-c04a1b6039b2 (Addr: tcp/127.0.0.1:16956) (DC: dc1)" -13:12.035 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fa5b2e8f-dc62-2b12-986f-c04a1b6039b2.dc1 area=wan -13:12.036 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16937 network=tcp -13:12.036 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16938 network=tcp -13:12.036 [INFO] TestAgent: started state syncer -13:12.036 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16957 network=tcp -13:12.161 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:12.161 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16956 [Candidate]" term=2 -13:12.162 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:12.162 [DEBUG] TestAgent.server.raft: vote granted: from=fa5b2e8f-dc62-2b12-986f-c04a1b6039b2 term=2 tally=1 -13:12.162 [INFO] TestAgent.server.raft: election won: tally=1 -13:12.162 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16956 [Leader]" -13:12.162 [INFO] TestAgent.server: cluster leadership acquired -13:12.162 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16956 -13:12.163 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:12.164 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:12.164 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:12.164 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:12.164 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:12.164 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:12.164 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fa5b2e8f-dc62-2b12-986f-c04a1b6039b2 -13:12.164 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fa5b2e8f-dc62-2b12-986f-c04a1b6039b2 -13:12.166 [INFO] TestAgent.server: federation state anti-entropy synced -13:12.166 [INFO] TestAgent.server: New leader elected: payload=Node-fa5b2e8f-dc62-2b12-986f-c04a1b6039b2 -13:12.202 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:12.202 [INFO] TestAgent: Synced node info -13:12.214 [DEBUG] TestAgent.dns: request served from client: name=db.query.consul. type=SRV class=IN latency=214.592µs client=127.0.0.1:55232 client_network=udp -13:12.214 [DEBUG] TestAgent.dns: request served from client: name=db-ttl.query.consul. type=SRV class=IN latency=96.728µs client=127.0.0.1:59056 client_network=udp -13:12.214 [DEBUG] TestAgent.dns: request served from client: name=dblb.query.consul. type=SRV class=IN latency=63.917µs client=127.0.0.1:37747 client_network=udp -13:12.215 [DEBUG] TestAgent.dns: request served from client: name=dblb-ttl.query.consul. type=SRV class=IN latency=61.694µs client=127.0.0.1:60339 client_network=udp -13:12.218 [DEBUG] TestAgent.dns: request served from client: name=dk.query.consul. type=SRV class=IN latency=185.061µs client=127.0.0.1:51200 client_network=udp -13:12.218 [DEBUG] TestAgent.dns: request served from client: name=dk-ttl.query.consul. type=SRV class=IN latency=67.904µs client=127.0.0.1:48665 client_network=udp -13:12.218 [DEBUG] TestAgent.dns: request served from client: name=api.query.consul. type=SRV class=IN latency=64.624µs client=127.0.0.1:55884 client_network=udp -13:12.219 [DEBUG] TestAgent.dns: request served from client: name=api-ttl.query.consul. type=SRV class=IN latency=62.039µs client=127.0.0.1:44501 client_network=udp -13:12.219 [INFO] TestAgent: Requesting shutdown -13:12.219 [INFO] TestAgent.server: shutting down server -13:12.219 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:12.219 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:12.219 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:12.219 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:12.219 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.219 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:12.219 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:12.219 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:12.219 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:12.219 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:12.219 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.219Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.219Z [INFO] agent.router.manager: shutting down -13:12.219 [INFO] TestAgent: consul server down -13:12.219 [INFO] TestAgent: shutdown complete -13:12.219 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16937 network=tcp -13:12.219 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16937 network=udp -13:12.219 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16938 network=tcp -13:12.721 [INFO] TestAgent: Waiting for endpoints to shut down -13:12.721 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_PreparedQuery_TTL (0.81s) +2022-12-04T05:49:24.498Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.514Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.517 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e9cab425-5f6f-e039-15ad-82bcec5f573a Address:127.0.0.1:23734}]" +49:24.517 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23734 [Follower]" leader= +49:24.518 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e9cab425-5f6f-e039-15ad-82bcec5f573a.dc1 127.0.0.1 +49:24.520 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e9cab425-5f6f-e039-15ad-82bcec5f573a 127.0.0.1 +2022-12-04T05:49:24.520Z [INFO] agent.router: Initializing LAN area manager +49:24.521 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e9cab425-5f6f-e039-15ad-82bcec5f573a.dc1 area=wan +49:24.521 [INFO] TestAgent.server: Adding LAN server: server="Node-e9cab425-5f6f-e039-15ad-82bcec5f573a (Addr: tcp/127.0.0.1:23734) (DC: dc1)" +49:24.521 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23729 network=udp +49:24.521 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23729 network=tcp +49:24.521 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23730 network=tcp +49:24.521 [INFO] TestAgent: started state syncer +49:24.521 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23735 network=tcp +49:24.572 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.572 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23734 [Candidate]" term=2 +49:24.572 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.572 [DEBUG] TestAgent.server.raft: vote granted: from=e9cab425-5f6f-e039-15ad-82bcec5f573a term=2 tally=1 +49:24.572 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.572 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23734 [Leader]" +49:24.572 [INFO] TestAgent.server: cluster leadership acquired +49:24.573 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23734 +49:24.573 [INFO] TestAgent.server: New leader elected: payload=Node-e9cab425-5f6f-e039-15ad-82bcec5f573a +49:24.574 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.575 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.575 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.575 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.575 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.575 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.575 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e9cab425-5f6f-e039-15ad-82bcec5f573a +49:24.575 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e9cab425-5f6f-e039-15ad-82bcec5f573a +49:24.576 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.743 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.744 [INFO] TestAgent: Synced node info +49:24.744 [DEBUG] TestAgent: Node info in sync +49:24.872 [DEBUG] TestAgent.dns: request served from client: name=db.query.consul. type=SRV class=IN latency=172.488µs client=127.0.0.1:56563 client_network=udp +49:24.873 [DEBUG] TestAgent.dns: request served from client: name=db-ttl.query.consul. type=SRV class=IN latency=88.38µs client=127.0.0.1:36710 client_network=udp +49:24.873 [DEBUG] TestAgent.dns: request served from client: name=dblb.query.consul. type=SRV class=IN latency=71.619µs client=127.0.0.1:34740 client_network=udp +49:24.873 [DEBUG] TestAgent.dns: request served from client: name=dblb-ttl.query.consul. type=SRV class=IN latency=69.795µs client=127.0.0.1:38937 client_network=udp +49:24.874 [DEBUG] TestAgent.dns: request served from client: name=dk.query.consul. type=SRV class=IN latency=65.515µs client=127.0.0.1:47600 client_network=udp +49:24.874 [DEBUG] TestAgent.dns: request served from client: name=dk-ttl.query.consul. type=SRV class=IN latency=137.844µs client=127.0.0.1:39772 client_network=udp +49:24.874 [DEBUG] TestAgent.dns: request served from client: name=api.query.consul. type=SRV class=IN latency=84.191µs client=127.0.0.1:42800 client_network=udp +49:24.875 [DEBUG] TestAgent.dns: request served from client: name=api-ttl.query.consul. type=SRV class=IN latency=64.257µs client=127.0.0.1:44705 client_network=udp +49:24.875 [INFO] TestAgent: Requesting shutdown +49:24.875 [INFO] TestAgent.server: shutting down server +49:24.875 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.875 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.875 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.875 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.875 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.875 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.875 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.875 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.875 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.875 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.875 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.875Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.875Z [INFO] agent.router.manager: shutting down +49:24.875 [INFO] TestAgent: consul server down +49:24.875 [INFO] TestAgent: shutdown complete +49:24.875 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23729 network=tcp +49:24.875 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23729 network=udp +49:24.875 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23730 network=tcp +49:25.375 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.375 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_PreparedQuery_TTL (0.90s) --- PASS: TestDNS_PreparedQuery_TTL/db.query.consul. (0.00s) --- PASS: TestDNS_PreparedQuery_TTL/db-ttl.query.consul. (0.00s) --- PASS: TestDNS_PreparedQuery_TTL/dblb.query.consul. (0.00s) @@ -28915,2828 +35069,2495 @@ --- PASS: TestDNS_PreparedQuery_TTL/dk-ttl.query.consul. (0.00s) --- PASS: TestDNS_PreparedQuery_TTL/api.query.consul. (0.00s) --- PASS: TestDNS_PreparedQuery_TTL/api-ttl.query.consul. (0.00s) -=== CONT TestDNS_Recurse_Truncation -2021-10-31T23:13:12.197Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:12.237Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:12.274 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ac19d6d1-a2cb-552f-1643-7ab13a684c5d Address:127.0.0.1:16991}]" -13:12.277 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ac19d6d1-a2cb-552f-1643-7ab13a684c5d.dc1 127.0.0.1 -13:12.283 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16991 [Follower]" leader= -13:12.332 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ac19d6d1-a2cb-552f-1643-7ab13a684c5d 127.0.0.1 -2021-10-31T23:13:12.332Z [INFO] agent.router: Initializing LAN area manager -13:12.332 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16986 network=udp -13:12.338 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ac19d6d1-a2cb-552f-1643-7ab13a684c5d.dc1 area=wan -13:12.338 [INFO] TestAgent.server: Adding LAN server: server="Node-ac19d6d1-a2cb-552f-1643-7ab13a684c5d (Addr: tcp/127.0.0.1:16991) (DC: dc1)" -13:12.338 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16986 network=tcp -13:12.339 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:12.339 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16991 [Candidate]" term=2 -13:12.339 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:12.339 [DEBUG] TestAgent.server.raft: vote granted: from=ac19d6d1-a2cb-552f-1643-7ab13a684c5d term=2 tally=1 -13:12.339 [INFO] TestAgent.server.raft: election won: tally=1 -13:12.339 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16991 [Leader]" -13:12.339 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16987 network=tcp -13:12.339 [INFO] TestAgent: started state syncer -13:12.341 [INFO] TestAgent.server: cluster leadership acquired -13:12.341 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16992 network=tcp -13:12.341 [INFO] TestAgent.server: New leader elected: payload=Node-ac19d6d1-a2cb-552f-1643-7ab13a684c5d -13:12.342 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16991 -13:12.342 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=264.318µs client=127.0.0.1:58631 client_network=udp -13:12.342 [INFO] TestAgent: Synced node info -13:12.342 [DEBUG] TestAgent: Node info in sync -13:12.342 [DEBUG] TestAgent.dns: request served from client: name=dblb.service.consul. type=SRV class=IN latency=136.109µs client=127.0.0.1:34058 client_network=udp -13:12.342 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:12.343 [DEBUG] TestAgent.dns: request served from client: name=dk.service.consul. type=SRV class=IN latency=102.128µs client=127.0.0.1:40121 client_network=udp -13:12.343 [DEBUG] TestAgent.dns: request served from client: name=api.service.consul. type=SRV class=IN latency=54.477µs client=127.0.0.1:38811 client_network=udp -13:12.343 [INFO] TestAgent: Requesting shutdown -13:12.343 [INFO] TestAgent.server: shutting down server -13:12.343 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.343 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.343Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.343Z [INFO] agent.router.manager: shutting down -13:12.343 [INFO] TestAgent: consul server down -13:12.343 [INFO] TestAgent: shutdown complete -13:12.343 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16986 network=tcp -13:12.344 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16986 network=udp -13:12.344 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16987 network=tcp -13:12.344 [ERROR] TestAgent.server.connect: Raft apply failed: error="raft is already shutdown" -13:12.344 [ERROR] TestAgent.server: failed to establish leadership: error="raft is already shutdown" -13:12.344 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=0 retry_limit=3 error="raft is already shutdown" -13:12.845 [INFO] TestAgent: Waiting for endpoints to shut down -13:12.845 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_TTL (0.68s) - --- PASS: TestDNS_ServiceLookup_TTL/db.service.consul. (0.00s) - --- PASS: TestDNS_ServiceLookup_TTL/dblb.service.consul. (0.00s) - --- PASS: TestDNS_ServiceLookup_TTL/dk.service.consul. (0.00s) - --- PASS: TestDNS_ServiceLookup_TTL/api.service.consul. (0.00s) +=== CONT TestDNS_ServiceLookup_ServiceAddress_A +2022-12-04T05:49:24.532Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.542Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.544 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4b845b50-a4ed-65c0-41fa-f2283f9a4fef Address:127.0.0.1:23713}]" +49:24.544 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23713 [Follower]" leader= +49:24.545 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4b845b50-a4ed-65c0-41fa-f2283f9a4fef.dc1 127.0.0.1 +49:24.545 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4b845b50-a4ed-65c0-41fa-f2283f9a4fef 127.0.0.1 +2022-12-04T05:49:24.545Z [INFO] agent.router: Initializing LAN area manager +49:24.546 [INFO] TestAgent.server: Adding LAN server: server="Node-4b845b50-a4ed-65c0-41fa-f2283f9a4fef (Addr: tcp/127.0.0.1:23713) (DC: dc1)" +49:24.546 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23722 network=udp +49:24.546 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4b845b50-a4ed-65c0-41fa-f2283f9a4fef.dc1 area=wan +49:24.546 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23722 network=tcp +49:24.546 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23723 network=tcp +49:24.546 [INFO] TestAgent: started state syncer +49:24.546 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23714 network=tcp +49:24.606 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.606 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23713 [Candidate]" term=2 +49:24.606 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.606 [DEBUG] TestAgent.server.raft: vote granted: from=4b845b50-a4ed-65c0-41fa-f2283f9a4fef term=2 tally=1 +49:24.606 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.606 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23713 [Leader]" +49:24.607 [INFO] TestAgent.server: cluster leadership acquired +49:24.607 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23713 +49:24.607 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.608 [INFO] TestAgent.server: New leader elected: payload=Node-4b845b50-a4ed-65c0-41fa-f2283f9a4fef +49:24.608 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.608 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.608 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.608 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.608 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.608 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4b845b50-a4ed-65c0-41fa-f2283f9a4fef +49:24.608 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4b845b50-a4ed-65c0-41fa-f2283f9a4fef +49:24.609 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.822 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.822 [INFO] TestAgent: Synced node info +49:24.898 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=290.134µs client=127.0.0.1:36719 client_network=udp +49:24.899 [DEBUG] TestAgent.dns: request served from client: name=84f150d8-b2dc-1668-9b5f-fd4602f8320c.query.consul. type=ANY class=IN latency=178.688µs client=127.0.0.1:56827 client_network=udp +49:24.899 [INFO] TestAgent: Requesting shutdown +49:24.899 [INFO] TestAgent.server: shutting down server +49:24.899 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.900 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.900 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.900Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.900Z [INFO] agent.router.manager: shutting down +49:24.900 [INFO] TestAgent: consul server down +49:24.900 [INFO] TestAgent: shutdown complete +49:24.900 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23722 network=tcp +49:24.900 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23722 network=udp +49:24.900 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23723 network=tcp +49:25.400 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.400 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_OnlyPassing (0.88s) === CONT TestDNS_Lookup_TaggedIPAddresses -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:12.178Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:12.202Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:12.204 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:64cef380-aedb-a35c-19e8-a3a08ba8a38c Address:127.0.0.1:16984}]" -13:12.204 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-64cef380-aedb-a35c-19e8-a3a08ba8a38c.dc1 127.0.0.1 -13:12.205 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-64cef380-aedb-a35c-19e8-a3a08ba8a38c 127.0.0.1 -2021-10-31T23:13:12.205Z [INFO] agent.router: Initializing LAN area manager -13:12.205 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16979 network=udp -13:12.205 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16984 [Follower]" leader= -13:12.206 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-64cef380-aedb-a35c-19e8-a3a08ba8a38c.dc1 area=wan -13:12.206 [INFO] TestAgent.server: Adding LAN server: server="Node-64cef380-aedb-a35c-19e8-a3a08ba8a38c (Addr: tcp/127.0.0.1:16984) (DC: dc1)" -13:12.206 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16979 network=tcp -13:12.214 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16980 network=tcp -13:12.214 [INFO] TestAgent: started state syncer -13:12.214 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16985 network=tcp -13:12.285 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:12.286 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16984 [Candidate]" term=2 -13:12.287 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:12.287 [DEBUG] TestAgent.server.raft: vote granted: from=64cef380-aedb-a35c-19e8-a3a08ba8a38c term=2 tally=1 -13:12.287 [INFO] TestAgent.server.raft: election won: tally=1 -13:12.287 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16984 [Leader]" -13:12.288 [INFO] TestAgent.server: cluster leadership acquired -13:12.289 [INFO] TestAgent.server: New leader elected: payload=Node-64cef380-aedb-a35c-19e8-a3a08ba8a38c -13:12.291 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16984 -13:12.294 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:12.331 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:12.331 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:12.331 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:12.331 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:12.331 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:12.332 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-64cef380-aedb-a35c-19e8-a3a08ba8a38c -13:12.332 [INFO] TestAgent.server: member joined, marking health alive: member=Node-64cef380-aedb-a35c-19e8-a3a08ba8a38c -13:12.333 [INFO] TestAgent.server: federation state anti-entropy synced -13:12.354 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=254.488µs client=127.0.0.1:46548 client_network=udp -13:12.358 [DEBUG] TestAgent.dns: request served from client: name=a92388c3-8bc9-04a9-da36-c5c051c7b71e.query.consul. type=ANY class=IN latency=363.404µs client=127.0.0.1:45643 client_network=udp -13:12.360 [INFO] TestAgent: Requesting shutdown -13:12.360 [INFO] TestAgent.server: shutting down server -13:12.360 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:12.360 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:12.360 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:12.360 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:12.360 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.360 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:12.360 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:12.360 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:12.360 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:12.360 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:12.360 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:12.360 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.360Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.360Z [INFO] agent.router.manager: shutting down -13:12.360 [INFO] TestAgent: consul server down -13:12.360 [INFO] TestAgent: shutdown complete -13:12.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16979 network=tcp -13:12.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16979 network=udp -13:12.360 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16980 network=tcp -13:12.863 [INFO] TestAgent: Waiting for endpoints to shut down -13:12.863 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_FilterCritical (0.77s) -=== CONT TestDNS_ServiceLookup_TagPeriod +2022-12-04T05:49:24.659Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.672Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.673 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5e5f5df9-8e72-9121-7b3b-125da17478f3 Address:127.0.0.1:23789}]" +49:24.673 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23789 [Follower]" leader= +49:24.674 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5e5f5df9-8e72-9121-7b3b-125da17478f3.dc1 127.0.0.1 +49:24.674 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5e5f5df9-8e72-9121-7b3b-125da17478f3 127.0.0.1 +2022-12-04T05:49:24.674Z [INFO] agent.router: Initializing LAN area manager +49:24.675 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5e5f5df9-8e72-9121-7b3b-125da17478f3.dc1 area=wan +49:24.675 [INFO] TestAgent.server: Adding LAN server: server="Node-5e5f5df9-8e72-9121-7b3b-125da17478f3 (Addr: tcp/127.0.0.1:23789) (DC: dc1)" +49:24.675 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23777 network=udp +49:24.675 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23777 network=tcp +49:24.675 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23785 network=tcp +49:24.675 [INFO] TestAgent: started state syncer +49:24.675 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23790 network=tcp +49:24.715 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.715 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23789 [Candidate]" term=2 +49:24.715 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.715 [DEBUG] TestAgent.server.raft: vote granted: from=5e5f5df9-8e72-9121-7b3b-125da17478f3 term=2 tally=1 +49:24.715 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.715 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23789 [Leader]" +49:24.715 [INFO] TestAgent.server: cluster leadership acquired +49:24.716 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23789 +49:24.716 [INFO] TestAgent.server: New leader elected: payload=Node-5e5f5df9-8e72-9121-7b3b-125da17478f3 +49:24.716 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.718 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.718 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.718 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.718 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.718 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.718 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5e5f5df9-8e72-9121-7b3b-125da17478f3 +49:24.718 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5e5f5df9-8e72-9121-7b3b-125da17478f3 +49:24.718 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.750 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.750 [INFO] TestAgent: Synced node info +49:24.750 [DEBUG] TestAgent: Node info in sync +49:24.968 [DEBUG] TestAgent.dns: request served from client: name=_this-is-a-very-very-very-very-very-long-name-for-a-service._primary.service.consul. type=SRV class=IN latency=199.66µs client=127.0.0.1:53756 client_network=udp +49:24.969 [DEBUG] TestAgent.dns: request served from client: name=this-is-a-very-very-very-very-very-long-name-for-a-service.query.consul. type=SRV class=IN latency=107.288µs client=127.0.0.1:54807 client_network=udp +49:24.969 [INFO] TestAgent: Requesting shutdown +49:24.969 [INFO] TestAgent.server: shutting down server +49:24.969 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.969 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.969 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.969 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.969 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.969 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.969 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.969 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.969 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.969 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.969 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.969Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.969Z [INFO] agent.router.manager: shutting down +49:24.969 [INFO] TestAgent: consul server down +49:24.969 [INFO] TestAgent: shutdown complete +49:24.969 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23777 network=tcp +49:24.969 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23777 network=udp +49:24.969 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23785 network=tcp +49:25.469 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.469 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_LargeResponses (0.84s) +=== CONT TestDNS_ExternalServiceToConsulCNAMENestedLookup +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=255)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=255)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=255)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=255)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=false +2022-12-04T05:49:24.543Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.555Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.556 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6d7e5e21-a754-b99a-2072-418ca4e2a4ad Address:127.0.0.1:23782}]" +49:24.556 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23782 [Follower]" leader= +49:24.557 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6d7e5e21-a754-b99a-2072-418ca4e2a4ad.dc1 127.0.0.1 +49:24.557 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6d7e5e21-a754-b99a-2072-418ca4e2a4ad 127.0.0.1 +2022-12-04T05:49:24.557Z [INFO] agent.router: Initializing LAN area manager +49:24.557 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6d7e5e21-a754-b99a-2072-418ca4e2a4ad.dc1 area=wan +49:24.557 [INFO] TestAgent.server: Adding LAN server: server="Node-6d7e5e21-a754-b99a-2072-418ca4e2a4ad (Addr: tcp/127.0.0.1:23782) (DC: dc1)" +49:24.557 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23766 network=udp +49:24.557 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23766 network=tcp +49:24.558 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23778 network=tcp +49:24.558 [INFO] TestAgent: started state syncer +49:24.558 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23783 network=tcp +49:24.625 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.625 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23782 [Candidate]" term=2 +49:24.625 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.625 [DEBUG] TestAgent.server.raft: vote granted: from=6d7e5e21-a754-b99a-2072-418ca4e2a4ad term=2 tally=1 +49:24.625 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.625 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23782 [Leader]" +49:24.625 [INFO] TestAgent.server: cluster leadership acquired +49:24.626 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23782 +49:24.626 [INFO] TestAgent.server: New leader elected: payload=Node-6d7e5e21-a754-b99a-2072-418ca4e2a4ad +49:24.626 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.628 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.628 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.628 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.628 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.628 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.628 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6d7e5e21-a754-b99a-2072-418ca4e2a4ad +49:24.628 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6d7e5e21-a754-b99a-2072-418ca4e2a4ad +49:24.628 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.738 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.739 [INFO] TestAgent: Synced node info +49:24.739 [DEBUG] TestAgent: Node info in sync +49:24.995 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=372.859µs client=127.0.0.1:53665 client_network=udp +49:24.996 [DEBUG] TestAgent.dns: request served from client: name=28e800cc-fbc4-a2de-ad00-04f1d6646f7a.query.consul. type=ANY class=IN latency=132.177µs client=127.0.0.1:60946 client_network=udp +49:24.996 [INFO] TestAgent: Requesting shutdown +49:24.996 [INFO] TestAgent.server: shutting down server +49:24.996 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.996 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.996 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.996 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.996 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.996 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.996 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.996 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.996 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:24.996 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.996 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:24.996Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.996Z [INFO] agent.router.manager: shutting down +49:24.996 [INFO] TestAgent: consul server down +49:24.996 [INFO] TestAgent: shutdown complete +49:24.996 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23766 network=tcp +49:24.997 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23766 network=udp +49:24.997 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23778 network=tcp +49:25.497 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.497 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_Dedup (0.97s) +=== CONT TestDNS_ExternalServiceLookup +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=1)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=1)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=1)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=1)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=33)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=33)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=33)_compressed=true Added 3 service nodes -2021-10-31T23:13:11.785Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:11.829Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:11.836 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9d793df8-d9ee-72a5-f7c6-aebae3273a16 Address:127.0.0.1:16935}]" -13:11.837 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9d793df8-d9ee-72a5-f7c6-aebae3273a16.dc1 127.0.0.1 -13:11.837 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9d793df8-d9ee-72a5-f7c6-aebae3273a16 127.0.0.1 -2021-10-31T23:13:11.837Z [INFO] agent.router: Initializing LAN area manager -13:11.837 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16935 [Follower]" leader= -13:11.837 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16943 network=udp -13:11.837 [INFO] TestAgent.server: Adding LAN server: server="Node-9d793df8-d9ee-72a5-f7c6-aebae3273a16 (Addr: tcp/127.0.0.1:16935) (DC: dc1)" -13:11.838 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9d793df8-d9ee-72a5-f7c6-aebae3273a16.dc1 area=wan -13:11.838 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16943 network=tcp -13:11.838 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16944 network=tcp -13:11.838 [INFO] TestAgent: started state syncer -13:11.838 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16936 network=tcp -13:11.903 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:11.903 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16935 [Candidate]" term=2 -13:11.903 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:11.903 [DEBUG] TestAgent.server.raft: vote granted: from=9d793df8-d9ee-72a5-f7c6-aebae3273a16 term=2 tally=1 -13:11.903 [INFO] TestAgent.server.raft: election won: tally=1 -13:11.903 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16935 [Leader]" -13:11.904 [INFO] TestAgent.server: cluster leadership acquired -13:11.904 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16935 -13:11.905 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.905 [INFO] TestAgent.server: New leader elected: payload=Node-9d793df8-d9ee-72a5-f7c6-aebae3273a16 -13:11.908 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.908 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.908 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.908 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.908 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.908 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9d793df8-d9ee-72a5-f7c6-aebae3273a16 -13:11.908 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9d793df8-d9ee-72a5-f7c6-aebae3273a16 -13:11.934 [INFO] TestAgent.server: federation state anti-entropy synced -13:12.282 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:12.295 [INFO] TestAgent: Synced node info -13:12.337 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.447021ms client=127.0.0.1:38014 client_network=udp -13:12.342 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=5.621923ms client=127.0.0.1:56010 client_network=udp -13:12.344 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.464117ms client=127.0.0.1:41894 client_network=udp -13:12.356 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.381539ms client=127.0.0.1:36563 client_network=udp -13:12.363 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.510517ms client=127.0.0.1:53680 client_network=udp -13:12.371 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.442266ms client=127.0.0.1:60919 client_network=udp -13:12.380 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.581127ms client=127.0.0.1:42377 client_network=udp -13:12.393 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.537143ms client=127.0.0.1:59852 client_network=udp -13:12.395 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.247998ms client=127.0.0.1:51768 client_network=udp -13:12.403 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.638453ms client=127.0.0.1:60802 client_network=udp -13:12.411 [DEBUG] TestAgent.dns: request served from client: name=70f028c2-2b0a-785a-d3b4-2434d867e66d.query.consul. type=ANY class=IN latency=1.50653ms client=127.0.0.1:32837 client_network=udp -13:12.412 [DEBUG] TestAgent.dns: request served from client: name=70f028c2-2b0a-785a-d3b4-2434d867e66d.query.consul. type=ANY class=IN latency=1.19603ms client=127.0.0.1:44457 client_network=udp -13:12.417 [DEBUG] TestAgent.dns: request served from client: name=70f028c2-2b0a-785a-d3b4-2434d867e66d.query.consul. type=ANY class=IN latency=1.531356ms client=127.0.0.1:49123 client_network=udp -13:12.419 [DEBUG] TestAgent.dns: request served from client: name=70f028c2-2b0a-785a-d3b4-2434d867e66d.query.consul. type=ANY class=IN latency=1.462742ms client=127.0.0.1:49883 client_network=udp -13:12.420 [DEBUG] TestAgent.dns: request served from client: name=70f028c2-2b0a-785a-d3b4-2434d867e66d.query.consul. type=ANY class=IN latency=1.291321ms client=127.0.0.1:33393 client_network=udp -13:12.431 [DEBUG] TestAgent.dns: request served from client: name=70f028c2-2b0a-785a-d3b4-2434d867e66d.query.consul. type=ANY class=IN latency=10.233912ms client=127.0.0.1:43518 client_network=udp -13:12.435 [DEBUG] TestAgent.dns: request served from client: name=70f028c2-2b0a-785a-d3b4-2434d867e66d.query.consul. type=ANY class=IN latency=1.527594ms client=127.0.0.1:47267 client_network=udp -13:12.437 [DEBUG] TestAgent.dns: request served from client: name=70f028c2-2b0a-785a-d3b4-2434d867e66d.query.consul. type=ANY class=IN latency=1.286865ms client=127.0.0.1:45184 client_network=udp -13:12.447 [DEBUG] TestAgent.dns: request served from client: name=70f028c2-2b0a-785a-d3b4-2434d867e66d.query.consul. type=ANY class=IN latency=10.12431ms client=127.0.0.1:44878 client_network=udp -13:12.449 [DEBUG] TestAgent.dns: request served from client: name=70f028c2-2b0a-785a-d3b4-2434d867e66d.query.consul. type=ANY class=IN latency=1.253825ms client=127.0.0.1:33388 client_network=udp -13:12.449 [INFO] TestAgent: Requesting shutdown -13:12.449 [INFO] TestAgent.server: shutting down server -13:12.449 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:12.449 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:12.449 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:12.449 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:12.449 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.449 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:12.449 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:12.449 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:12.449 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:12.449 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:12.449 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.449Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.449Z [INFO] agent.router.manager: shutting down -13:12.450 [INFO] TestAgent: consul server down -13:12.450 [INFO] TestAgent: shutdown complete -13:12.450 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16943 network=tcp -13:12.450 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16943 network=udp -13:12.450 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16944 network=tcp -13:12.951 [INFO] TestAgent: Waiting for endpoints to shut down -13:12.951 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_Randomize (1.18s) +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=33)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=false#01 +2022-12-04T05:49:24.489Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.501Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.503 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b0a27aea-a333-b33d-5ee6-b3a7ff0b7180 Address:127.0.0.1:23748}]" +49:24.503 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23748 [Follower]" leader= +49:24.503 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b0a27aea-a333-b33d-5ee6-b3a7ff0b7180.dc1 127.0.0.1 +49:24.504 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b0a27aea-a333-b33d-5ee6-b3a7ff0b7180 127.0.0.1 +2022-12-04T05:49:24.504Z [INFO] agent.router: Initializing LAN area manager +49:24.504 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23743 network=udp +49:24.504 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b0a27aea-a333-b33d-5ee6-b3a7ff0b7180.dc1 area=wan +49:24.504 [INFO] TestAgent.server: Adding LAN server: server="Node-b0a27aea-a333-b33d-5ee6-b3a7ff0b7180 (Addr: tcp/127.0.0.1:23748) (DC: dc1)" +49:24.504 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23743 network=tcp +49:24.510 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23744 network=tcp +49:24.510 [INFO] TestAgent: started state syncer +49:24.510 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23749 network=tcp +49:24.561 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.561 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23748 [Candidate]" term=2 +49:24.561 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.561 [DEBUG] TestAgent.server.raft: vote granted: from=b0a27aea-a333-b33d-5ee6-b3a7ff0b7180 term=2 tally=1 +49:24.561 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.561 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23748 [Leader]" +49:24.561 [INFO] TestAgent.server: cluster leadership acquired +49:24.561 [INFO] TestAgent.server: New leader elected: payload=Node-b0a27aea-a333-b33d-5ee6-b3a7ff0b7180 +49:24.561 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23748 +49:24.562 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.563 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.563 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.563 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.563 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.563 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.563 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b0a27aea-a333-b33d-5ee6-b3a7ff0b7180 +49:24.563 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b0a27aea-a333-b33d-5ee6-b3a7ff0b7180 +49:24.564 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.924 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.925 [INFO] TestAgent: Synced node info +49:24.966 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=961.943µs client=127.0.0.1:50698 client_network=udp +49:24.968 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.002143ms client=127.0.0.1:50768 client_network=udp +49:24.969 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=775.629µs client=127.0.0.1:55874 client_network=udp +49:24.970 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.111574ms client=127.0.0.1:57360 client_network=udp +49:24.971 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=868.573µs client=127.0.0.1:37072 client_network=udp +49:24.973 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.300282ms client=127.0.0.1:58456 client_network=udp +49:24.974 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.026112ms client=127.0.0.1:36306 client_network=udp +49:24.975 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=883.473µs client=127.0.0.1:58193 client_network=udp +49:24.976 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=794.041µs client=127.0.0.1:34010 client_network=udp +49:24.977 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=794.64µs client=127.0.0.1:60822 client_network=udp +49:24.978 [DEBUG] TestAgent.dns: request served from client: name=06bacc69-5985-8b7f-113c-8b11dca200fa.query.consul. type=ANY class=IN latency=1.175133ms client=127.0.0.1:58337 client_network=udp +49:24.979 [DEBUG] TestAgent.dns: request served from client: name=06bacc69-5985-8b7f-113c-8b11dca200fa.query.consul. type=ANY class=IN latency=881.303µs client=127.0.0.1:45960 client_network=udp +49:24.985 [DEBUG] TestAgent.dns: request served from client: name=06bacc69-5985-8b7f-113c-8b11dca200fa.query.consul. type=ANY class=IN latency=5.362507ms client=127.0.0.1:50468 client_network=udp +49:24.987 [DEBUG] TestAgent.dns: request served from client: name=06bacc69-5985-8b7f-113c-8b11dca200fa.query.consul. type=ANY class=IN latency=1.695797ms client=127.0.0.1:35089 client_network=udp +49:24.989 [DEBUG] TestAgent.dns: request served from client: name=06bacc69-5985-8b7f-113c-8b11dca200fa.query.consul. type=ANY class=IN latency=2.256492ms client=127.0.0.1:46918 client_network=udp +49:24.991 [DEBUG] TestAgent.dns: request served from client: name=06bacc69-5985-8b7f-113c-8b11dca200fa.query.consul. type=ANY class=IN latency=1.56177ms client=127.0.0.1:47386 client_network=udp +49:24.999 [DEBUG] TestAgent.dns: request served from client: name=06bacc69-5985-8b7f-113c-8b11dca200fa.query.consul. type=ANY class=IN latency=7.859063ms client=127.0.0.1:34796 client_network=udp +49:25.001 [DEBUG] TestAgent.dns: request served from client: name=06bacc69-5985-8b7f-113c-8b11dca200fa.query.consul. type=ANY class=IN latency=1.138234ms client=127.0.0.1:56652 client_network=udp +49:25.002 [DEBUG] TestAgent.dns: request served from client: name=06bacc69-5985-8b7f-113c-8b11dca200fa.query.consul. type=ANY class=IN latency=869.961µs client=127.0.0.1:51391 client_network=udp +49:25.003 [DEBUG] TestAgent.dns: request served from client: name=06bacc69-5985-8b7f-113c-8b11dca200fa.query.consul. type=ANY class=IN latency=1.418262ms client=127.0.0.1:47900 client_network=udp +49:25.003 [INFO] TestAgent: Requesting shutdown +49:25.004 [INFO] TestAgent.server: shutting down server +49:25.004 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.004 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.004 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.004 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.004 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.004 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.004 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.004 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.004 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.004 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.004 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:25.004Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.004Z [INFO] agent.router.manager: shutting down +49:25.004 [INFO] TestAgent: consul server down +49:25.004 [INFO] TestAgent: shutdown complete +49:25.004 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23743 network=tcp +49:25.004 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23743 network=udp +49:25.004 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23744 network=tcp +49:25.504 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.504 [INFO] TestAgent: Endpoints down +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=255)_compressed=true#01 +--- PASS: TestDNS_ServiceLookup_Randomize (1.03s) === CONT TestDNS_ServiceLookup_WanTranslation -Added 3 service nodes -2021-10-31T23:13:12.022Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:12.022Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:12.086Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:12.086Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:12.089 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:77c186ba-b7b5-3bfb-71a6-2dd8e0172498 Address:127.0.0.1:16963}]" -13:12.089 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16963 [Follower]" leader= -13:12.091 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-77c186ba-b7b5-3bfb-71a6-2dd8e0172498.dc1 127.0.0.1 -13:12.091 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-77c186ba-b7b5-3bfb-71a6-2dd8e0172498 127.0.0.1 -2021-10-31T23:13:12.092Z [INFO] agent.router: Initializing LAN area manager -13:12.142 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16951 network=tcp -13:12.146 [INFO] TestAgent.server: Adding LAN server: server="Node-77c186ba-b7b5-3bfb-71a6-2dd8e0172498 (Addr: tcp/127.0.0.1:16963) (DC: dc1)" -13:12.146 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16951 network=udp -13:12.146 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16952 network=tcp -13:12.146 [INFO] TestAgent: started state syncer -13:12.149 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16964 network=tcp -13:12.149 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:12.149 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16963 [Candidate]" term=2 -13:12.149 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-77c186ba-b7b5-3bfb-71a6-2dd8e0172498.dc1 area=wan -13:12.151 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:12.151 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:12.151 [DEBUG] TestAgent.server.raft: vote granted: from=77c186ba-b7b5-3bfb-71a6-2dd8e0172498 term=2 tally=1 -13:12.151 [INFO] TestAgent.server.raft: election won: tally=1 -13:12.151 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16963 [Leader]" -13:12.152 [INFO] TestAgent.server: cluster leadership acquired -13:12.152 [INFO] TestAgent.server: initializing acls -13:12.152 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:12.152 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:12.152 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:12.153 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:12.153 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:12.153 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:12.153 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-77c186ba-b7b5-3bfb-71a6-2dd8e0172498 -13:12.153 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-77c186ba-b7b5-3bfb-71a6-2dd8e0172498.dc1 -13:12.154 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:12.154 [INFO] TestAgent.server: New leader elected: payload=Node-77c186ba-b7b5-3bfb-71a6-2dd8e0172498 -13:12.154 [INFO] TestAgent.server: Updating LAN server: server="Node-77c186ba-b7b5-3bfb-71a6-2dd8e0172498 (Addr: tcp/127.0.0.1:16963) (DC: dc1)" -13:12.154 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-77c186ba-b7b5-3bfb-71a6-2dd8e0172498.dc1 area=wan -13:12.156 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:12.156 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:12.156 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:12.156 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:12.156 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:12.156 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-77c186ba-b7b5-3bfb-71a6-2dd8e0172498 -13:12.156 [INFO] TestAgent.server: member joined, marking health alive: member=Node-77c186ba-b7b5-3bfb-71a6-2dd8e0172498 -13:12.156 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-77c186ba-b7b5-3bfb-71a6-2dd8e0172498 -13:12.157 [INFO] TestAgent.server: federation state anti-entropy synced -13:12.458 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -13:12.458 [WARN] TestAgent: Node info update blocked by ACLs: node=77c186ba-b7b5-3bfb-71a6-2dd8e0172498 accessorID=00000000-0000-0000-0000-000000000002 -13:12.458 [DEBUG] TestAgent: Node info in sync -13:12.540 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-77c186ba-b7b5-3bfb-71a6-2dd8e0172498 -13:12.540 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-77c186ba-b7b5-3bfb-71a6-2dd8e0172498 -13:12.542 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=foo -13:12.542 [DEBUG] TestAgent.dns: request served from client: name=foo.service.consul. type=A class=IN latency=335.4µs client=127.0.0.1:49196 client_network=udp -13:12.542 [INFO] TestAgent: Requesting shutdown -13:12.542 [INFO] TestAgent.server: shutting down server -13:12.542 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:12.542 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:12.542 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:12.542 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:12.542 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:12.542 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:12.542 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.542 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:12.542 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:12.542 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:12.542 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:12.542 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:12.542 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.543Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.543Z [INFO] agent.router.manager: shutting down -13:12.543 [INFO] TestAgent: consul server down -13:12.543 [INFO] TestAgent: shutdown complete -13:12.543 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16951 network=tcp -13:12.543 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16951 network=udp -13:12.543 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16952 network=tcp -13:12.545 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="raft is already shutdown" -13:12.545 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:13.045 [INFO] TestAgent: Waiting for endpoints to shut down -13:13.045 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_FilterACL (2.18s) - --- PASS: TestDNS_ServiceLookup_FilterACL/ACLToken_==_root (1.12s) - --- PASS: TestDNS_ServiceLookup_FilterACL/ACLToken_==_anonymous (1.06s) -=== CONT TestDNS_Recurse -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:12.160Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:12.190Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:12.192 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:00a45c51-b988-f5ed-e638-7eb904f39b1c Address:127.0.0.1:16977}]" -13:12.192 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16977 [Follower]" leader= -13:12.198 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-00a45c51-b988-f5ed-e638-7eb904f39b1c.dc1 127.0.0.1 -13:12.199 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-00a45c51-b988-f5ed-e638-7eb904f39b1c 127.0.0.1 -2021-10-31T23:13:12.199Z [INFO] agent.router: Initializing LAN area manager -13:12.199 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16972 network=udp -13:12.199 [INFO] TestAgent.server: Adding LAN server: server="Node-00a45c51-b988-f5ed-e638-7eb904f39b1c (Addr: tcp/127.0.0.1:16977) (DC: dc1)" -13:12.199 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-00a45c51-b988-f5ed-e638-7eb904f39b1c.dc1 area=wan -13:12.199 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16972 network=tcp -13:12.200 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16973 network=tcp -13:12.200 [INFO] TestAgent: started state syncer -13:12.200 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16978 network=tcp -13:12.284 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:12.284 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16977 [Candidate]" term=2 -13:12.292 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:12.292 [DEBUG] TestAgent.server.raft: vote granted: from=00a45c51-b988-f5ed-e638-7eb904f39b1c term=2 tally=1 -13:12.292 [INFO] TestAgent.server.raft: election won: tally=1 -13:12.292 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16977 [Leader]" -13:12.294 [INFO] TestAgent.server: cluster leadership acquired -13:12.294 [INFO] TestAgent.server: New leader elected: payload=Node-00a45c51-b988-f5ed-e638-7eb904f39b1c -13:12.310 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16977 -13:12.343 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:12.345 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:12.345 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:12.345 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:12.345 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:12.345 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:12.345 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-00a45c51-b988-f5ed-e638-7eb904f39b1c -13:12.345 [INFO] TestAgent.server: member joined, marking health alive: member=Node-00a45c51-b988-f5ed-e638-7eb904f39b1c -13:12.346 [INFO] TestAgent.server: federation state anti-entropy synced -13:12.394 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:12.394 [INFO] TestAgent: Synced node info -13:12.619 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=303.583µs client=127.0.0.1:48322 client_network=udp -13:12.622 [DEBUG] TestAgent.dns: request served from client: name=cd02c634-9201-ac73-6f81-58ce2e8da33e.query.consul. type=ANY class=IN latency=227.827µs client=127.0.0.1:32840 client_network=udp -13:12.622 [INFO] TestAgent: Requesting shutdown -13:12.622 [INFO] TestAgent.server: shutting down server -13:12.622 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:12.622 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:12.622 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:12.622 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:12.622 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.622 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:12.622 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:12.622 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:12.622 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:12.622 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:12.622 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.623Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.623Z [INFO] agent.router.manager: shutting down -13:12.623 [INFO] TestAgent: consul server down -13:12.623 [INFO] TestAgent: shutdown complete -13:12.623 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16972 network=tcp -13:12.623 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16972 network=udp -13:12.623 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16973 network=tcp -13:13.125 [INFO] TestAgent: Waiting for endpoints to shut down -13:13.125 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_Dedup (1.10s) -=== CONT TestDNS_ServiceLookup_ServiceAddress_A -2021-10-31T23:13:10.970Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:11.083Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:11.085 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c8f8c837-0b91-2561-31c3-8145c90702c9 Address:127.0.0.1:16826}]" -13:11.085 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16826 [Follower]" leader= -13:11.086 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c8f8c837-0b91-2561-31c3-8145c90702c9.dc1 127.0.0.1 -13:11.087 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c8f8c837-0b91-2561-31c3-8145c90702c9 127.0.0.1 -2021-10-31T23:13:11.087Z [INFO] agent.router: Initializing LAN area manager -13:11.087 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16821 network=udp -13:11.087 [INFO] TestAgent.server: Adding LAN server: server="Node-c8f8c837-0b91-2561-31c3-8145c90702c9 (Addr: tcp/127.0.0.1:16826) (DC: dc1)" -13:11.087 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c8f8c837-0b91-2561-31c3-8145c90702c9.dc1 area=wan -13:11.088 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16821 network=tcp -13:11.088 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16822 network=tcp -13:11.088 [INFO] TestAgent: started state syncer -13:11.088 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16827 network=tcp -13:11.133 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:11.133 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16826 [Candidate]" term=2 -13:11.134 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:11.134 [DEBUG] TestAgent.server.raft: vote granted: from=c8f8c837-0b91-2561-31c3-8145c90702c9 term=2 tally=1 -13:11.134 [INFO] TestAgent.server.raft: election won: tally=1 -13:11.134 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16826 [Leader]" -13:11.134 [INFO] TestAgent.server: cluster leadership acquired -13:11.134 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16826 -13:11.135 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.135 [INFO] TestAgent.server: New leader elected: payload=Node-c8f8c837-0b91-2561-31c3-8145c90702c9 -13:11.136 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.136 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.136 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.136 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.136 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.137 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c8f8c837-0b91-2561-31c3-8145c90702c9 -13:11.137 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c8f8c837-0b91-2561-31c3-8145c90702c9 -13:11.146 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.274 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:11.274 [INFO] TestAgent: Synced node info -13:11.274 [DEBUG] TestAgent: Node info in sync -13:12.508 [WARN] TestAgent.server.rpc: Attempting to apply large raft entry: size_in_bytes=4719032 -13:12.637 [INFO] TestAgent: Requesting shutdown -13:12.637 [INFO] TestAgent.server: shutting down server -13:12.637 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:12.637 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:12.637 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:12.637 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:12.637 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.637 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:12.637 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:12.637 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:12.637 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:12.637 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:12.637 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.637Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.637Z [INFO] agent.router.manager: shutting down -13:12.637 [INFO] TestAgent: consul server down -13:12.637 [INFO] TestAgent: shutdown complete -13:12.637 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16821 network=tcp -13:12.637 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16821 network=udp -13:12.637 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16822 network=tcp -13:13.137 [INFO] TestAgent: Waiting for endpoints to shut down -13:13.137 [INFO] TestAgent: Endpoints down -=== RUN TestTxnEndpoint_Bad_Size_Net/allowed_kv_max_backward_compatible -2021-10-31T23:13:12.563Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:12.642Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:12.644 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef222217-666a-2d47-2a47-1f529ab67041 Address:127.0.0.1:17012}]" -13:12.645 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef222217-666a-2d47-2a47-1f529ab67041.dc1 127.0.0.1 -13:12.645 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef222217-666a-2d47-2a47-1f529ab67041 127.0.0.1 -2021-10-31T23:13:12.645Z [INFO] agent.router: Initializing LAN area manager -13:12.645 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17012 [Follower]" leader= -13:12.646 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17007 network=udp -13:12.646 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef222217-666a-2d47-2a47-1f529ab67041.dc1 area=wan -13:12.646 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17007 network=tcp -13:12.646 [INFO] TestAgent.server: Adding LAN server: server="Node-ef222217-666a-2d47-2a47-1f529ab67041 (Addr: tcp/127.0.0.1:17012) (DC: dc1)" -13:12.646 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17008 network=tcp -13:12.646 [INFO] TestAgent: started state syncer -13:12.647 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17013 network=tcp -13:12.683 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:12.683 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17012 [Candidate]" term=2 -13:12.683 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:12.683 [DEBUG] TestAgent.server.raft: vote granted: from=ef222217-666a-2d47-2a47-1f529ab67041 term=2 tally=1 -13:12.683 [INFO] TestAgent.server.raft: election won: tally=1 -13:12.683 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17012 [Leader]" -13:12.683 [INFO] TestAgent.server: cluster leadership acquired -13:12.683 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17012 -13:12.684 [INFO] TestAgent.server: New leader elected: payload=Node-ef222217-666a-2d47-2a47-1f529ab67041 -13:12.684 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:12.690 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:12.690 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:12.690 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:12.690 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:12.690 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:12.690 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef222217-666a-2d47-2a47-1f529ab67041 -13:12.690 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef222217-666a-2d47-2a47-1f529ab67041 -13:12.691 [INFO] TestAgent.server: federation state anti-entropy synced -13:12.710 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=SRV class=IN latency=258.501µs client=127.0.0.1:33811 client_network=udp -13:12.710 [INFO] TestAgent: Requesting shutdown -13:12.710 [INFO] TestAgent.server: shutting down server -13:12.710 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:12.710 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:12.710 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:12.710 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:12.710 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.711 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:12.711 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:12.711 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:12.711 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:12.711 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:12.711 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:12.711 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.711Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.711Z [INFO] agent.router.manager: shutting down -13:12.711 [INFO] TestAgent: consul server down -13:12.711 [INFO] TestAgent: shutdown complete -13:12.711 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17007 network=tcp -13:12.711 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17007 network=udp -13:12.711 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17008 network=tcp -13:13.213 [INFO] TestAgent: Waiting for endpoints to shut down -13:13.213 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_PreparedQueryNamePeriod (0.71s) +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=255)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=255)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=255)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=1)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=1)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=1)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=1)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=33)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=33)_compressed=false#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=33)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=33)_compressed=false#01 +49:24.503 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 +2022-12-04T05:49:24.503Z [INFO] agent.router: Initializing LAN area manager +49:24.503 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23736 network=udp +49:24.503 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23736 network=tcp +49:24.504 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23737 network=tcp +49:24.504 [INFO] TestAgent: started state syncer +49:24.504 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:23669] +49:24.504 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23693 network=tcp +2022-12-04T05:49:24.504Z [WARN] agent.router.manager: No servers available +49:24.504 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +49:24.504 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:23669 +49:24.509 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: test-server 127.0.0.1 +49:24.510 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +49:24.510 [DEBUG] TestAgent: systemd notify failed: error="No socket" +2022-12-04T05:49:24.510Z [WARN] agent.router.manager: No servers available +49:24.510 [INFO] TestAgent.client: adding server: server="test-server (Addr: tcp/127.0.0.1:23671) (DC: dc1)" +49:24.537 [ERROR] TestAgent.client: RPC failed to server: method=Health.ServiceNodes server=127.0.0.1:23671 error="rpc error making call: No path to datacenter" +49:24.537 [ERROR] TestAgent.dns: rpc error: error="rpc error making call: No path to datacenter" +49:24.537 [DEBUG] TestAgent.dns: request served from client: name=consul.service.dc2.consul. type=ANY class=IN latency=374.746µs client=127.0.0.1:52756 client_network=udp +49:24.537 [INFO] TestAgent: Requesting shutdown +49:24.538 [INFO] TestAgent.client: shutting down client +49:24.538 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +49:24.538 [INFO] TestAgent: consul client down +49:24.538 [INFO] TestAgent: shutdown complete +49:24.538 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23736 network=tcp +49:24.538 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23736 network=udp +49:24.538 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23737 network=tcp +49:25.038 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.038 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:24.335Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.346Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.348 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8ef529b6-0bae-abd6-d0ea-75cb388cc6e9 Address:127.0.0.1:23671}]" +49:24.348 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23671 [Follower]" leader= +49:24.349 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-server.dc1 127.0.0.1 +49:24.349 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-server 127.0.0.1 +2022-12-04T05:49:24.349Z [INFO] agent.router: Initializing LAN area manager +49:24.350 [INFO] TestAgent.server: Adding LAN server: server="test-server (Addr: tcp/127.0.0.1:23671) (DC: dc1)" +49:24.350 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23484 network=udp +49:24.350 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-server.dc1 area=wan +49:24.350 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23484 network=tcp +49:24.350 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23485 network=tcp +49:24.350 [INFO] TestAgent: started state syncer +49:24.350 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23672 network=tcp +49:24.396 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.396 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23671 [Candidate]" term=2 +49:24.396 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.396 [DEBUG] TestAgent.server.raft: vote granted: from=8ef529b6-0bae-abd6-d0ea-75cb388cc6e9 term=2 tally=1 +49:24.396 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.396 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23671 [Leader]" +49:24.396 [INFO] TestAgent.server: cluster leadership acquired +49:24.396 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23671 +49:24.397 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.397 [INFO] TestAgent.server: New leader elected: payload=test-server +49:24.398 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.398 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.398 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.398 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.398 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.398 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-server +49:24.398 [INFO] TestAgent.server: member joined, marking health alive: member=test-server +49:24.399 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.504 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:45014 +49:24.508 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-client 127.0.0.1 +49:24.509 [INFO] TestAgent.server: member joined, marking health alive: member=test-client +49:24.537 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc2 method=Health.ServiceNodes +49:24.628 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.629 [INFO] TestAgent: Synced node info +49:24.719 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.719 [DEBUG] TestAgent: Node info in sync +49:24.719 [DEBUG] TestAgent: Node info in sync +49:25.038 [INFO] TestAgent: Requesting shutdown +49:25.038 [INFO] TestAgent.server: shutting down server +49:25.038 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.038 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.038 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.038 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.038 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.040 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.040 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.040 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:25.040 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.040 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.040 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:49:25.040Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.040Z [INFO] agent.router.manager: shutting down +49:25.040 [INFO] TestAgent: consul server down +49:25.040 [INFO] TestAgent: shutdown complete +49:25.040 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23484 network=tcp +49:25.040 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23484 network=udp +49:25.040 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23485 network=tcp +49:25.541 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.541 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NonExistentDC_RPC (1.22s) +=== CONT TestDNS_IngressServiceLookup +2022-12-04T05:49:24.727Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.738Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.739 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5b4b5ee1-8db1-63ba-ba74-53fdfa3356aa Address:127.0.0.1:23699}]" +49:24.740 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23699 [Follower]" leader= +49:24.740 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5b4b5ee1-8db1-63ba-ba74-53fdfa3356aa.dc1 127.0.0.1 +49:24.741 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5b4b5ee1-8db1-63ba-ba74-53fdfa3356aa 127.0.0.1 +2022-12-04T05:49:24.741Z [INFO] agent.router: Initializing LAN area manager +49:24.741 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23708 network=udp +49:24.741 [INFO] TestAgent.server: Adding LAN server: server="Node-5b4b5ee1-8db1-63ba-ba74-53fdfa3356aa (Addr: tcp/127.0.0.1:23699) (DC: dc1)" +49:24.741 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5b4b5ee1-8db1-63ba-ba74-53fdfa3356aa.dc1 area=wan +49:24.741 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23708 network=tcp +49:24.742 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23709 network=tcp +49:24.742 [INFO] TestAgent: started state syncer +49:24.742 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23700 network=tcp +49:24.793 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.793 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23699 [Candidate]" term=2 +49:24.793 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.793 [DEBUG] TestAgent.server.raft: vote granted: from=5b4b5ee1-8db1-63ba-ba74-53fdfa3356aa term=2 tally=1 +49:24.793 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.793 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23699 [Leader]" +49:24.794 [INFO] TestAgent.server: cluster leadership acquired +49:24.794 [INFO] TestAgent.server: New leader elected: payload=Node-5b4b5ee1-8db1-63ba-ba74-53fdfa3356aa +49:24.794 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23699 +49:24.794 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.796 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.796 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.796 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.796 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.796 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.796 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5b4b5ee1-8db1-63ba-ba74-53fdfa3356aa +49:24.796 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5b4b5ee1-8db1-63ba-ba74-53fdfa3356aa +49:24.797 [INFO] TestAgent.server: federation state anti-entropy synced +49:25.044 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=ANY class=IN latency=1.059739ms client=127.0.0.1:48550 client_network=udp +49:25.045 [DEBUG] TestAgent.dns: request served from client: name=067a0cfc-b24d-8019-3c69-cf9a5e258f9d.query.consul. type=ANY class=IN latency=1.125111ms client=127.0.0.1:49129 client_network=udp +49:25.045 [INFO] TestAgent: Requesting shutdown +49:25.046 [INFO] TestAgent.server: shutting down server +49:25.046 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.046 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.046 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.046 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.046 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.046 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.046 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.046 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:25.046 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.046 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.046 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.046 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:25.046Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.046Z [INFO] agent.router.manager: shutting down +49:25.046 [INFO] TestAgent: consul server down +49:25.046 [INFO] TestAgent: shutdown complete +49:25.046 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23708 network=tcp +49:25.046 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23708 network=udp +49:25.046 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23709 network=tcp +49:25.546 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.546 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_Truncate (0.83s) === CONT TestDNS_ServiceLookup_ServiceAddressIPV6 -=== RUN TestDNS_Lookup_TaggedIPAddresses/ipv6-with-tagged-ipv4 +[WARN] freeport: 10 out of 87 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:24.734Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.744Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.745 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d81745ed-814c-06be-2e36-8a41aeadfd80 Address:127.0.0.1:23796}]" +49:24.745 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23796 [Follower]" leader= +49:24.745 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d81745ed-814c-06be-2e36-8a41aeadfd80.dc1 127.0.0.1 +49:24.746 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d81745ed-814c-06be-2e36-8a41aeadfd80 127.0.0.1 +2022-12-04T05:49:24.746Z [INFO] agent.router: Initializing LAN area manager +49:24.746 [INFO] TestAgent.server: Adding LAN server: server="Node-d81745ed-814c-06be-2e36-8a41aeadfd80 (Addr: tcp/127.0.0.1:23796) (DC: dc1)" +49:24.746 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d81745ed-814c-06be-2e36-8a41aeadfd80.dc1 area=wan +49:24.746 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23687 network=udp +49:24.746 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23687 network=tcp +49:24.747 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23688 network=tcp +49:24.747 [INFO] TestAgent: started state syncer +49:24.747 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23797 network=tcp +49:24.807 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.807 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23796 [Candidate]" term=2 +49:24.808 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.808 [DEBUG] TestAgent.server.raft: vote granted: from=d81745ed-814c-06be-2e36-8a41aeadfd80 term=2 tally=1 +49:24.808 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.808 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23796 [Leader]" +49:24.808 [INFO] TestAgent.server: cluster leadership acquired +49:24.808 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23796 +49:24.808 [INFO] TestAgent.server: New leader elected: payload=Node-d81745ed-814c-06be-2e36-8a41aeadfd80 +49:24.808 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.810 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.810 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.810 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.810 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.810 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.810 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d81745ed-814c-06be-2e36-8a41aeadfd80 +49:24.810 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d81745ed-814c-06be-2e36-8a41aeadfd80 +49:24.810 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.969 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=129.121µs client=127.0.0.1:57920 client_network=udp +49:24.995 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=212.496µs client=127.0.0.1:37359 client_network=udp +49:25.022 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=177.426µs client=127.0.0.1:40783 client_network=udp +49:25.052 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:25.052 [INFO] TestAgent: Synced node info +49:25.052 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=215.612µs client=127.0.0.1:42505 client_network=udp +49:25.053 [INFO] TestAgent: Requesting shutdown +49:25.053 [INFO] TestAgent.server: shutting down server +49:25.053 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.053 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.053 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.053 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.053 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.053 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.053 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.053 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.053 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.053 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.053 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:25.053Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.053Z [INFO] agent.router.manager: shutting down +49:25.053 [INFO] TestAgent: consul server down +49:25.053 [INFO] TestAgent: shutdown complete +49:25.053 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23687 network=tcp +49:25.053 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23687 network=udp +49:25.053 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23688 network=tcp +49:25.553 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.553 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_PreparedQueryNearIP (0.83s) +=== CONT TestDNS_ConnectServiceLookup +2022-12-04T05:49:24.666Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.675Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.677 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fbcd4d26-2e61-6438-2905-365699495963 Address:127.0.0.1:23824}]" +49:24.677 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23824 [Follower]" leader= +49:24.678 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fbcd4d26-2e61-6438-2905-365699495963.dc1 127.0.0.1 +49:24.678 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fbcd4d26-2e61-6438-2905-365699495963 127.0.0.1 +2022-12-04T05:49:24.678Z [INFO] agent.router: Initializing LAN area manager +49:24.678 [INFO] TestAgent.server: Adding LAN server: server="Node-fbcd4d26-2e61-6438-2905-365699495963 (Addr: tcp/127.0.0.1:23824) (DC: dc1)" +49:24.679 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fbcd4d26-2e61-6438-2905-365699495963.dc1 area=wan +49:24.679 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23791 network=udp +49:24.679 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23791 network=tcp +49:24.679 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23792 network=tcp +49:24.679 [INFO] TestAgent: started state syncer +49:24.679 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23825 network=tcp +49:24.738 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.738 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23824 [Candidate]" term=2 +49:24.739 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.739 [DEBUG] TestAgent.server.raft: vote granted: from=fbcd4d26-2e61-6438-2905-365699495963 term=2 tally=1 +49:24.739 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.739 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23824 [Leader]" +49:24.739 [INFO] TestAgent.server: cluster leadership acquired +49:24.739 [INFO] TestAgent.server: New leader elected: payload=Node-fbcd4d26-2e61-6438-2905-365699495963 +49:24.739 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23824 +49:24.740 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.741 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.741 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.741 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.741 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.741 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.741 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fbcd4d26-2e61-6438-2905-365699495963 +49:24.741 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fbcd4d26-2e61-6438-2905-365699495963 +49:24.742 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.869 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.869 [INFO] TestAgent: Synced node info +49:24.869 [DEBUG] TestAgent: Node info in sync +49:25.101 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=SRV class=IN latency=95.144µs client=127.0.0.1:54069 client_network=udp +49:25.101 [INFO] TestAgent: Requesting shutdown +49:25.101 [INFO] TestAgent.server: shutting down server +49:25.101 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.101 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.101 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.101 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.101 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.101 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.101 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.101 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.101 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.101 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:25.101Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.101Z [INFO] agent.router.manager: shutting down +49:25.102 [INFO] TestAgent: consul server down +49:25.102 [INFO] TestAgent: shutdown complete +49:25.102 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23791 network=tcp +49:25.102 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23791 network=udp +49:25.102 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23792 network=tcp +49:25.602 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.602 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_PreparedQueryNamePeriod (0.95s) +=== CONT TestDNS_ServiceLookupWithInternalServiceAddress === RUN TestDNS_Lookup_TaggedIPAddresses/simple-ipv4 === RUN TestDNS_Lookup_TaggedIPAddresses/simple-ipv6 === RUN TestDNS_Lookup_TaggedIPAddresses/ipv4-with-tagged-ipv6 -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:12.144Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:12.144Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:12.174Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:12.174Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:12.176 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1df75b70-68de-01ca-32ce-51ca0db327ed Address:127.0.0.1:16970}]" -13:12.176 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1df75b70-68de-01ca-32ce-51ca0db327ed.dc2 127.0.0.1 -13:12.177 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1df75b70-68de-01ca-32ce-51ca0db327ed 127.0.0.1 -2021-10-31T23:13:12.177Z [INFO] agent.router: Initializing LAN area manager -13:12.177 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16965 network=udp -13:12.177 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16970 [Follower]" leader= -13:12.177 [INFO] TestAgent.server: Adding LAN server: server="Node-1df75b70-68de-01ca-32ce-51ca0db327ed (Addr: tcp/127.0.0.1:16970) (DC: dc2)" -13:12.178 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16965 network=tcp -13:12.178 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16966 network=tcp -13:12.178 [INFO] TestAgent: started state syncer -13:12.178 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16971 network=tcp -13:12.185 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1df75b70-68de-01ca-32ce-51ca0db327ed.dc2 area=wan -13:12.228 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:12.229 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:12.230 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16970 [Candidate]" term=2 -13:12.230 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:12.230 [DEBUG] TestAgent.server.raft: vote granted: from=1df75b70-68de-01ca-32ce-51ca0db327ed term=2 tally=1 -13:12.230 [INFO] TestAgent.server.raft: election won: tally=1 -13:12.230 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16970 [Leader]" -13:12.230 [INFO] TestAgent.server: cluster leadership acquired -13:12.230 [INFO] TestAgent.server: initializing acls -13:12.230 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:12.231 [INFO] TestAgent.server: New leader elected: payload=Node-1df75b70-68de-01ca-32ce-51ca0db327ed -13:12.231 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:12.231 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:12.231 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:12.231 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1df75b70-68de-01ca-32ce-51ca0db327ed -13:12.231 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1df75b70-68de-01ca-32ce-51ca0db327ed.dc2 -13:12.232 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:12.233 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:12.233 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:12.233 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:12.233 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:12.233 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:12.233 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1df75b70-68de-01ca-32ce-51ca0db327ed -13:12.233 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1df75b70-68de-01ca-32ce-51ca0db327ed -13:12.233 [INFO] TestAgent.server: Updating LAN server: server="Node-1df75b70-68de-01ca-32ce-51ca0db327ed (Addr: tcp/127.0.0.1:16970) (DC: dc2)" -13:12.233 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1df75b70-68de-01ca-32ce-51ca0db327ed.dc2 area=wan -13:12.234 [INFO] TestAgent.server: federation state anti-entropy synced -13:12.234 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1df75b70-68de-01ca-32ce-51ca0db327ed -13:12.278 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:12.289 [INFO] TestAgent: Synced node info -13:12.339 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:16923] -13:12.340 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:16923 -13:12.341 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c7f43a55-7313-c302-cd93-be3fe9506c34.dc1 127.0.0.1 -13:12.341 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -13:12.342 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c7f43a55-7313-c302-cd93-be3fe9506c34.dc1 area=wan -13:12.355 [INFO] TestAgent: Requesting shutdown -13:12.355 [INFO] TestAgent.server: shutting down server -13:12.355 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:12.355 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:12.355 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:12.355 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:12.355 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:12.355 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:12.355 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.356 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:12.356 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:12.356 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:12.356 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:12.356 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:12.356 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:12.356 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:12.356 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.356Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.356Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.356Z [INFO] agent.router.manager: shutting down -13:12.356 [INFO] TestAgent: consul server down -13:12.356 [INFO] TestAgent: shutdown complete -13:12.356 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16965 network=tcp -13:12.356 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16965 network=udp -13:12.356 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16966 network=tcp -13:12.857 [INFO] TestAgent: Waiting for endpoints to shut down -13:12.857 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:11.759Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:11.759Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:11.802Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:11.802Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:11.804 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c7f43a55-7313-c302-cd93-be3fe9506c34 Address:127.0.0.1:16924}]" -13:11.804 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c7f43a55-7313-c302-cd93-be3fe9506c34.dc1 127.0.0.1 -13:11.805 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16924 [Follower]" leader= -13:11.805 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c7f43a55-7313-c302-cd93-be3fe9506c34 127.0.0.1 -2021-10-31T23:13:11.805Z [INFO] agent.router: Initializing LAN area manager -13:11.805 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16919 network=udp -13:11.805 [INFO] TestAgent.server: Adding LAN server: server="Node-c7f43a55-7313-c302-cd93-be3fe9506c34 (Addr: tcp/127.0.0.1:16924) (DC: dc1)" -13:11.805 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c7f43a55-7313-c302-cd93-be3fe9506c34.dc1 area=wan -13:11.805 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16919 network=tcp -13:11.806 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16920 network=tcp -13:11.806 [INFO] TestAgent: started state syncer -13:11.806 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16941 network=tcp -13:11.862 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:11.873 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:11.873 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16924 [Candidate]" term=2 -13:11.874 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:11.874 [DEBUG] TestAgent.server.raft: vote granted: from=c7f43a55-7313-c302-cd93-be3fe9506c34 term=2 tally=1 -13:11.874 [INFO] TestAgent.server.raft: election won: tally=1 -13:11.874 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16924 [Leader]" -13:11.874 [INFO] TestAgent.server: cluster leadership acquired -13:11.874 [INFO] TestAgent.server: initializing acls -13:11.874 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:11.875 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:11.875 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:11.875 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:11.875 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c7f43a55-7313-c302-cd93-be3fe9506c34 -13:11.875 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c7f43a55-7313-c302-cd93-be3fe9506c34.dc1 -13:11.875 [INFO] TestAgent.server: New leader elected: payload=Node-c7f43a55-7313-c302-cd93-be3fe9506c34 -13:11.875 [INFO] TestAgent.server: Updating LAN server: server="Node-c7f43a55-7313-c302-cd93-be3fe9506c34 (Addr: tcp/127.0.0.1:16924) (DC: dc1)" -13:11.875 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c7f43a55-7313-c302-cd93-be3fe9506c34.dc1 area=wan -13:11.876 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.877 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.877 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.877 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.877 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.877 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.877 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c7f43a55-7313-c302-cd93-be3fe9506c34 -13:11.877 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c7f43a55-7313-c302-cd93-be3fe9506c34 -13:11.878 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.878 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c7f43a55-7313-c302-cd93-be3fe9506c34 -13:11.990 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:11.990 [INFO] TestAgent: Synced node info -13:12.340 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:45850 -13:12.340 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1df75b70-68de-01ca-32ce-51ca0db327ed.dc2 127.0.0.1 -13:12.340 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1df75b70-68de-01ca-32ce-51ca0db327ed.dc2 area=wan -13:12.355 [DEBUG] TestAgent.dns: request served from client: name=my-query.query.consul. type=SRV class=IN latency=1.245441ms client=127.0.0.1:36941 client_network=udp -13:12.858 [INFO] TestAgent: Requesting shutdown -13:12.858 [INFO] TestAgent.server: shutting down server -13:12.858 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:12.858 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:12.858 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:12.858 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:12.858 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:12.858 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:12.858 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.858 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:12.858 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:12.858 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:12.858 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:12.858 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:12.858 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:12.858 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:12.858 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.858Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.858Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.858Z [INFO] agent.router.manager: shutting down -13:12.858 [INFO] TestAgent: consul server down -13:12.858 [INFO] TestAgent: shutdown complete -13:12.858 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16919 network=tcp -13:12.858 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16919 network=udp -13:12.858 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16920 network=tcp -13:13.358 [INFO] TestAgent: Waiting for endpoints to shut down -13:13.359 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_PreparedQuery_Failover (1.66s) -=== CONT TestDNS_ExternalServiceToConsulCNAMENestedLookup -2021-10-31T23:13:12.466Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:12.491Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:12.493 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c8d8f594-39b3-0977-1a84-1c62646d51b1 Address:127.0.0.1:17005}]" -13:12.493 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17005 [Follower]" leader= -13:12.506 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c8d8f594-39b3-0977-1a84-1c62646d51b1.dc1 127.0.0.1 -13:12.507 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c8d8f594-39b3-0977-1a84-1c62646d51b1 127.0.0.1 -2021-10-31T23:13:12.507Z [INFO] agent.router: Initializing LAN area manager -13:12.507 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17000 network=udp -13:12.507 [INFO] TestAgent.server: Adding LAN server: server="Node-c8d8f594-39b3-0977-1a84-1c62646d51b1 (Addr: tcp/127.0.0.1:17005) (DC: dc1)" -13:12.507 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c8d8f594-39b3-0977-1a84-1c62646d51b1.dc1 area=wan -13:12.507 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17000 network=tcp -13:12.508 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17001 network=tcp -13:12.508 [INFO] TestAgent: started state syncer -13:12.508 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17006 network=tcp -13:12.557 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:12.558 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17005 [Candidate]" term=2 -13:12.558 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:12.558 [DEBUG] TestAgent.server.raft: vote granted: from=c8d8f594-39b3-0977-1a84-1c62646d51b1 term=2 tally=1 -13:12.558 [INFO] TestAgent.server.raft: election won: tally=1 -13:12.558 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17005 [Leader]" -13:12.558 [INFO] TestAgent.server: cluster leadership acquired -13:12.558 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17005 -13:12.558 [INFO] TestAgent.server: New leader elected: payload=Node-c8d8f594-39b3-0977-1a84-1c62646d51b1 -13:12.559 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:12.568 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:12.568 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:12.569 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:12.569 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:12.569 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:12.569 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c8d8f594-39b3-0977-1a84-1c62646d51b1 -13:12.569 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c8d8f594-39b3-0977-1a84-1c62646d51b1 -13:12.577 [INFO] TestAgent.server: federation state anti-entropy synced -13:12.618 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:12.618 [INFO] TestAgent: Synced node info -13:12.618 [DEBUG] TestAgent: Node info in sync -13:12.769 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:12.769 [DEBUG] TestAgent: Node info in sync -13:12.882 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=226.751µs client=127.0.0.1:38462 client_network=udp -13:12.883 [DEBUG] TestAgent.dns: request served from client: name=cbb5d4ea-a7c6-633f-13e9-883d490d40b4.query.consul. type=ANY class=IN latency=206.113µs client=127.0.0.1:47892 client_network=udp -13:12.883 [INFO] TestAgent: Requesting shutdown -13:12.883 [INFO] TestAgent.server: shutting down server -13:12.883 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:12.883 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:12.883 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:12.883 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:12.883 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.883 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:12.883 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:12.883 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:12.883 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:12.883 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:12.883 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.883Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.883Z [INFO] agent.router.manager: shutting down -13:12.883 [INFO] TestAgent: consul server down -13:12.883 [INFO] TestAgent: shutdown complete -13:12.883 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17000 network=tcp -13:12.883 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17000 network=udp -13:12.883 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17001 network=tcp -13:13.385 [INFO] TestAgent: Waiting for endpoints to shut down -13:13.385 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_OnlyFailing (0.98s) -=== CONT TestDNS_ServiceLookup_ServiceAddress_SRV -2021-10-31T23:13:12.743Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:12.773Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:12.776 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cba0c0f8-2dc6-0d2a-1757-a1d9297390ba Address:127.0.0.1:17033}]" -13:12.777 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cba0c0f8-2dc6-0d2a-1757-a1d9297390ba.dc1 127.0.0.1 -13:12.777 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cba0c0f8-2dc6-0d2a-1757-a1d9297390ba 127.0.0.1 -2021-10-31T23:13:12.777Z [INFO] agent.router: Initializing LAN area manager -13:12.777 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17033 [Follower]" leader= -13:12.778 [INFO] TestAgent.server: Adding LAN server: server="Node-cba0c0f8-2dc6-0d2a-1757-a1d9297390ba (Addr: tcp/127.0.0.1:17033) (DC: dc1)" -13:12.778 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cba0c0f8-2dc6-0d2a-1757-a1d9297390ba.dc1 area=wan -13:12.782 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17028 network=udp -13:12.782 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17028 network=tcp -13:12.783 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17029 network=tcp -13:12.783 [INFO] TestAgent: started state syncer -13:12.783 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17034 network=tcp -13:12.817 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:12.817 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17033 [Candidate]" term=2 -13:12.818 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:12.818 [DEBUG] TestAgent.server.raft: vote granted: from=cba0c0f8-2dc6-0d2a-1757-a1d9297390ba term=2 tally=1 -13:12.818 [INFO] TestAgent.server.raft: election won: tally=1 -13:12.818 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17033 [Leader]" -13:12.818 [INFO] TestAgent.server: cluster leadership acquired -13:12.818 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17033 -13:12.818 [INFO] TestAgent.server: New leader elected: payload=Node-cba0c0f8-2dc6-0d2a-1757-a1d9297390ba -13:12.819 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:12.820 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:12.820 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:12.820 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:12.820 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:12.820 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:12.820 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cba0c0f8-2dc6-0d2a-1757-a1d9297390ba -13:12.820 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cba0c0f8-2dc6-0d2a-1757-a1d9297390ba -13:12.821 [INFO] TestAgent.server: federation state anti-entropy synced -13:12.865 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=209.468µs client=127.0.0.1:58117 client_network=udp -13:12.894 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=321.491µs client=127.0.0.1:53126 client_network=udp -13:12.894 [INFO] TestAgent: Requesting shutdown -13:12.894 [INFO] TestAgent.server: shutting down server -13:12.894 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:12.894 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:12.894 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:12.894 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:12.894 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.894 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:12.894 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:12.894 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:12.895 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:12.895 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:12.895 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:12.895 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.895Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.895Z [INFO] agent.router.manager: shutting down -13:12.895 [INFO] TestAgent: consul server down -13:12.895 [INFO] TestAgent: shutdown complete -13:12.895 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17028 network=tcp -13:12.895 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17028 network=udp -13:12.895 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17029 network=tcp -13:13.397 [INFO] TestAgent: Waiting for endpoints to shut down -13:13.397 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_PreparedQueryNearIPEDNS (0.68s) +=== RUN TestDNS_Lookup_TaggedIPAddresses/ipv6-with-tagged-ipv4 +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:24.580Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:24.580Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.589Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:24.589Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.591 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:344a617e-109b-96c9-4253-aaf636dda4d2 Address:127.0.0.1:23741}]" +49:24.591 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23741 [Follower]" leader= +49:24.591 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-344a617e-109b-96c9-4253-aaf636dda4d2.dc2 127.0.0.1 +49:24.591 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-344a617e-109b-96c9-4253-aaf636dda4d2 127.0.0.1 +2022-12-04T05:49:24.591Z [INFO] agent.router: Initializing LAN area manager +49:24.592 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-344a617e-109b-96c9-4253-aaf636dda4d2.dc2 area=wan +49:24.592 [INFO] TestAgent.server: Adding LAN server: server="Node-344a617e-109b-96c9-4253-aaf636dda4d2 (Addr: tcp/127.0.0.1:23741) (DC: dc2)" +49:24.592 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23784 network=udp +49:24.592 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23784 network=tcp +49:24.592 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23771 network=tcp +49:24.592 [INFO] TestAgent: started state syncer +49:24.592 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23742 network=tcp +49:24.628 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.628 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23741 [Candidate]" term=2 +49:24.628 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.628 [DEBUG] TestAgent.server.raft: vote granted: from=344a617e-109b-96c9-4253-aaf636dda4d2 term=2 tally=1 +49:24.628 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.628 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23741 [Leader]" +49:24.628 [INFO] TestAgent.server: cluster leadership acquired +49:24.628 [INFO] TestAgent.server: New leader elected: payload=Node-344a617e-109b-96c9-4253-aaf636dda4d2 +49:24.628 [INFO] TestAgent.server: initializing acls +49:24.628 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:24.629 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:24.629 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:24.629 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:24.629 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-344a617e-109b-96c9-4253-aaf636dda4d2 +49:24.629 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-344a617e-109b-96c9-4253-aaf636dda4d2.dc2 +49:24.629 [INFO] TestAgent.server: Updating LAN server: server="Node-344a617e-109b-96c9-4253-aaf636dda4d2 (Addr: tcp/127.0.0.1:23741) (DC: dc2)" +49:24.629 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-344a617e-109b-96c9-4253-aaf636dda4d2.dc2 area=wan +49:24.629 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.630 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.630 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.630 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.630 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.630 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.630 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-344a617e-109b-96c9-4253-aaf636dda4d2 +49:24.630 [INFO] TestAgent.server: member joined, marking health alive: member=Node-344a617e-109b-96c9-4253-aaf636dda4d2 +49:24.631 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.631 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-344a617e-109b-96c9-4253-aaf636dda4d2 +49:24.654 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.655 [INFO] TestAgent: Synced node info +49:24.845 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:23474] +49:24.846 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:23474 +49:24.846 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7f8b619a-288a-e26b-1ce8-f65ea56a6d3b.dc1 127.0.0.1 +49:24.846 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +49:24.847 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7f8b619a-288a-e26b-1ce8-f65ea56a6d3b.dc1 area=wan +49:24.849 [INFO] TestAgent: Requesting shutdown +49:24.849 [INFO] TestAgent.server: shutting down server +49:24.849 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:24.849 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:24.849 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:24.849 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:24.849 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:24.849 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:24.849 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:24.849 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:24.849 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:24.849 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:24.849 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:24.849 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:24.849 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:24.849 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:24.849 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:24.849Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.849Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:24.849Z [INFO] agent.router.manager: shutting down +49:24.849 [INFO] TestAgent: consul server down +49:24.849 [INFO] TestAgent: shutdown complete +49:24.849 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23784 network=tcp +49:24.849 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23784 network=udp +49:24.849 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23771 network=tcp +49:25.352 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.352 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:24.300Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:24.300Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:24.317Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:24.317Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:24.319 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7f8b619a-288a-e26b-1ce8-f65ea56a6d3b Address:127.0.0.1:23475}]" +49:24.319 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23475 [Follower]" leader= +49:24.319 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7f8b619a-288a-e26b-1ce8-f65ea56a6d3b.dc1 127.0.0.1 +49:24.320 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7f8b619a-288a-e26b-1ce8-f65ea56a6d3b 127.0.0.1 +2022-12-04T05:49:24.320Z [INFO] agent.router: Initializing LAN area manager +49:24.320 [INFO] TestAgent.server: Adding LAN server: server="Node-7f8b619a-288a-e26b-1ce8-f65ea56a6d3b (Addr: tcp/127.0.0.1:23475) (DC: dc1)" +49:24.320 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7f8b619a-288a-e26b-1ce8-f65ea56a6d3b.dc1 area=wan +49:24.320 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23477 network=udp +49:24.320 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23477 network=tcp +49:24.321 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23478 network=tcp +49:24.321 [INFO] TestAgent: started state syncer +49:24.321 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23476 network=tcp +49:24.370 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:24.384 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:24.384 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23475 [Candidate]" term=2 +49:24.384 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:24.384 [DEBUG] TestAgent.server.raft: vote granted: from=7f8b619a-288a-e26b-1ce8-f65ea56a6d3b term=2 tally=1 +49:24.384 [INFO] TestAgent.server.raft: election won: tally=1 +49:24.384 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23475 [Leader]" +49:24.384 [INFO] TestAgent.server: cluster leadership acquired +49:24.384 [INFO] TestAgent.server: initializing acls +49:24.384 [INFO] TestAgent.server: New leader elected: payload=Node-7f8b619a-288a-e26b-1ce8-f65ea56a6d3b +49:24.385 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:24.385 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:24.385 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:24.385 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:24.385 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7f8b619a-288a-e26b-1ce8-f65ea56a6d3b +49:24.385 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7f8b619a-288a-e26b-1ce8-f65ea56a6d3b.dc1 +49:24.385 [INFO] TestAgent.server: Updating LAN server: server="Node-7f8b619a-288a-e26b-1ce8-f65ea56a6d3b (Addr: tcp/127.0.0.1:23475) (DC: dc1)" +49:24.385 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7f8b619a-288a-e26b-1ce8-f65ea56a6d3b.dc1 area=wan +49:24.386 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:24.387 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:24.387 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:24.387 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:24.387 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:24.387 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:24.387 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7f8b619a-288a-e26b-1ce8-f65ea56a6d3b +49:24.387 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7f8b619a-288a-e26b-1ce8-f65ea56a6d3b +49:24.387 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7f8b619a-288a-e26b-1ce8-f65ea56a6d3b +49:24.387 [INFO] TestAgent.server: federation state anti-entropy synced +49:24.495 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:24.496 [INFO] TestAgent: Synced node info +49:24.846 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:58754 +49:24.846 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-344a617e-109b-96c9-4253-aaf636dda4d2.dc2 127.0.0.1 +49:24.846 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-344a617e-109b-96c9-4253-aaf636dda4d2.dc2 area=wan +49:24.849 [DEBUG] TestAgent.dns: request served from client: name=my-query.query.consul. type=SRV class=IN latency=1.271043ms client=127.0.0.1:52455 client_network=udp +49:25.352 [INFO] TestAgent: Requesting shutdown +49:25.353 [INFO] TestAgent.server: shutting down server +49:25.353 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.353 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.353 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.353 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:25.353 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:25.353 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.353 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.353 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.353 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.353 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.353 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.353 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.353 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:25.353 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:25.353 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:25.353Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.353Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.353Z [INFO] agent.router.manager: shutting down +49:25.353 [INFO] TestAgent: consul server down +49:25.353 [INFO] TestAgent: shutdown complete +49:25.353 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23477 network=tcp +49:25.353 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23477 network=udp +49:25.353 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23478 network=tcp +49:25.853 [INFO] TestAgent: Waiting for endpoints to shut down +49:25.853 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_PreparedQuery_Failover (1.56s) === CONT TestDNS_NSRecords_IPV6 -2021-10-31T23:13:12.752Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:12.798Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:12.800 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:48162f34-c545-fa0c-11b7-bf5e663742dc Address:127.0.0.1:17040}]" -13:12.801 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-48162f34-c545-fa0c-11b7-bf5e663742dc.dc1 127.0.0.1 -13:12.801 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-48162f34-c545-fa0c-11b7-bf5e663742dc 127.0.0.1 -2021-10-31T23:13:12.801Z [INFO] agent.router: Initializing LAN area manager -13:12.801 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17040 [Follower]" leader= -13:12.802 [INFO] TestAgent.server: Adding LAN server: server="Node-48162f34-c545-fa0c-11b7-bf5e663742dc (Addr: tcp/127.0.0.1:17040) (DC: dc1)" -13:12.802 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-48162f34-c545-fa0c-11b7-bf5e663742dc.dc1 area=wan -13:12.806 [DEBUG] TestAgent.dns: recursor enabled -13:12.806 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17035 network=udp -13:12.806 [DEBUG] TestAgent.dns: recursor enabled -13:12.806 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17035 network=tcp -13:12.807 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17036 network=tcp -13:12.807 [INFO] TestAgent: started state syncer -13:12.807 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17041 network=tcp -13:12.853 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:12.853 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17040 [Candidate]" term=2 -13:12.854 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:12.854 [DEBUG] TestAgent.server.raft: vote granted: from=48162f34-c545-fa0c-11b7-bf5e663742dc term=2 tally=1 -13:12.854 [INFO] TestAgent.server.raft: election won: tally=1 -13:12.854 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17040 [Leader]" -13:12.854 [INFO] TestAgent.server: cluster leadership acquired -13:12.854 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17040 -13:12.855 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:12.857 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:12.857 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:12.857 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:12.857 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:12.857 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:12.857 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-48162f34-c545-fa0c-11b7-bf5e663742dc -13:12.857 [INFO] TestAgent.server: member joined, marking health alive: member=Node-48162f34-c545-fa0c-11b7-bf5e663742dc -13:12.857 [INFO] TestAgent.server: New leader elected: payload=Node-48162f34-c545-fa0c-11b7-bf5e663742dc -13:12.863 [INFO] TestAgent.server: federation state anti-entropy synced -13:12.946 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{apple.com. 255 1}" rtt=85.751µs recursor=127.0.0.1:56928 -13:12.946 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=188.162µs client=127.0.0.1:48456 client_network=udp -13:12.946 [INFO] TestAgent: Requesting shutdown -13:12.946 [INFO] TestAgent.server: shutting down server -13:12.946 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:12.946 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:12.946 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:12.946 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:12.946 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:12.946 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:12.946 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:12.946 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:12.946 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:12.947 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:12.947 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:12.947 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:12.947Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:12.947Z [INFO] agent.router.manager: shutting down -13:12.947 [INFO] TestAgent: consul server down -13:12.947 [INFO] TestAgent: shutdown complete -13:12.947 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17035 network=tcp -13:12.947 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17035 network=udp -13:12.947 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17036 network=tcp -13:13.464 [INFO] TestAgent: Waiting for endpoints to shut down -13:13.464 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_Recurse_Truncation (0.74s) -=== CONT TestDNS_ServiceLookup_Dedup_SRV -2021-10-31T23:13:12.630Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:12.661Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:12.663 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:97e37e4e-9caa-758a-0ec4-971d114abbbc Address:127.0.0.1:17019}]" -13:12.664 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-97e37e4e-9caa-758a-0ec4-971d114abbbc.dc1 127.0.0.1 -13:12.664 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-97e37e4e-9caa-758a-0ec4-971d114abbbc 127.0.0.1 -2021-10-31T23:13:12.664Z [INFO] agent.router: Initializing LAN area manager -13:12.664 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17014 network=udp -13:12.664 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17019 [Follower]" leader= -13:12.665 [INFO] TestAgent.server: Adding LAN server: server="Node-97e37e4e-9caa-758a-0ec4-971d114abbbc (Addr: tcp/127.0.0.1:17019) (DC: dc1)" -13:12.665 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-97e37e4e-9caa-758a-0ec4-971d114abbbc.dc1 area=wan -13:12.665 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17014 network=tcp -13:12.665 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17015 network=tcp -13:12.666 [INFO] TestAgent: started state syncer -13:12.666 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17020 network=tcp -13:12.711 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:12.711 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17019 [Candidate]" term=2 -13:12.711 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:12.711 [DEBUG] TestAgent.server.raft: vote granted: from=97e37e4e-9caa-758a-0ec4-971d114abbbc term=2 tally=1 -13:12.711 [INFO] TestAgent.server.raft: election won: tally=1 -13:12.711 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17019 [Leader]" -13:12.712 [INFO] TestAgent.server: cluster leadership acquired -13:12.712 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17019 -13:12.713 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:12.713 [INFO] TestAgent.server: New leader elected: payload=Node-97e37e4e-9caa-758a-0ec4-971d114abbbc -13:12.714 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:12.714 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:12.714 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:12.714 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:12.714 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:12.714 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-97e37e4e-9caa-758a-0ec4-971d114abbbc -13:12.714 [INFO] TestAgent.server: member joined, marking health alive: member=Node-97e37e4e-9caa-758a-0ec4-971d114abbbc -13:12.715 [INFO] TestAgent.server: federation state anti-entropy synced -13:13.036 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=130.899µs client=127.0.0.1:35080 client_network=udp -13:13.053 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:13.054 [INFO] TestAgent: Synced node info -13:13.054 [DEBUG] TestAgent: Node info in sync -13:13.062 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=231.704µs client=127.0.0.1:36237 client_network=udp -13:13.062 [INFO] TestAgent: Requesting shutdown -13:13.062 [INFO] TestAgent.server: shutting down server -13:13.062 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:13.062 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:13.062 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:13.062 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:13.062 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:13.062 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:13.062 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:13.062 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:13.062 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:13.062 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:13.062 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:13.063Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:13.063Z [INFO] agent.router.manager: shutting down -13:13.063 [INFO] TestAgent: consul server down -13:13.063 [INFO] TestAgent: shutdown complete -13:13.063 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17014 network=tcp -13:13.063 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17014 network=udp -13:13.063 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17015 network=tcp -13:13.566 [INFO] TestAgent: Waiting for endpoints to shut down -13:13.566 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_PreparedQueryNearIP (1.03s) +=== RUN TestDNS_IngressServiceLookup/api.ingress.consul. +=== RUN TestDNS_IngressServiceLookup/api.ingress.dc1.consul. +=== RUN TestDNS_IngressServiceLookup/db.ingress.consul. +=== RUN TestDNS_IngressServiceLookup/db.ingress.dc1.consul. +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:25.389Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.417Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.418 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b8173ba7-9ac5-4b0f-f264-97d502be816c Address:127.0.0.1:23755}]" +49:25.418 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23755 [Follower]" leader= +49:25.418 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b8173ba7-9ac5-4b0f-f264-97d502be816c.dc1 127.0.0.1 +49:25.419 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b8173ba7-9ac5-4b0f-f264-97d502be816c 127.0.0.1 +2022-12-04T05:49:25.419Z [INFO] agent.router: Initializing LAN area manager +49:25.419 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23805 network=udp +49:25.419 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b8173ba7-9ac5-4b0f-f264-97d502be816c.dc1 area=wan +49:25.419 [INFO] TestAgent.server: Adding LAN server: server="Node-b8173ba7-9ac5-4b0f-f264-97d502be816c (Addr: tcp/127.0.0.1:23755) (DC: dc1)" +49:25.420 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23805 network=tcp +49:25.420 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23806 network=tcp +49:25.420 [INFO] TestAgent: started state syncer +49:25.420 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23756 network=tcp +49:25.460 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.460 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23755 [Candidate]" term=2 +49:25.460 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.460 [DEBUG] TestAgent.server.raft: vote granted: from=b8173ba7-9ac5-4b0f-f264-97d502be816c term=2 tally=1 +49:25.460 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.460 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23755 [Leader]" +49:25.460 [INFO] TestAgent.server: cluster leadership acquired +49:25.460 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23755 +49:25.461 [INFO] TestAgent.server: New leader elected: payload=Node-b8173ba7-9ac5-4b0f-f264-97d502be816c +49:25.461 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.462 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.462 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.462 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.462 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.462 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.462 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b8173ba7-9ac5-4b0f-f264-97d502be816c +49:25.462 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b8173ba7-9ac5-4b0f-f264-97d502be816c +49:25.463 [INFO] TestAgent.server: federation state anti-entropy synced +49:25.549 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=180.275µs client=127.0.0.1:38270 client_network=udp +49:25.553 [DEBUG] TestAgent.dns: request served from client: name=f3ada8e3-7533-1303-b5ba-728c5e1c095e.query.consul. type=SRV class=IN latency=189.715µs client=127.0.0.1:43026 client_network=udp +49:25.554 [INFO] TestAgent: Requesting shutdown +49:25.554 [INFO] TestAgent.server: shutting down server +49:25.554 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.554 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.554 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.554 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.554 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.554 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.554 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.554 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:25.554 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.554 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.554 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.554 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:25.554Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.554Z [INFO] agent.router.manager: shutting down +49:25.554 [INFO] TestAgent: consul server down +49:25.554 [INFO] TestAgent: shutdown complete +49:25.554 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23805 network=tcp +49:25.554 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23805 network=udp +49:25.554 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23806 network=tcp +49:26.055 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.055 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_ServiceAddress_A (0.68s) +=== CONT TestDNS_ServiceLookup +2022-12-04T05:49:25.132Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.141Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.142 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:17b31335-5723-2b5b-5fd2-4cc039ab68a8 Address:127.0.0.1:23831}]" +49:25.143 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23831 [Follower]" leader= +49:25.143 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-17b31335-5723-2b5b-5fd2-4cc039ab68a8.dc1 127.0.0.1 +49:25.143 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-17b31335-5723-2b5b-5fd2-4cc039ab68a8 127.0.0.1 +2022-12-04T05:49:25.143Z [INFO] agent.router: Initializing LAN area manager +49:25.143 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-17b31335-5723-2b5b-5fd2-4cc039ab68a8.dc1 area=wan +49:25.143 [INFO] TestAgent.server: Adding LAN server: server="Node-17b31335-5723-2b5b-5fd2-4cc039ab68a8 (Addr: tcp/127.0.0.1:23831) (DC: dc1)" +49:25.144 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23854 network=udp +49:25.144 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23854 network=tcp +49:25.144 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23869 network=tcp +49:25.144 [INFO] TestAgent: started state syncer +49:25.144 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23832 network=tcp +49:25.184 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.184 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23831 [Candidate]" term=2 +49:25.184 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.184 [DEBUG] TestAgent.server.raft: vote granted: from=17b31335-5723-2b5b-5fd2-4cc039ab68a8 term=2 tally=1 +49:25.184 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.184 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23831 [Leader]" +49:25.184 [INFO] TestAgent.server: cluster leadership acquired +49:25.184 [INFO] TestAgent.server: New leader elected: payload=Node-17b31335-5723-2b5b-5fd2-4cc039ab68a8 +49:25.184 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23831 +49:25.185 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.187 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.187 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.187 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.187 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.187 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.187 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-17b31335-5723-2b5b-5fd2-4cc039ab68a8 +49:25.187 [INFO] TestAgent.server: member joined, marking health alive: member=Node-17b31335-5723-2b5b-5fd2-4cc039ab68a8 +49:25.188 [INFO] TestAgent.server: federation state anti-entropy synced +49:25.364 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:25.365 [INFO] TestAgent: Synced node info +49:25.503 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=227.28µs client=127.0.0.1:56130 client_network=udp +49:25.529 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=238.465µs client=127.0.0.1:44897 client_network=udp +49:25.555 [DEBUG] TestAgent.dns: request served from client: name=some.query.we.like.query.consul. type=A class=IN latency=214.046µs client=127.0.0.1:41193 client_network=udp +49:25.555 [INFO] TestAgent: Requesting shutdown +49:25.555 [INFO] TestAgent.server: shutting down server +49:25.555 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.555 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.555 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.555 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.555 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.555 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.555 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.555 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.555 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.555 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.555 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:25.555Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.555Z [INFO] agent.router.manager: shutting down +49:25.555 [INFO] TestAgent: consul server down +49:25.555 [INFO] TestAgent: shutdown complete +49:25.555 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23854 network=tcp +49:25.555 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23854 network=udp +49:25.555 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23869 network=tcp +49:26.056 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.057 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_PreparedQueryNearIPEDNS (0.93s) === CONT TestDNS_NSRecords -=== RUN TestDNS_ServiceLookup_WanTranslation/service-addr-from-dc2 -[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_ServiceLookup_WanTranslation/service-wan-from-dc2 +2022-12-04T05:49:25.369Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.378Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.380 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e11ed268-5b97-59a8-117f-afb57bcbeb62 Address:127.0.0.1:23817}]" +49:25.380 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23817 [Follower]" leader= +49:25.381 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e11ed268-5b97-59a8-117f-afb57bcbeb62.dc1 127.0.0.1 +49:25.381 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e11ed268-5b97-59a8-117f-afb57bcbeb62 127.0.0.1 +2022-12-04T05:49:25.381Z [INFO] agent.router: Initializing LAN area manager +49:25.381 [INFO] TestAgent.server: Adding LAN server: server="Node-e11ed268-5b97-59a8-117f-afb57bcbeb62 (Addr: tcp/127.0.0.1:23817) (DC: dc1)" +49:25.381 [DEBUG] TestAgent.dns: recursor enabled +49:25.381 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e11ed268-5b97-59a8-117f-afb57bcbeb62.dc1 area=wan +49:25.381 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23847 network=udp +49:25.381 [DEBUG] TestAgent.dns: recursor enabled +49:25.381 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23847 network=tcp +49:25.382 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23848 network=tcp +49:25.382 [INFO] TestAgent: started state syncer +49:25.382 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23818 network=tcp +49:25.452 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.452 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23817 [Candidate]" term=2 +49:25.452 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.452 [DEBUG] TestAgent.server.raft: vote granted: from=e11ed268-5b97-59a8-117f-afb57bcbeb62 term=2 tally=1 +49:25.452 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.452 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23817 [Leader]" +49:25.452 [INFO] TestAgent.server: cluster leadership acquired +49:25.452 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23817 +49:25.453 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.454 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.454 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.454 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.454 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.454 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.454 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e11ed268-5b97-59a8-117f-afb57bcbeb62 +49:25.454 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e11ed268-5b97-59a8-117f-afb57bcbeb62 +49:25.455 [INFO] TestAgent.server: New leader elected: payload=Node-e11ed268-5b97-59a8-117f-afb57bcbeb62 +49:25.455 [INFO] TestAgent.server: federation state anti-entropy synced +49:25.569 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{apple.com. 255 1}" rtt=112.038µs recursor=127.0.0.1:50734 +49:25.569 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=203.769µs client=127.0.0.1:42988 client_network=udp +49:25.570 [INFO] TestAgent: Requesting shutdown +49:25.570 [INFO] TestAgent.server: shutting down server +49:25.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.570 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.570 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.570 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.571 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.571 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.571 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:25.571 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.571 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.571 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.571 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:25.572 [INFO] TestAgent: consul server down +49:25.572 [INFO] TestAgent: shutdown complete +49:25.572 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23847 network=tcp +49:25.572 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23847 network=udp +49:25.572 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23848 network=tcp +2022-12-04T05:49:25.572Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.572Z [INFO] agent.router.manager: shutting down +49:26.073 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.073 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_Recurse_Truncation (0.72s) +=== CONT TestDNS_ServiceLookup_ServiceAddress_SRV +2022-12-04T05:49:25.379Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.398Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.405 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9d005ef8-b779-e619-7091-aa81fb1b0f06 Address:127.0.0.1:23803}]" +49:25.406 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23803 [Follower]" leader= +49:25.406 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9d005ef8-b779-e619-7091-aa81fb1b0f06.dc1 127.0.0.1 +49:25.407 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9d005ef8-b779-e619-7091-aa81fb1b0f06 127.0.0.1 +2022-12-04T05:49:25.407Z [INFO] agent.router: Initializing LAN area manager +49:25.408 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9d005ef8-b779-e619-7091-aa81fb1b0f06.dc1 area=wan +49:25.408 [INFO] TestAgent.server: Adding LAN server: server="Node-9d005ef8-b779-e619-7091-aa81fb1b0f06 (Addr: tcp/127.0.0.1:23803) (DC: dc1)" +49:25.409 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23819 network=tcp +49:25.409 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23819 network=udp +49:25.410 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23820 network=tcp +49:25.410 [INFO] TestAgent: started state syncer +49:25.410 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23804 network=tcp +49:25.455 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.455 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23803 [Candidate]" term=2 +49:25.455 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.455 [DEBUG] TestAgent.server.raft: vote granted: from=9d005ef8-b779-e619-7091-aa81fb1b0f06 term=2 tally=1 +49:25.455 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.455 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23803 [Leader]" +49:25.455 [INFO] TestAgent.server: cluster leadership acquired +49:25.455 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23803 +49:25.456 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.456 [INFO] TestAgent.server: New leader elected: payload=Node-9d005ef8-b779-e619-7091-aa81fb1b0f06 +49:25.457 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.457 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.457 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.457 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.457 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.457 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9d005ef8-b779-e619-7091-aa81fb1b0f06 +49:25.457 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9d005ef8-b779-e619-7091-aa81fb1b0f06 +49:25.457 [INFO] TestAgent.server: federation state anti-entropy synced +49:25.568 [DEBUG] TestAgent.dns: request served from client: name=primary.db.service.consul. type=SRV class=IN latency=149.334µs client=127.0.0.1:33380 client_network=udp +49:25.568 [DEBUG] TestAgent.dns: request served from client: name=pRIMARY.dB.service.consul. type=SRV class=IN latency=83.42µs client=127.0.0.1:52389 client_network=udp +49:25.568 [DEBUG] TestAgent.dns: request served from client: name=PRIMARY.dB.service.consul. type=SRV class=IN latency=75.139µs client=127.0.0.1:49405 client_network=udp +49:25.569 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=91.577µs client=127.0.0.1:33136 client_network=udp +49:25.569 [DEBUG] TestAgent.dns: request served from client: name=DB.service.consul. type=SRV class=IN latency=82.245µs client=127.0.0.1:37130 client_network=udp +49:25.569 [DEBUG] TestAgent.dns: request served from client: name=Db.service.consul. type=SRV class=IN latency=73.289µs client=127.0.0.1:47758 client_network=udp +49:25.569 [DEBUG] TestAgent.dns: request served from client: name=somequery.query.consul. type=SRV class=IN latency=88.134µs client=127.0.0.1:50068 client_network=udp +49:25.570 [DEBUG] TestAgent.dns: request served from client: name=SomeQuery.query.consul. type=SRV class=IN latency=195.977µs client=127.0.0.1:53241 client_network=udp +49:25.573 [DEBUG] TestAgent.dns: request served from client: name=SOMEQUERY.query.consul. type=SRV class=IN latency=209.165µs client=127.0.0.1:60104 client_network=udp +49:25.573 [INFO] TestAgent: Requesting shutdown +49:25.573 [INFO] TestAgent.server: shutting down server +49:25.573 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.573 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.573 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.573 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.573 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.573 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.573 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.573 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.574 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:25.574 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.574 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.574 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +2022-12-04T05:49:25.574Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.574Z [INFO] agent.router.manager: shutting down +49:25.574 [INFO] TestAgent: consul server down +49:25.574 [INFO] TestAgent: shutdown complete +49:25.574 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23819 network=tcp +49:25.574 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23819 network=udp +49:25.574 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23820 network=tcp +49:26.078 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.078 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_CaseInsensitiveServiceLookup (0.71s) +=== CONT TestDNS_ServiceLookupMultiAddrNoCNAME === RUN TestDNS_ServiceLookup_WanTranslation/node-addr-from-dc1 === RUN TestDNS_ServiceLookup_WanTranslation/node-wan-from-dc1 === RUN TestDNS_ServiceLookup_WanTranslation/service-addr-from-dc1 -2021-10-31T23:13:12.711Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:12.741Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:12.747 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:25fe5a74-480b-a227-246e-c510dcd9318b Address:127.0.0.1:17026}]" -13:12.748 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-25fe5a74-480b-a227-246e-c510dcd9318b.dc1 127.0.0.1 -13:12.748 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-25fe5a74-480b-a227-246e-c510dcd9318b 127.0.0.1 -2021-10-31T23:13:12.748Z [INFO] agent.router: Initializing LAN area manager -13:12.748 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17021 network=udp -13:12.748 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17026 [Follower]" leader= -13:12.749 [INFO] TestAgent.server: Adding LAN server: server="Node-25fe5a74-480b-a227-246e-c510dcd9318b (Addr: tcp/127.0.0.1:17026) (DC: dc1)" -13:12.749 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-25fe5a74-480b-a227-246e-c510dcd9318b.dc1 area=wan -13:12.749 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17021 network=tcp -13:12.749 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17022 network=tcp -13:12.749 [INFO] TestAgent: started state syncer -13:12.749 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17027 network=tcp -13:12.789 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:12.789 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17026 [Candidate]" term=2 -13:12.789 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:12.789 [DEBUG] TestAgent.server.raft: vote granted: from=25fe5a74-480b-a227-246e-c510dcd9318b term=2 tally=1 -13:12.789 [INFO] TestAgent.server.raft: election won: tally=1 -13:12.789 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17026 [Leader]" -13:12.789 [INFO] TestAgent.server: cluster leadership acquired -13:12.789 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17026 -13:12.789 [INFO] TestAgent.server: New leader elected: payload=Node-25fe5a74-480b-a227-246e-c510dcd9318b -13:12.790 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:12.791 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:12.792 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:12.792 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:12.792 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:12.792 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:12.792 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-25fe5a74-480b-a227-246e-c510dcd9318b -13:12.792 [INFO] TestAgent.server: member joined, marking health alive: member=Node-25fe5a74-480b-a227-246e-c510dcd9318b -13:12.792 [INFO] TestAgent.server: federation state anti-entropy synced -13:12.978 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:12.979 [INFO] TestAgent: Synced node info -13:12.979 [DEBUG] TestAgent: Node info in sync -13:13.146 [DEBUG] TestAgent.dns: request served from client: name=primary.db.service.consul. type=SRV class=IN latency=239.692µs client=127.0.0.1:43576 client_network=udp -13:13.154 [DEBUG] TestAgent.dns: request served from client: name=pRIMARY.dB.service.consul. type=SRV class=IN latency=211.338µs client=127.0.0.1:46729 client_network=udp -13:13.154 [DEBUG] TestAgent.dns: request served from client: name=PRIMARY.dB.service.consul. type=SRV class=IN latency=90.666µs client=127.0.0.1:33295 client_network=udp -13:13.154 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=85.198µs client=127.0.0.1:43321 client_network=udp -13:13.154 [DEBUG] TestAgent.dns: request served from client: name=DB.service.consul. type=SRV class=IN latency=81.085µs client=127.0.0.1:48842 client_network=udp -13:13.155 [DEBUG] TestAgent.dns: request served from client: name=Db.service.consul. type=SRV class=IN latency=82.848µs client=127.0.0.1:52795 client_network=udp -13:13.155 [DEBUG] TestAgent.dns: request served from client: name=somequery.query.consul. type=SRV class=IN latency=103.736µs client=127.0.0.1:44137 client_network=udp -13:13.155 [DEBUG] TestAgent.dns: request served from client: name=SomeQuery.query.consul. type=SRV class=IN latency=124.073µs client=127.0.0.1:40326 client_network=udp -13:13.155 [DEBUG] TestAgent.dns: request served from client: name=SOMEQUERY.query.consul. type=SRV class=IN latency=71.623µs client=127.0.0.1:32849 client_network=udp -13:13.155 [INFO] TestAgent: Requesting shutdown -13:13.155 [INFO] TestAgent.server: shutting down server -13:13.155 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:13.155 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:13.155 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:13.155 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:13.155 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:13.156 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:13.156 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:13.156 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:13.156 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:13.156 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:13.156 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:13.156Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:13.156Z [INFO] agent.router.manager: shutting down -13:13.156 [INFO] TestAgent: consul server down -13:13.156 [INFO] TestAgent: shutdown complete -13:13.156 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17021 network=tcp -13:13.156 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17021 network=udp -13:13.156 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17022 network=tcp -13:13.657 [INFO] TestAgent: Waiting for endpoints to shut down -13:13.657 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_CaseInsensitiveServiceLookup (0.98s) -=== CONT TestDNS_ExternalServiceToConsulCNAMELookup === RUN TestDNS_ServiceLookup_WanTranslation/service-wan-from-dc1 === RUN TestDNS_ServiceLookup_WanTranslation/node-addr-from-dc2 === RUN TestDNS_ServiceLookup_WanTranslation/node-wan-from-dc2 -2021-10-31T23:13:12.880Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:12.913Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:12.935 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6857bbb5-f24f-3ee2-2cb6-72a8225296a1 Address:127.0.0.1:17047}]" -13:12.936 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6857bbb5-f24f-3ee2-2cb6-72a8225296a1.dc1 127.0.0.1 -13:12.937 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6857bbb5-f24f-3ee2-2cb6-72a8225296a1 127.0.0.1 -2021-10-31T23:13:12.937Z [INFO] agent.router: Initializing LAN area manager -13:12.937 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17042 network=udp -13:12.937 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17047 [Follower]" leader= -13:12.939 [INFO] TestAgent.server: Adding LAN server: server="Node-6857bbb5-f24f-3ee2-2cb6-72a8225296a1 (Addr: tcp/127.0.0.1:17047) (DC: dc1)" -13:12.939 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6857bbb5-f24f-3ee2-2cb6-72a8225296a1.dc1 area=wan -13:12.939 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17042 network=tcp -13:12.941 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17043 network=tcp -13:12.941 [INFO] TestAgent: started state syncer -13:12.941 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17048 network=tcp -13:12.973 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:12.973 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17047 [Candidate]" term=2 -13:12.974 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:12.974 [DEBUG] TestAgent.server.raft: vote granted: from=6857bbb5-f24f-3ee2-2cb6-72a8225296a1 term=2 tally=1 -13:12.974 [INFO] TestAgent.server.raft: election won: tally=1 -13:12.974 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17047 [Leader]" -13:12.974 [INFO] TestAgent.server: cluster leadership acquired -13:12.974 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17047 -13:12.975 [INFO] TestAgent.server: New leader elected: payload=Node-6857bbb5-f24f-3ee2-2cb6-72a8225296a1 -13:12.975 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:12.977 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:12.977 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:12.977 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:12.977 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:12.977 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:12.978 [INFO] TestAgent.server: federation state anti-entropy synced -13:12.985 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6857bbb5-f24f-3ee2-2cb6-72a8225296a1 -13:12.986 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6857bbb5-f24f-3ee2-2cb6-72a8225296a1 -13:13.002 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:13.002 [INFO] TestAgent: Synced node info -13:13.231 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=120.984µs client=127.0.0.1:44407 client_network=udp -13:13.231 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=AAAA class=IN latency=84.845µs client=127.0.0.1:33462 client_network=udp -13:13.232 [DEBUG] TestAgent.dns: request served from client: name=b89b6e31-7cdc-c0c4-e178-1e7a22c19a67.query.consul. type=A class=IN latency=103.592µs client=127.0.0.1:57128 client_network=udp -13:13.232 [DEBUG] TestAgent.dns: request served from client: name=b89b6e31-7cdc-c0c4-e178-1e7a22c19a67.query.consul. type=AAAA class=IN latency=93.492µs client=127.0.0.1:36110 client_network=udp -13:13.232 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=63.319µs client=127.0.0.1:50582 client_network=udp -13:13.232 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=AAAA class=IN latency=55.777µs client=127.0.0.1:39961 client_network=udp -13:13.238 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=4.432063ms client=127.0.0.1:43383 client_network=udp -13:13.238 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=AAAA class=IN latency=3.996432ms client=127.0.0.1:45898 client_network=udp -13:13.238 [DEBUG] TestAgent.dns: request served from client: name=b89b6e31-7cdc-c0c4-e178-1e7a22c19a67.query.consul. type=A class=IN latency=219.914µs client=127.0.0.1:56487 client_network=udp -13:13.242 [DEBUG] TestAgent.dns: request served from client: name=b89b6e31-7cdc-c0c4-e178-1e7a22c19a67.query.consul. type=AAAA class=IN latency=156.42µs client=127.0.0.1:45637 client_network=udp -13:13.242 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=67.658µs client=127.0.0.1:40893 client_network=udp -13:13.242 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=AAAA class=IN latency=42.415µs client=127.0.0.1:45564 client_network=udp -13:13.243 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=72.476µs client=127.0.0.1:34926 client_network=udp -13:13.243 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=AAAA class=IN latency=84.989µs client=127.0.0.1:57433 client_network=udp -13:13.244 [DEBUG] TestAgent.dns: request served from client: name=b89b6e31-7cdc-c0c4-e178-1e7a22c19a67.query.consul. type=A class=IN latency=84.243µs client=127.0.0.1:47020 client_network=udp -13:13.244 [DEBUG] TestAgent.dns: request served from client: name=b89b6e31-7cdc-c0c4-e178-1e7a22c19a67.query.consul. type=AAAA class=IN latency=74.397µs client=127.0.0.1:44846 client_network=udp -13:13.244 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=45.503µs client=127.0.0.1:43072 client_network=udp -13:13.244 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=AAAA class=IN latency=42.07µs client=127.0.0.1:54126 client_network=udp -13:13.245 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=61.942µs client=127.0.0.1:60348 client_network=udp -13:13.245 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=AAAA class=IN latency=51.54µs client=127.0.0.1:33954 client_network=udp -13:13.245 [DEBUG] TestAgent.dns: request served from client: name=b89b6e31-7cdc-c0c4-e178-1e7a22c19a67.query.consul. type=A class=IN latency=120.168µs client=127.0.0.1:34300 client_network=udp -13:13.246 [DEBUG] TestAgent.dns: request served from client: name=b89b6e31-7cdc-c0c4-e178-1e7a22c19a67.query.consul. type=AAAA class=IN latency=249.904µs client=127.0.0.1:47975 client_network=udp -13:13.246 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=51.703µs client=127.0.0.1:56863 client_network=udp -13:13.247 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=AAAA class=IN latency=62.393µs client=127.0.0.1:60060 client_network=udp -13:13.247 [INFO] TestAgent: Requesting shutdown -13:13.247 [INFO] TestAgent.server: shutting down server -13:13.247 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:13.247 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:13.247 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:13.247 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:13.247 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:13.247 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:13.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:13.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:13.247 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:13.247 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:13.247 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:13.247Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:13.247Z [INFO] agent.router.manager: shutting down -13:13.247 [INFO] TestAgent: consul server down -13:13.247 [INFO] TestAgent: shutdown complete -13:13.247 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17042 network=tcp -13:13.247 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17042 network=udp -13:13.247 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17043 network=tcp -13:13.750 [INFO] TestAgent: Waiting for endpoints to shut down -13:13.750 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_Lookup_TaggedIPAddresses (0.90s) - --- PASS: TestDNS_Lookup_TaggedIPAddresses/ipv6-with-tagged-ipv4 (0.00s) - --- PASS: TestDNS_Lookup_TaggedIPAddresses/simple-ipv4 (0.01s) +=== RUN TestDNS_ServiceLookup_WanTranslation/service-addr-from-dc2 +=== RUN TestDNS_ServiceLookup_WanTranslation/service-wan-from-dc2 +2022-12-04T05:49:25.343Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.358Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.360 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:17e1e0d1-2c96-c221-657e-26b79693f15e Address:127.0.0.1:23845}]" +49:25.360 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23845 [Follower]" leader= +49:25.365 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-17e1e0d1-2c96-c221-657e-26b79693f15e.dc1 127.0.0.1 +49:25.365 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-17e1e0d1-2c96-c221-657e-26b79693f15e 127.0.0.1 +2022-12-04T05:49:25.365Z [INFO] agent.router: Initializing LAN area manager +49:25.365 [INFO] TestAgent.server: Adding LAN server: server="Node-17e1e0d1-2c96-c221-657e-26b79693f15e (Addr: tcp/127.0.0.1:23845) (DC: dc1)" +49:25.365 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-17e1e0d1-2c96-c221-657e-26b79693f15e.dc1 area=wan +49:25.365 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23840 network=udp +49:25.365 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23840 network=tcp +49:25.366 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23841 network=tcp +49:25.366 [INFO] TestAgent: started state syncer +49:25.366 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23846 network=tcp +49:25.423 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.423 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23845 [Candidate]" term=2 +49:25.423 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.423 [DEBUG] TestAgent.server.raft: vote granted: from=17e1e0d1-2c96-c221-657e-26b79693f15e term=2 tally=1 +49:25.423 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.423 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23845 [Leader]" +49:25.423 [INFO] TestAgent.server: cluster leadership acquired +49:25.423 [INFO] TestAgent.server: New leader elected: payload=Node-17e1e0d1-2c96-c221-657e-26b79693f15e +49:25.423 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23845 +49:25.424 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.425 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.425 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.425 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.425 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.425 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.425 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-17e1e0d1-2c96-c221-657e-26b79693f15e +49:25.425 [INFO] TestAgent.server: member joined, marking health alive: member=Node-17e1e0d1-2c96-c221-657e-26b79693f15e +49:25.426 [INFO] TestAgent.server: federation state anti-entropy synced +49:25.698 [DEBUG] TestAgent.dns: request served from client: name=v1.primary2.db.service.consul. type=SRV class=IN latency=59.999µs client=127.0.0.1:53349 client_network=udp +49:25.698 [DEBUG] TestAgent.dns: request served from client: name=v1.primary.db.service.consul. type=SRV class=IN latency=61.507µs client=127.0.0.1:40982 client_network=udp +49:25.698 [INFO] TestAgent: Requesting shutdown +49:25.698 [INFO] TestAgent.server: shutting down server +49:25.698 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.698 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.698 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.698 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.698 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.698 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.698 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.698 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:25.698 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.698 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.698 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.698 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:25.698Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.698Z [INFO] agent.router.manager: shutting down +49:25.698 [INFO] TestAgent: consul server down +49:25.698 [INFO] TestAgent: shutdown complete +49:25.698 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23840 network=tcp +49:25.698 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23840 network=udp +49:25.698 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23841 network=tcp +49:26.199 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.199 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_TagPeriod (0.86s) +=== CONT TestDNS_ServiceLookupPreferNoCNAME +2022-12-04T05:49:25.565Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.584Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.586 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0eb1d5f0-0493-fca3-710d-ffd8ea2e3f99 Address:127.0.0.1:23887}]" +49:25.586 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23887 [Follower]" leader= +49:25.586 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0eb1d5f0-0493-fca3-710d-ffd8ea2e3f99.dc1 127.0.0.1 +49:25.587 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0eb1d5f0-0493-fca3-710d-ffd8ea2e3f99 127.0.0.1 +2022-12-04T05:49:25.587Z [INFO] agent.router: Initializing LAN area manager +49:25.587 [INFO] TestAgent.server: Adding LAN server: server="Node-0eb1d5f0-0493-fca3-710d-ffd8ea2e3f99 (Addr: tcp/127.0.0.1:23887) (DC: dc1)" +49:25.587 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0eb1d5f0-0493-fca3-710d-ffd8ea2e3f99.dc1 area=wan +49:25.587 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23882 network=udp +49:25.587 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23882 network=tcp +49:25.588 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23862 network=tcp +49:25.588 [INFO] TestAgent: started state syncer +49:25.588 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23888 network=tcp +49:25.654 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.654 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23887 [Candidate]" term=2 +49:25.654 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.654 [DEBUG] TestAgent.server.raft: vote granted: from=0eb1d5f0-0493-fca3-710d-ffd8ea2e3f99 term=2 tally=1 +49:25.654 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.654 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23887 [Leader]" +49:25.654 [INFO] TestAgent.server: cluster leadership acquired +49:25.654 [INFO] TestAgent.server: New leader elected: payload=Node-0eb1d5f0-0493-fca3-710d-ffd8ea2e3f99 +49:25.654 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23887 +49:25.655 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.656 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.656 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.656 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.656 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.656 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.656 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0eb1d5f0-0493-fca3-710d-ffd8ea2e3f99 +49:25.656 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0eb1d5f0-0493-fca3-710d-ffd8ea2e3f99 +49:25.657 [INFO] TestAgent.server: federation state anti-entropy synced +49:25.659 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:25.659 [INFO] TestAgent: Synced node info +49:25.744 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=267.206µs client=127.0.0.1:37566 client_network=udp +49:25.748 [DEBUG] TestAgent.dns: request served from client: name=260b809c-d382-3cf3-3384-573ba599743a.query.consul. type=SRV class=IN latency=232.043µs client=127.0.0.1:59804 client_network=udp +49:25.748 [INFO] TestAgent: Requesting shutdown +49:25.748 [INFO] TestAgent.server: shutting down server +49:25.748 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.748 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.748 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.748 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.748 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.749 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.749 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.749 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.749 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.749 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.749 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:25.749Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.749Z [INFO] agent.router.manager: shutting down +49:25.749 [INFO] TestAgent: consul server down +49:25.749 [INFO] TestAgent: shutdown complete +49:25.749 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23882 network=tcp +49:25.749 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23882 network=udp +49:25.749 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23862 network=tcp +49:26.249 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.249 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_ServiceAddressIPV6 (0.70s) +=== CONT TestDNS_ExternalServiceToConsulCNAMELookup +2022-12-04T05:49:25.507Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.517Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.518 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5d9c1dca-f789-f0c5-437c-5c6c772f08b0 Address:127.0.0.1:23873}]" +49:25.518 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23873 [Follower]" leader= +49:25.519 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5d9c1dca-f789-f0c5-437c-5c6c772f08b0.dc1 127.0.0.1 +49:25.519 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5d9c1dca-f789-f0c5-437c-5c6c772f08b0 127.0.0.1 +2022-12-04T05:49:25.519Z [INFO] agent.router: Initializing LAN area manager +49:25.519 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23896 network=udp +49:25.519 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5d9c1dca-f789-f0c5-437c-5c6c772f08b0.dc1 area=wan +49:25.519 [INFO] TestAgent.server: Adding LAN server: server="Node-5d9c1dca-f789-f0c5-437c-5c6c772f08b0 (Addr: tcp/127.0.0.1:23873) (DC: dc1)" +49:25.519 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23896 network=tcp +49:25.520 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23883 network=tcp +49:25.520 [INFO] TestAgent: started state syncer +49:25.520 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23874 network=tcp +49:25.582 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.582 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23873 [Candidate]" term=2 +49:25.582 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.582 [DEBUG] TestAgent.server.raft: vote granted: from=5d9c1dca-f789-f0c5-437c-5c6c772f08b0 term=2 tally=1 +49:25.582 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.582 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23873 [Leader]" +49:25.582 [INFO] TestAgent.server: cluster leadership acquired +49:25.582 [INFO] TestAgent.server: New leader elected: payload=Node-5d9c1dca-f789-f0c5-437c-5c6c772f08b0 +49:25.582 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23873 +49:25.583 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.584 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.584 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.584 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.584 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.584 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.584 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5d9c1dca-f789-f0c5-437c-5c6c772f08b0 +49:25.584 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5d9c1dca-f789-f0c5-437c-5c6c772f08b0 +49:25.585 [INFO] TestAgent.server: federation state anti-entropy synced +49:25.751 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=189.236µs client=127.0.0.1:56716 client_network=udp +49:25.752 [INFO] TestAgent: Requesting shutdown +49:25.752 [INFO] TestAgent.server: shutting down server +49:25.752 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.752 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.752 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.752 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.752 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.752 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.752 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.752 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:25.752 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.752 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.752 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.752 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:25.752Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.752Z [INFO] agent.router.manager: shutting down +49:25.752 [INFO] TestAgent: consul server down +49:25.752 [INFO] TestAgent: shutdown complete +49:25.752 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23896 network=tcp +49:25.752 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23896 network=udp +49:25.752 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23883 network=tcp +49:26.252 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.252 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ExternalServiceLookup (0.76s) +=== CONT TestDNS_ServiceReverseLookup_IPV6 +2022-12-04T05:49:25.480Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.490Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.491 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f3fd861f-9bf1-39db-cabe-89e005e0286b Address:127.0.0.1:23894}]" +49:25.492 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23894 [Follower]" leader= +49:25.492 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:25.492 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:25.492Z [INFO] agent.router: Initializing LAN area manager +49:25.493 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23812 network=udp +49:25.493 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23894) (DC: dc1)" +49:25.493 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:25.493 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23812 network=tcp +49:25.493 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23813 network=tcp +49:25.493 [INFO] TestAgent: started state syncer +49:25.493 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23895 network=tcp +49:25.535 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.535 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23894 [Candidate]" term=2 +49:25.535 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.535 [DEBUG] TestAgent.server.raft: vote granted: from=f3fd861f-9bf1-39db-cabe-89e005e0286b term=2 tally=1 +49:25.535 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.535 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23894 [Leader]" +49:25.535 [INFO] TestAgent.server: cluster leadership acquired +49:25.535 [INFO] TestAgent.server: New leader elected: payload=test-node +49:25.535 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23894 +49:25.536 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.537 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.537 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.537 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.537 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.537 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.537 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:25.537 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:25.538 [INFO] TestAgent.server: federation state anti-entropy synced +49:25.731 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:25.732 [INFO] TestAgent: Synced node info +49:25.760 [DEBUG] TestAgent.dns: request served from client: name=alias2.service.consul. type=SRV class=IN latency=320.981µs client=127.0.0.1:37022 client_network=udp +49:25.760 [INFO] TestAgent: Requesting shutdown +49:25.760 [INFO] TestAgent.server: shutting down server +49:25.760 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.760 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.760 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.760 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.760 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.761 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.761 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.761 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.761 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.761 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.761 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:25.761Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.761Z [INFO] agent.router.manager: shutting down +49:25.761 [INFO] TestAgent: consul server down +49:25.761 [INFO] TestAgent: shutdown complete +49:25.761 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23812 network=tcp +49:25.761 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23812 network=udp +49:25.761 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23813 network=tcp +49:26.261 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.261 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ExternalServiceToConsulCNAMENestedLookup (0.79s) +=== CONT TestDNS_ServiceLookupNoMultiCNAME +2022-12-04T05:49:25.417Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.426Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.428 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3bdc6746-f7c5-f53b-a347-2926541ee2de Address:127.0.0.1:23810}]" +49:25.428 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23810 [Follower]" leader= +49:25.428 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3bdc6746-f7c5-f53b-a347-2926541ee2de.dc1 127.0.0.1 +49:25.428 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3bdc6746-f7c5-f53b-a347-2926541ee2de 127.0.0.1 +2022-12-04T05:49:25.428Z [INFO] agent.router: Initializing LAN area manager +49:25.429 [INFO] TestAgent.server: Adding LAN server: server="Node-3bdc6746-f7c5-f53b-a347-2926541ee2de (Addr: tcp/127.0.0.1:23810) (DC: dc1)" +49:25.429 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3bdc6746-f7c5-f53b-a347-2926541ee2de.dc1 area=wan +49:25.429 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23757 network=udp +49:25.429 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23757 network=tcp +49:25.429 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23758 network=tcp +49:25.429 [INFO] TestAgent: started state syncer +49:25.429 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23811 network=tcp +49:25.484 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.484 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23810 [Candidate]" term=2 +49:25.484 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.484 [DEBUG] TestAgent.server.raft: vote granted: from=3bdc6746-f7c5-f53b-a347-2926541ee2de term=2 tally=1 +49:25.484 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.484 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23810 [Leader]" +49:25.484 [INFO] TestAgent.server: cluster leadership acquired +49:25.484 [INFO] TestAgent.server: New leader elected: payload=Node-3bdc6746-f7c5-f53b-a347-2926541ee2de +49:25.484 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23810 +49:25.485 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.487 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.487 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.487 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.487 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.487 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.487 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3bdc6746-f7c5-f53b-a347-2926541ee2de +49:25.487 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3bdc6746-f7c5-f53b-a347-2926541ee2de +49:25.487 [INFO] TestAgent.server: federation state anti-entropy synced +49:25.660 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:25.660 [INFO] TestAgent: Synced node info +49:25.660 [DEBUG] TestAgent: Node info in sync +49:25.777 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=182.344µs client=127.0.0.1:35149 client_network=udp +49:25.778 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=AAAA class=IN latency=70.569µs client=127.0.0.1:44148 client_network=udp +49:25.778 [DEBUG] TestAgent.dns: request served from client: name=a997727a-5612-7f2a-4bb4-3dff280595b0.query.consul. type=A class=IN latency=97.865µs client=127.0.0.1:45400 client_network=udp +49:25.778 [DEBUG] TestAgent.dns: request served from client: name=a997727a-5612-7f2a-4bb4-3dff280595b0.query.consul. type=AAAA class=IN latency=71.906µs client=127.0.0.1:40764 client_network=udp +49:25.778 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=52.985µs client=127.0.0.1:39584 client_network=udp +49:25.779 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=AAAA class=IN latency=43.704µs client=127.0.0.1:59167 client_network=udp +49:25.779 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=76.231µs client=127.0.0.1:55080 client_network=udp +49:25.779 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=AAAA class=IN latency=64.086µs client=127.0.0.1:51205 client_network=udp +49:25.780 [DEBUG] TestAgent.dns: request served from client: name=a997727a-5612-7f2a-4bb4-3dff280595b0.query.consul. type=A class=IN latency=80.789µs client=127.0.0.1:41836 client_network=udp +49:25.780 [DEBUG] TestAgent.dns: request served from client: name=a997727a-5612-7f2a-4bb4-3dff280595b0.query.consul. type=AAAA class=IN latency=87.296µs client=127.0.0.1:49151 client_network=udp +49:25.780 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=45.895µs client=127.0.0.1:51068 client_network=udp +49:25.780 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=AAAA class=IN latency=40.417µs client=127.0.0.1:58498 client_network=udp +49:25.781 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=49.27µs client=127.0.0.1:42337 client_network=udp +49:25.781 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=AAAA class=IN latency=62.445µs client=127.0.0.1:46402 client_network=udp +49:25.782 [DEBUG] TestAgent.dns: request served from client: name=a997727a-5612-7f2a-4bb4-3dff280595b0.query.consul. type=A class=IN latency=79.208µs client=127.0.0.1:53864 client_network=udp +49:25.782 [DEBUG] TestAgent.dns: request served from client: name=a997727a-5612-7f2a-4bb4-3dff280595b0.query.consul. type=AAAA class=IN latency=75.71µs client=127.0.0.1:41845 client_network=udp +49:25.782 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=81.901µs client=127.0.0.1:48958 client_network=udp +49:25.782 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=AAAA class=IN latency=67.562µs client=127.0.0.1:56403 client_network=udp +49:25.783 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=82.831µs client=127.0.0.1:56347 client_network=udp +49:25.783 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=AAAA class=IN latency=90.479µs client=127.0.0.1:56309 client_network=udp +49:25.783 [DEBUG] TestAgent.dns: request served from client: name=a997727a-5612-7f2a-4bb4-3dff280595b0.query.consul. type=A class=IN latency=76.09µs client=127.0.0.1:60212 client_network=udp +49:25.784 [DEBUG] TestAgent.dns: request served from client: name=a997727a-5612-7f2a-4bb4-3dff280595b0.query.consul. type=AAAA class=IN latency=60.434µs client=127.0.0.1:46895 client_network=udp +49:25.784 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=57.856µs client=127.0.0.1:48756 client_network=udp +49:25.784 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=AAAA class=IN latency=38.031µs client=127.0.0.1:49682 client_network=udp +49:25.784 [INFO] TestAgent: Requesting shutdown +49:25.784 [INFO] TestAgent.server: shutting down server +49:25.784 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.784 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.784 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.784 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.784 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.784 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.784 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.784 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.784 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.784 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.784 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:25.784Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.784Z [INFO] agent.router.manager: shutting down +49:25.785 [INFO] TestAgent: consul server down +49:25.785 [INFO] TestAgent: shutdown complete +49:25.785 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23757 network=tcp +49:25.785 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23757 network=udp +49:25.785 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23758 network=tcp +49:26.285 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.285 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_Lookup_TaggedIPAddresses (0.88s) + --- PASS: TestDNS_Lookup_TaggedIPAddresses/simple-ipv4 (0.00s) --- PASS: TestDNS_Lookup_TaggedIPAddresses/simple-ipv6 (0.00s) --- PASS: TestDNS_Lookup_TaggedIPAddresses/ipv4-with-tagged-ipv6 (0.00s) -=== CONT TestDNS_ConnectServiceLookup -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:13.093Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:13.132Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:13.138 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b8e5449a-d2db-1d09-b097-fb02180c24c8 Address:127.0.0.1:17068}]" -13:13.139 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b8e5449a-d2db-1d09-b097-fb02180c24c8.dc1 127.0.0.1 -13:13.139 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b8e5449a-d2db-1d09-b097-fb02180c24c8 127.0.0.1 -2021-10-31T23:13:13.139Z [INFO] agent.router: Initializing LAN area manager -13:13.139 [DEBUG] TestAgent.dns: recursor enabled -13:13.139 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17063 network=udp -13:13.139 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17068 [Follower]" leader= -13:13.140 [INFO] TestAgent.server: Adding LAN server: server="Node-b8e5449a-d2db-1d09-b097-fb02180c24c8 (Addr: tcp/127.0.0.1:17068) (DC: dc1)" -13:13.140 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b8e5449a-d2db-1d09-b097-fb02180c24c8.dc1 area=wan -13:13.140 [DEBUG] TestAgent.dns: recursor enabled -13:13.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17063 network=tcp -13:13.140 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17064 network=tcp -13:13.141 [INFO] TestAgent: started state syncer -13:13.141 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17069 network=tcp -13:13.197 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:13.197 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17068 [Candidate]" term=2 -13:13.197 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:13.197 [DEBUG] TestAgent.server.raft: vote granted: from=b8e5449a-d2db-1d09-b097-fb02180c24c8 term=2 tally=1 -13:13.197 [INFO] TestAgent.server.raft: election won: tally=1 -13:13.197 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17068 [Leader]" -13:13.197 [INFO] TestAgent.server: cluster leadership acquired -13:13.198 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17068 -13:13.198 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:13.200 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:13.200 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:13.200 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:13.200 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:13.200 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:13.200 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b8e5449a-d2db-1d09-b097-fb02180c24c8 -13:13.200 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b8e5449a-d2db-1d09-b097-fb02180c24c8 -13:13.200 [INFO] TestAgent.server: New leader elected: payload=Node-b8e5449a-d2db-1d09-b097-fb02180c24c8 -13:13.201 [INFO] TestAgent.server: federation state anti-entropy synced -13:13.335 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:13.335 [INFO] TestAgent: Synced node info -13:13.335 [DEBUG] TestAgent: Node info in sync -13:13.397 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{apple.com. 255 1}" rtt=173.517µs recursor=127.0.0.1:45377 -13:13.397 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=397.942µs client=127.0.0.1:50301 client_network=udp -13:13.397 [INFO] TestAgent: Requesting shutdown -13:13.397 [INFO] TestAgent.server: shutting down server -13:13.397 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:13.397 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:13.397 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:13.397 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:13.397 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:13.398 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:13.398 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:13.398 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:13.398 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:13.398 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:13.398 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:13.398Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:13.398Z [INFO] agent.router.manager: shutting down -13:13.398 [INFO] TestAgent: consul server down -13:13.398 [INFO] TestAgent: shutdown complete -13:13.398 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17063 network=tcp -13:13.398 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17063 network=udp -13:13.398 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17064 network=tcp -13:13.899 [INFO] TestAgent: Waiting for endpoints to shut down -13:13.899 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_Recurse (0.85s) -=== CONT TestDNS_ServiceLookupMultiAddrNoCNAME -2021-10-31T23:13:12.893Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:12.994Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:13.010 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:00d92723-a6d7-ea25-b00f-fcbc077dbc2a Address:127.0.0.1:17054}]" -13:13.011 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-00d92723-a6d7-ea25-b00f-fcbc077dbc2a.dc1 127.0.0.1 -13:13.012 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-00d92723-a6d7-ea25-b00f-fcbc077dbc2a 127.0.0.1 -2021-10-31T23:13:13.012Z [INFO] agent.router: Initializing LAN area manager -13:13.012 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17049 network=udp -13:13.012 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17054 [Follower]" leader= -13:13.013 [INFO] TestAgent.server: Adding LAN server: server="Node-00d92723-a6d7-ea25-b00f-fcbc077dbc2a (Addr: tcp/127.0.0.1:17054) (DC: dc1)" -13:13.013 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-00d92723-a6d7-ea25-b00f-fcbc077dbc2a.dc1 area=wan -13:13.013 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17049 network=tcp -13:13.029 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17050 network=tcp -13:13.029 [INFO] TestAgent: started state syncer -13:13.034 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17055 network=tcp -13:13.053 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:13.054 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17054 [Candidate]" term=2 -13:13.054 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:13.054 [DEBUG] TestAgent.server.raft: vote granted: from=00d92723-a6d7-ea25-b00f-fcbc077dbc2a term=2 tally=1 -13:13.054 [INFO] TestAgent.server.raft: election won: tally=1 -13:13.054 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17054 [Leader]" -13:13.054 [INFO] TestAgent.server: cluster leadership acquired -13:13.054 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17054 -13:13.055 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:13.054 [INFO] TestAgent.server: New leader elected: payload=Node-00d92723-a6d7-ea25-b00f-fcbc077dbc2a -13:13.056 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:13.056 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:13.056 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:13.056 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:13.056 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:13.056 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-00d92723-a6d7-ea25-b00f-fcbc077dbc2a -13:13.056 [INFO] TestAgent.server: member joined, marking health alive: member=Node-00d92723-a6d7-ea25-b00f-fcbc077dbc2a -13:13.056 [INFO] TestAgent.server: federation state anti-entropy synced -13:13.165 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:13.166 [INFO] TestAgent: Synced node info -13:13.166 [DEBUG] TestAgent: Node info in sync -13:13.422 [DEBUG] TestAgent.dns: request served from client: name=v1.primary2.db.service.consul. type=SRV class=IN latency=178.036µs client=127.0.0.1:37082 client_network=udp -13:13.422 [DEBUG] TestAgent.dns: request served from client: name=v1.primary.db.service.consul. type=SRV class=IN latency=94.141µs client=127.0.0.1:46031 client_network=udp -13:13.422 [INFO] TestAgent: Requesting shutdown -13:13.422 [INFO] TestAgent.server: shutting down server -13:13.422 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:13.422 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:13.422 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:13.422 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:13.422 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:13.422 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:13.422 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:13.422 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:13.422 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:13.422 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:13.422 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:13.423Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:13.423Z [INFO] agent.router.manager: shutting down -13:13.423 [INFO] TestAgent: consul server down -13:13.423 [INFO] TestAgent: shutdown complete -13:13.423 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17049 network=tcp -13:13.423 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17049 network=udp -13:13.423 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17050 network=tcp -13:13.925 [INFO] TestAgent: Waiting for endpoints to shut down -13:13.925 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_TagPeriod (1.06s) + --- PASS: TestDNS_Lookup_TaggedIPAddresses/ipv6-with-tagged-ipv4 (0.00s) +=== CONT TestDNS_ServiceReverseLookup +[WARN] freeport: 10 out of 73 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:25.582Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.593Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.595 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:525385e0-f08c-d1ff-2d69-a1bbc9bd34fa Address:127.0.0.1:23936}]" +49:25.595 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23936 [Follower]" leader= +49:25.596 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-525385e0-f08c-d1ff-2d69-a1bbc9bd34fa.dc1 127.0.0.1 +49:25.596 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-525385e0-f08c-d1ff-2d69-a1bbc9bd34fa 127.0.0.1 +2022-12-04T05:49:25.596Z [INFO] agent.router: Initializing LAN area manager +49:25.596 [INFO] TestAgent.server: Adding LAN server: server="Node-525385e0-f08c-d1ff-2d69-a1bbc9bd34fa (Addr: tcp/127.0.0.1:23936) (DC: dc1)" +49:25.596 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-525385e0-f08c-d1ff-2d69-a1bbc9bd34fa.dc1 area=wan +49:25.596 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23889 network=udp +49:25.597 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23889 network=tcp +49:25.597 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23897 network=tcp +49:25.597 [INFO] TestAgent: started state syncer +49:25.597 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23937 network=tcp +49:25.637 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.637 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23936 [Candidate]" term=2 +49:25.637 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.637 [DEBUG] TestAgent.server.raft: vote granted: from=525385e0-f08c-d1ff-2d69-a1bbc9bd34fa term=2 tally=1 +49:25.637 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.637 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23936 [Leader]" +49:25.637 [INFO] TestAgent.server: cluster leadership acquired +49:25.637 [INFO] TestAgent.server: New leader elected: payload=Node-525385e0-f08c-d1ff-2d69-a1bbc9bd34fa +49:25.637 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23936 +49:25.638 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.639 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.639 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.639 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.639 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.639 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.639 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-525385e0-f08c-d1ff-2d69-a1bbc9bd34fa +49:25.639 [INFO] TestAgent.server: member joined, marking health alive: member=Node-525385e0-f08c-d1ff-2d69-a1bbc9bd34fa +49:25.640 [INFO] TestAgent.server: federation state anti-entropy synced +49:25.695 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:25.696 [INFO] TestAgent: Synced node info +49:25.748 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:25.748 [DEBUG] TestAgent: Node info in sync +49:25.748 [DEBUG] TestAgent: Node info in sync +49:25.814 [DEBUG] TestAgent.dns: request served from client: name=db.connect.consul. type=SRV class=IN latency=199.002µs client=127.0.0.1:48940 client_network=udp +49:25.814 [INFO] TestAgent: Requesting shutdown +49:25.814 [INFO] TestAgent.server: shutting down server +49:25.814 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.814 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.814 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.814 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.814 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.814 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.814 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.814 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.814 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.814 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.814 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:25.814Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.814Z [INFO] agent.router.manager: shutting down +49:25.814 [INFO] TestAgent: consul server down +49:25.814 [INFO] TestAgent: shutdown complete +49:25.815 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23889 network=tcp +49:25.815 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23889 network=udp +49:25.815 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23897 network=tcp +49:26.315 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.315 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ConnectServiceLookup (0.76s) === CONT TestDNS_InifiniteRecursion -2021-10-31T23:13:13.160Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:13.191Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:13.193 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:539fc96a-bc8d-8736-2940-3647f9703f4c Address:127.0.0.1:17075}]" -13:13.193 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17075 [Follower]" leader= -13:13.201 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-539fc96a-bc8d-8736-2940-3647f9703f4c.dc1 127.0.0.1 -13:13.202 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-539fc96a-bc8d-8736-2940-3647f9703f4c 127.0.0.1 -2021-10-31T23:13:13.202Z [INFO] agent.router: Initializing LAN area manager -13:13.202 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17070 network=udp -13:13.202 [INFO] TestAgent.server: Adding LAN server: server="Node-539fc96a-bc8d-8736-2940-3647f9703f4c (Addr: tcp/127.0.0.1:17075) (DC: dc1)" -13:13.202 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-539fc96a-bc8d-8736-2940-3647f9703f4c.dc1 area=wan -13:13.202 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17070 network=tcp -13:13.203 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17071 network=tcp -13:13.203 [INFO] TestAgent: started state syncer -13:13.203 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17076 network=tcp -13:13.258 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:13.258 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17075 [Candidate]" term=2 -13:13.258 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:13.258 [DEBUG] TestAgent.server.raft: vote granted: from=539fc96a-bc8d-8736-2940-3647f9703f4c term=2 tally=1 -13:13.258 [INFO] TestAgent.server.raft: election won: tally=1 -13:13.258 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17075 [Leader]" -13:13.259 [INFO] TestAgent.server: cluster leadership acquired -13:13.259 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17075 -13:13.260 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:13.261 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:13.261 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:13.261 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:13.261 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:13.261 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:13.261 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-539fc96a-bc8d-8736-2940-3647f9703f4c -13:13.261 [INFO] TestAgent.server: member joined, marking health alive: member=Node-539fc96a-bc8d-8736-2940-3647f9703f4c -13:13.261 [INFO] TestAgent.server: New leader elected: payload=Node-539fc96a-bc8d-8736-2940-3647f9703f4c -13:13.262 [INFO] TestAgent.server: federation state anti-entropy synced -13:13.498 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:13.498 [INFO] TestAgent: Synced node info -13:13.535 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=288.756µs client=127.0.0.1:39203 client_network=udp -13:13.536 [DEBUG] TestAgent.dns: request served from client: name=7c4a7859-4c11-5559-872c-fce9dd26f162.query.consul. type=SRV class=IN latency=143.861µs client=127.0.0.1:46457 client_network=udp -13:13.536 [INFO] TestAgent: Requesting shutdown -13:13.536 [INFO] TestAgent.server: shutting down server -13:13.536 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:13.536 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:13.536 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:13.536 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:13.536 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:13.536 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:13.536 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:13.536 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:13.536 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:13.536 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:13.536 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:13.536Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:13.536Z [INFO] agent.router.manager: shutting down -13:13.536 [INFO] TestAgent: consul server down -13:13.536 [INFO] TestAgent: shutdown complete -13:13.536 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17070 network=tcp -13:13.536 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17070 network=udp -13:13.536 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17071 network=tcp -13:14.038 [INFO] TestAgent: Waiting for endpoints to shut down -13:14.038 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_ServiceAddress_A (0.91s) -=== CONT TestDNS_ServiceLookupPreferNoCNAME -2021-10-31T23:13:13.240Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:13.291Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:13.293 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:337632d7-a277-b304-abb1-5ad6ab80a1b9 Address:127.0.0.1:17089}]" -13:13.302 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17089 [Follower]" leader= -13:13.303 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-337632d7-a277-b304-abb1-5ad6ab80a1b9.dc1 127.0.0.1 -13:13.304 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-337632d7-a277-b304-abb1-5ad6ab80a1b9 127.0.0.1 -2021-10-31T23:13:13.304Z [INFO] agent.router: Initializing LAN area manager -13:13.304 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17084 network=udp -13:13.304 [INFO] TestAgent.server: Adding LAN server: server="Node-337632d7-a277-b304-abb1-5ad6ab80a1b9 (Addr: tcp/127.0.0.1:17089) (DC: dc1)" -13:13.304 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-337632d7-a277-b304-abb1-5ad6ab80a1b9.dc1 area=wan -13:13.304 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17084 network=tcp -13:13.306 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17085 network=tcp -13:13.306 [INFO] TestAgent: started state syncer -13:13.306 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17090 network=tcp -13:13.365 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:13.365 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17089 [Candidate]" term=2 -13:13.366 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:13.366 [DEBUG] TestAgent.server.raft: vote granted: from=337632d7-a277-b304-abb1-5ad6ab80a1b9 term=2 tally=1 -13:13.366 [INFO] TestAgent.server.raft: election won: tally=1 -13:13.366 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17089 [Leader]" -13:13.366 [INFO] TestAgent.server: cluster leadership acquired -13:13.366 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17089 -13:13.367 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:13.368 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:13.368 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:13.368 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:13.368 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:13.368 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:13.368 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-337632d7-a277-b304-abb1-5ad6ab80a1b9 -13:13.368 [INFO] TestAgent.server: member joined, marking health alive: member=Node-337632d7-a277-b304-abb1-5ad6ab80a1b9 -13:13.368 [INFO] TestAgent.server: New leader elected: payload=Node-337632d7-a277-b304-abb1-5ad6ab80a1b9 -13:13.369 [INFO] TestAgent.server: federation state anti-entropy synced -13:13.562 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=313.891µs client=127.0.0.1:49595 client_network=udp -13:13.566 [DEBUG] TestAgent.dns: request served from client: name=60fa4f07-938e-e788-53f6-aca912ef80ee.query.consul. type=SRV class=IN latency=190.059µs client=127.0.0.1:60456 client_network=udp -13:13.566 [INFO] TestAgent: Requesting shutdown -13:13.566 [INFO] TestAgent.server: shutting down server -13:13.566 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:13.566 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:13.566 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:13.566 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:13.566 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:13.566 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:13.566 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:13.566 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:13.567 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:13.567 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:13.567 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:13.567 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:13.567Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:13.567Z [INFO] agent.router.manager: shutting down -13:13.567 [INFO] TestAgent: consul server down -13:13.567 [INFO] TestAgent: shutdown complete -13:13.567 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17084 network=tcp -13:13.567 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17084 network=udp -13:13.567 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17085 network=tcp -13:14.076 [INFO] TestAgent: Waiting for endpoints to shut down -13:14.076 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_ServiceAddressIPV6 (0.86s) -=== CONT TestDNS_ExternalServiceLookup -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:13.394Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:13.423Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:13.425 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0c48ec08-4cf6-3836-e71d-e8d7cdc622de Address:127.0.0.1:17103}]" -13:13.425 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17103 [Follower]" leader= -13:13.426 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:13.426 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:13.426Z [INFO] agent.router: Initializing LAN area manager -13:13.427 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:17103) (DC: dc1)" -13:13.427 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:13.427 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17098 network=udp -13:13.427 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17098 network=tcp -13:13.427 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17099 network=tcp -13:13.427 [INFO] TestAgent: started state syncer -13:13.427 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17104 network=tcp -13:13.489 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:13.489 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17103 [Candidate]" term=2 -13:13.490 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:13.490 [DEBUG] TestAgent.server.raft: vote granted: from=0c48ec08-4cf6-3836-e71d-e8d7cdc622de term=2 tally=1 -13:13.490 [INFO] TestAgent.server.raft: election won: tally=1 -13:13.490 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17103 [Leader]" -13:13.490 [INFO] TestAgent.server: cluster leadership acquired -13:13.490 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17103 -13:13.491 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:13.493 [INFO] TestAgent.server: New leader elected: payload=test-node -13:13.500 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:13.500 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:13.500 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:13.500 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:13.500 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:13.500 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:13.500 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:13.501 [INFO] TestAgent.server: federation state anti-entropy synced -13:13.602 [DEBUG] TestAgent.dns: request served from client: name=alias2.service.consul. type=SRV class=IN latency=398.372µs client=127.0.0.1:37073 client_network=udp -13:13.605 [INFO] TestAgent: Requesting shutdown -13:13.606 [INFO] TestAgent.server: shutting down server -13:13.606 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:13.606 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:13.606 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:13.606 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:13.606 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:13.606 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:13.606 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:13.606 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:13.606 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:13.606 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:13.606 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:13.606 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:13.606Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:13.606Z [INFO] agent.router.manager: shutting down -13:13.606 [INFO] TestAgent: consul server down -13:13.606 [INFO] TestAgent: shutdown complete -13:13.606 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17098 network=tcp -13:13.606 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17098 network=udp -13:13.606 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17099 network=tcp -13:14.110 [INFO] TestAgent: Waiting for endpoints to shut down -13:14.110 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ExternalServiceToConsulCNAMENestedLookup (0.75s) -=== CONT TestDNS_ServiceLookupWithInternalServiceAddress -2021-10-31T23:13:13.456Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:13.539Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:13.541 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4f699306-970b-3e37-c194-b5fe34e20107 Address:[::1]:17117}]" -13:13.541 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: server1.dc1 ::1 -13:13.542 [INFO] TestAgent.server.raft: entering follower state: follower="Node at [::1]:17117 [Follower]" leader= -13:13.546 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: server1 ::1 -2021-10-31T23:13:13.546Z [INFO] agent.router: Initializing LAN area manager -13:13.546 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17112 network=udp -13:13.547 [INFO] TestAgent.server: Adding LAN server: server="server1 (Addr: tcp/[::1]:17117) (DC: dc1)" -13:13.547 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=server1.dc1 area=wan -13:13.547 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17112 network=tcp -13:13.547 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17113 network=tcp -13:13.547 [INFO] TestAgent: started state syncer -13:13.547 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17118 network=tcp -13:13.606 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:13.606 [INFO] TestAgent.server.raft: entering candidate state: node="Node at [::1]:17117 [Candidate]" term=2 -13:13.606 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:13.606 [DEBUG] TestAgent.server.raft: vote granted: from=4f699306-970b-3e37-c194-b5fe34e20107 term=2 tally=1 -13:13.606 [INFO] TestAgent.server.raft: election won: tally=1 -13:13.606 [INFO] TestAgent.server.raft: entering leader state: leader="Node at [::1]:17117 [Leader]" -13:13.606 [INFO] TestAgent.server: cluster leadership acquired -13:13.607 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=[::1]:17117 -13:13.607 [INFO] TestAgent.server: New leader elected: payload=server1 -13:13.607 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:13.609 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:13.609 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:13.609 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:13.609 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:13.609 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:13.609 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=server1 -13:13.609 [INFO] TestAgent.server: member joined, marking health alive: member=server1 -13:13.610 [INFO] TestAgent.server: federation state anti-entropy synced -13:13.786 [DEBUG] TestAgent.dns: request served from client: name=server1.node.dc1.consul. type=NS class=IN latency=151.649µs client=127.0.0.1:41475 client_network=udp -13:13.787 [INFO] TestAgent: Requesting shutdown -13:13.787 [INFO] TestAgent.server: shutting down server -13:13.787 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:13.787 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:13.787 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:13.787 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:13.787 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:13.787 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:13.787 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:13.787 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:13.787 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:13.787 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:13.787 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:13.787 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:13.787Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:13.787Z [INFO] agent.router.manager: shutting down -13:13.787 [INFO] TestAgent: consul server down -13:13.787 [INFO] TestAgent: shutdown complete -13:13.787 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17112 network=tcp -13:13.787 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17112 network=udp -13:13.787 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17113 network=tcp -13:14.288 [INFO] TestAgent: Waiting for endpoints to shut down -13:14.288 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NSRecords_IPV6 (0.89s) -=== CONT TestDNS_ServiceLookupNoMultiCNAME -2021-10-31T23:13:13.532Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:13.574Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:13.576 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5d018ea1-2677-f7b0-2803-0c773c2afa41 Address:127.0.0.1:17124}]" -13:13.576 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5d018ea1-2677-f7b0-2803-0c773c2afa41.dc1 127.0.0.1 -13:13.577 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17124 [Follower]" leader= -13:13.586 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5d018ea1-2677-f7b0-2803-0c773c2afa41 127.0.0.1 -2021-10-31T23:13:13.586Z [INFO] agent.router: Initializing LAN area manager -13:13.586 [INFO] TestAgent.server: Adding LAN server: server="Node-5d018ea1-2677-f7b0-2803-0c773c2afa41 (Addr: tcp/127.0.0.1:17124) (DC: dc1)" -13:13.586 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5d018ea1-2677-f7b0-2803-0c773c2afa41.dc1 area=wan -13:13.586 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17119 network=udp -13:13.586 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17119 network=tcp -13:13.587 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17120 network=tcp -13:13.587 [INFO] TestAgent: started state syncer -13:13.587 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17125 network=tcp -13:13.621 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:13.621 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17124 [Candidate]" term=2 -13:13.622 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:13.622 [DEBUG] TestAgent.server.raft: vote granted: from=5d018ea1-2677-f7b0-2803-0c773c2afa41 term=2 tally=1 -13:13.622 [INFO] TestAgent.server.raft: election won: tally=1 -13:13.622 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17124 [Leader]" -13:13.622 [INFO] TestAgent.server: cluster leadership acquired -13:13.622 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17124 -13:13.623 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:13.624 [INFO] TestAgent.server: New leader elected: payload=Node-5d018ea1-2677-f7b0-2803-0c773c2afa41 -13:13.624 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:13.624 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:13.624 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:13.624 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:13.624 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:13.624 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5d018ea1-2677-f7b0-2803-0c773c2afa41 -13:13.624 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5d018ea1-2677-f7b0-2803-0c773c2afa41 -13:13.625 [INFO] TestAgent.server: federation state anti-entropy synced -13:13.783 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=522.493µs client=127.0.0.1:57520 client_network=udp -13:13.787 [DEBUG] TestAgent.dns: request served from client: name=997d2ad4-b887-2f04-3ec3-ca8e6a50e259.query.consul. type=SRV class=IN latency=197.53µs client=127.0.0.1:35975 client_network=udp -13:13.788 [INFO] TestAgent: Requesting shutdown -13:13.788 [INFO] TestAgent.server: shutting down server -13:13.788 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:13.788 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:13.788 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:13.788 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:13.788 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:13.788 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:13.788 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:13.788 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:13.788 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:13.788 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:13.788 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:13.788 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:13.788Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:13.788Z [INFO] agent.router.manager: shutting down -13:13.788 [INFO] TestAgent: consul server down -13:13.788 [INFO] TestAgent: shutdown complete -13:13.788 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17119 network=tcp -13:13.788 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17119 network=udp -13:13.788 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17120 network=tcp -13:14.293 [INFO] TestAgent: Waiting for endpoints to shut down -13:14.293 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_Dedup_SRV (0.83s) -=== CONT TestDNS_ServiceLookup -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:13.591Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:13.625Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:13.635 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2464575e-bffd-18be-9b8d-30b824920f95 Address:127.0.0.1:17131}]" -13:13.636 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: server1.dc1 127.0.0.1 -13:13.636 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: server1 127.0.0.1 -2021-10-31T23:13:13.636Z [INFO] agent.router: Initializing LAN area manager -13:13.636 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17126 network=udp -13:13.636 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17131 [Follower]" leader= -13:13.637 [INFO] TestAgent.server: Adding LAN server: server="server1 (Addr: tcp/127.0.0.1:17131) (DC: dc1)" -13:13.637 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=server1.dc1 area=wan -13:13.637 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17126 network=tcp -13:13.637 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17127 network=tcp -13:13.643 [INFO] TestAgent: started state syncer -13:13.643 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17132 network=tcp -13:13.693 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:13.694 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17131 [Candidate]" term=2 -13:13.694 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:13.694 [DEBUG] TestAgent.server.raft: vote granted: from=2464575e-bffd-18be-9b8d-30b824920f95 term=2 tally=1 -13:13.694 [INFO] TestAgent.server.raft: election won: tally=1 -13:13.694 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17131 [Leader]" -13:13.694 [INFO] TestAgent.server: cluster leadership acquired -13:13.694 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17131 -13:13.695 [INFO] TestAgent.server: New leader elected: payload=server1 -13:13.695 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:13.696 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:13.696 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:13.696 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:13.696 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:13.696 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:13.696 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=server1 -13:13.696 [INFO] TestAgent.server: member joined, marking health alive: member=server1 -13:13.697 [INFO] TestAgent.server: federation state anti-entropy synced -13:13.900 [DEBUG] TestAgent.dns: request served from client: name=something.node.consul. type=NS class=IN latency=131.131µs client=127.0.0.1:58783 client_network=udp -13:13.901 [INFO] TestAgent: Requesting shutdown -13:13.901 [INFO] TestAgent.server: shutting down server -13:13.901 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:13.901 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:13.901 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:13.901 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:13.901 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:13.901 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:13.901 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:13.901 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:13.901 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:13.901 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:13.901 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:13.901 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:13.901Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:13.901Z [INFO] agent.router.manager: shutting down -13:13.901 [INFO] TestAgent: consul server down -13:13.901 [INFO] TestAgent: shutdown complete -13:13.901 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17126 network=tcp -13:13.901 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17126 network=udp -13:13.901 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17127 network=tcp -13:14.401 [INFO] TestAgent: Waiting for endpoints to shut down -13:14.443 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NSRecords (0.88s) -=== CONT TestDNS_IngressServiceLookup -2021-10-31T23:13:13.408Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:13.459Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:13.479 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:22b205ab-837d-da91-73bb-a53695a6e3d4 Address:127.0.0.1:17110}]" -13:13.479 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-22b205ab-837d-da91-73bb-a53695a6e3d4.dc1 127.0.0.1 -13:13.480 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17110 [Follower]" leader= -13:13.480 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-22b205ab-837d-da91-73bb-a53695a6e3d4 127.0.0.1 -2021-10-31T23:13:13.480Z [INFO] agent.router: Initializing LAN area manager -13:13.480 [DEBUG] TestAgent.dns: recursor enabled -13:13.481 [INFO] TestAgent.server: Adding LAN server: server="Node-22b205ab-837d-da91-73bb-a53695a6e3d4 (Addr: tcp/127.0.0.1:17110) (DC: dc1)" -13:13.481 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17105 network=udp -13:13.481 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-22b205ab-837d-da91-73bb-a53695a6e3d4.dc1 area=wan -13:13.481 [DEBUG] TestAgent.dns: recursor enabled -13:13.481 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17105 network=tcp -13:13.485 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17106 network=tcp -13:13.485 [INFO] TestAgent: started state syncer -13:13.485 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17111 network=tcp -13:13.534 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:13.534 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17110 [Candidate]" term=2 -13:13.534 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:13.534 [DEBUG] TestAgent.server.raft: vote granted: from=22b205ab-837d-da91-73bb-a53695a6e3d4 term=2 tally=1 -13:13.534 [INFO] TestAgent.server.raft: election won: tally=1 -13:13.534 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17110 [Leader]" -13:13.534 [INFO] TestAgent.server: cluster leadership acquired -13:13.534 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17110 -13:13.535 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:13.537 [INFO] TestAgent.server: New leader elected: payload=Node-22b205ab-837d-da91-73bb-a53695a6e3d4 -13:13.537 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:13.537 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:13.537 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:13.537 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:13.537 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:13.537 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-22b205ab-837d-da91-73bb-a53695a6e3d4 -13:13.537 [INFO] TestAgent.server: member joined, marking health alive: member=Node-22b205ab-837d-da91-73bb-a53695a6e3d4 -13:13.541 [INFO] TestAgent.server: federation state anti-entropy synced -13:13.854 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:13.854 [INFO] TestAgent: Synced node info -13:13.927 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=4.753244ms -13:13.927 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=5.17466ms client=127.0.0.1:60789 client_network=udp -13:13.927 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=79.475µs -13:13.927 [DEBUG] TestAgent.dns: request served from client: name=01afbeb2-5862-f85e-4f1d-0a5002b819d3.query.consul. type=SRV class=IN latency=314.556µs client=127.0.0.1:43612 client_network=udp -13:13.927 [INFO] TestAgent: Requesting shutdown -13:13.927 [INFO] TestAgent.server: shutting down server -13:13.927 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:13.927 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:13.927 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:13.927 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:13.927 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:13.928 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:13.928 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:13.928 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:13.928 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:13.928 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:13.928 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:13.928Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:13.928Z [INFO] agent.router.manager: shutting down -13:13.928 [INFO] TestAgent: consul server down -13:13.928 [INFO] TestAgent: shutdown complete -13:13.928 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17105 network=tcp -13:13.928 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17105 network=udp -13:13.928 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17106 network=tcp -13:14.438 [INFO] TestAgent: Waiting for endpoints to shut down -13:14.438 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_ServiceAddress_SRV (1.06s) -=== CONT TestDNS_ServiceReverseLookup_IPV6 -2021-10-31T23:13:13.692Z [WARN] agent: Node name "test node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2021-10-31T23:13:13.692Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:13.751Z [WARN] agent.auto_config: Node name "test node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2021-10-31T23:13:13.751Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:13.753 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5c42e68f-db18-3b91-33d7-aeec716f6f50 Address:127.0.0.1:17138}]" -13:13.753 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17138 [Follower]" leader= -13:13.771 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test node.dc1 127.0.0.1 -13:13.772 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test node 127.0.0.1 -2021-10-31T23:13:13.772Z [INFO] agent.router: Initializing LAN area manager -13:13.772 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17133 network=udp -13:13.773 [INFO] TestAgent.server: Adding LAN server: server="test node (Addr: tcp/127.0.0.1:17138) (DC: dc1)" -13:13.773 [INFO] TestAgent.server: Handled event for server in area: event=member-join server="test node.dc1" area=wan -13:13.773 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17133 network=tcp -13:13.776 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17134 network=tcp -13:13.776 [INFO] TestAgent: started state syncer -13:13.776 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17139 network=tcp -13:13.810 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:13.810 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17138 [Candidate]" term=2 -13:13.810 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:13.810 [DEBUG] TestAgent.server.raft: vote granted: from=5c42e68f-db18-3b91-33d7-aeec716f6f50 term=2 tally=1 -13:13.810 [INFO] TestAgent.server.raft: election won: tally=1 -13:13.810 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17138 [Leader]" -13:13.810 [INFO] TestAgent.server: cluster leadership acquired -13:13.810 [INFO] TestAgent.server: New leader elected: payload="test node" -13:13.810 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17138 -13:13.811 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:13.813 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:13.813 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:13.813 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:13.813 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:13.813 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:13.813 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node="test node" -13:13.813 [INFO] TestAgent.server: member joined, marking health alive: member="test node" -13:13.817 [INFO] TestAgent.server: federation state anti-entropy synced -13:14.074 [DEBUG] TestAgent.dns: request served from client: name=alias.service.consul. type=SRV class=IN latency=464.592µs client=127.0.0.1:36570 client_network=udp -13:14.075 [DEBUG] TestAgent.dns: request served from client: name=alias.service.CoNsUl. type=SRV class=IN latency=179.238µs client=127.0.0.1:48057 client_network=udp -13:14.075 [INFO] TestAgent: Requesting shutdown -13:14.075 [INFO] TestAgent.server: shutting down server -13:14.075 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.075 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.075 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.075 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.075 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.075 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.075 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.075 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:14.075 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:14.075 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.076 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.076 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:14.076Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.076Z [INFO] agent.router.manager: shutting down -13:14.076 [INFO] TestAgent: consul server down -13:14.076 [INFO] TestAgent: shutdown complete -13:14.076 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17133 network=tcp -13:14.076 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17133 network=udp -13:14.076 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17134 network=tcp -13:14.577 [INFO] TestAgent: Waiting for endpoints to shut down -13:14.577 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ExternalServiceToConsulCNAMELookup (0.92s) -=== CONT TestDNS_ReverseLookup_CustomDomain -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:13.380Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:13.380Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:13.429Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:13.429Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:13.431 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1c700d04-6381-81b9-0ca4-e2a11a334e15 Address:127.0.0.1:17096}]" -13:13.432 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1c700d04-6381-81b9-0ca4-e2a11a334e15.dc2 127.0.0.1 -13:13.432 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1c700d04-6381-81b9-0ca4-e2a11a334e15 127.0.0.1 -2021-10-31T23:13:13.432Z [INFO] agent.router: Initializing LAN area manager -13:13.432 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17091 network=udp -13:13.432 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17096 [Follower]" leader= -13:13.458 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1c700d04-6381-81b9-0ca4-e2a11a334e15.dc2 area=wan -13:13.458 [INFO] TestAgent.server: Adding LAN server: server="Node-1c700d04-6381-81b9-0ca4-e2a11a334e15 (Addr: tcp/127.0.0.1:17096) (DC: dc2)" -13:13.458 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17091 network=tcp -13:13.459 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17092 network=tcp -13:13.459 [INFO] TestAgent: started state syncer -13:13.459 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17097 network=tcp -13:13.505 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:13.505 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17096 [Candidate]" term=2 -13:13.506 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:13.506 [DEBUG] TestAgent.server.raft: vote granted: from=1c700d04-6381-81b9-0ca4-e2a11a334e15 term=2 tally=1 -13:13.506 [INFO] TestAgent.server.raft: election won: tally=1 -13:13.506 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17096 [Leader]" -13:13.506 [INFO] TestAgent.server: cluster leadership acquired -13:13.506 [INFO] TestAgent.server: initializing acls -13:13.507 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:13.507 [INFO] TestAgent.server: New leader elected: payload=Node-1c700d04-6381-81b9-0ca4-e2a11a334e15 -13:13.509 [INFO] TestAgent.server: initializing acls -13:13.534 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:13.534 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:13.534 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:13.534 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1c700d04-6381-81b9-0ca4-e2a11a334e15 -13:13.534 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1c700d04-6381-81b9-0ca4-e2a11a334e15.dc2 -13:13.534 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1c700d04-6381-81b9-0ca4-e2a11a334e15.dc2 area=wan -13:13.537 [INFO] TestAgent.server: Updating LAN server: server="Node-1c700d04-6381-81b9-0ca4-e2a11a334e15 (Addr: tcp/127.0.0.1:17096) (DC: dc2)" -13:13.542 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:13.542 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -13:13.542 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1c700d04-6381-81b9-0ca4-e2a11a334e15 -13:13.543 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1c700d04-6381-81b9-0ca4-e2a11a334e15.dc2 -13:13.543 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1c700d04-6381-81b9-0ca4-e2a11a334e15.dc2 area=wan -13:13.543 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:13.543 [INFO] TestAgent.server: Updating LAN server: server="Node-1c700d04-6381-81b9-0ca4-e2a11a334e15 (Addr: tcp/127.0.0.1:17096) (DC: dc2)" -13:13.544 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:13.544 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:13.544 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:13.544 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:13.544 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:13.544 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1c700d04-6381-81b9-0ca4-e2a11a334e15 -13:13.544 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1c700d04-6381-81b9-0ca4-e2a11a334e15 -13:13.544 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1c700d04-6381-81b9-0ca4-e2a11a334e15 -13:13.545 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1c700d04-6381-81b9-0ca4-e2a11a334e15 -13:13.545 [INFO] TestAgent.server: federation state anti-entropy synced -13:13.596 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:17060] -13:13.597 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:17060 -13:13.598 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-71787e0c-912a-8cba-80f2-80b28f29998c.dc1 127.0.0.1 -13:13.598 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -13:13.598 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-71787e0c-912a-8cba-80f2-80b28f29998c.dc1 area=wan -13:13.602 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=263.163µs client=127.0.0.1:60702 client_network=udp -13:13.606 [DEBUG] TestAgent.dns: request served from client: name=b23d3889-5950-c8fe-7f02-7f9cf2308b79.query.dc2.consul. type=SRV class=IN latency=212.578µs client=127.0.0.1:35458 client_network=udp -13:13.606 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=90.925µs client=127.0.0.1:38442 client_network=udp -13:13.607 [DEBUG] TestAgent.dns: request served from client: name=b23d3889-5950-c8fe-7f02-7f9cf2308b79.query.dc2.consul. type=A class=IN latency=89.372µs client=127.0.0.1:40826 client_network=udp -13:13.609 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=259.44µs client=127.0.0.1:51425 client_network=udp -13:13.614 [DEBUG] TestAgent.dns: request served from client: name=b23d3889-5950-c8fe-7f02-7f9cf2308b79.query.dc2.consul. type=SRV class=IN latency=335.615µs client=127.0.0.1:46927 client_network=udp -13:13.624 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=341.494µs client=127.0.0.1:40659 client_network=udp -13:13.626 [DEBUG] TestAgent.dns: request served from client: name=b23d3889-5950-c8fe-7f02-7f9cf2308b79.query.dc2.consul. type=A class=IN latency=284.922µs client=127.0.0.1:57536 client_network=udp -13:13.644 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:13.644 [INFO] TestAgent: Synced node info -13:13.644 [DEBUG] TestAgent: Node info in sync -13:13.687 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=183.459µs client=127.0.0.1:37773 client_network=udp -13:13.688 [DEBUG] TestAgent.dns: request served from client: name=b23d3889-5950-c8fe-7f02-7f9cf2308b79.query.dc2.consul. type=SRV class=IN latency=143.831µs client=127.0.0.1:58419 client_network=udp -13:13.688 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=109.17µs client=127.0.0.1:37586 client_network=udp -13:13.688 [DEBUG] TestAgent.dns: request served from client: name=b23d3889-5950-c8fe-7f02-7f9cf2308b79.query.dc2.consul. type=A class=IN latency=107.485µs client=127.0.0.1:44005 client_network=udp -13:13.689 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=164.857µs client=127.0.0.1:55271 client_network=udp -13:13.693 [DEBUG] TestAgent.dns: request served from client: name=b23d3889-5950-c8fe-7f02-7f9cf2308b79.query.dc2.consul. type=SRV class=IN latency=236.676µs client=127.0.0.1:55291 client_network=udp -13:13.693 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=134.14µs client=127.0.0.1:43102 client_network=udp -13:13.694 [DEBUG] TestAgent.dns: request served from client: name=b23d3889-5950-c8fe-7f02-7f9cf2308b79.query.dc2.consul. type=A class=IN latency=141.149µs client=127.0.0.1:45798 client_network=udp -13:13.694 [INFO] TestAgent: Requesting shutdown -13:13.694 [INFO] TestAgent.server: shutting down server -13:13.694 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:13.694 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:13.694 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:13.694 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:13.694 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:13.694 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:13.694 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:13.694 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:13.694 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:13.694 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:13.694 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:13.694 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:13.694 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:13.694 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:13.694 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:13.694Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:13.694Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:13.694Z [INFO] agent.router.manager: shutting down -13:13.694 [INFO] TestAgent: consul server down -13:13.694 [INFO] TestAgent: shutdown complete -13:13.694 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17091 network=tcp -13:13.694 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17091 network=udp -13:13.694 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17092 network=tcp -13:14.195 [INFO] TestAgent: Waiting for endpoints to shut down -13:14.195 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:13.034Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:13.034Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:13.075Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:13.075Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:13.076 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:71787e0c-912a-8cba-80f2-80b28f29998c Address:127.0.0.1:17061}]" -13:13.077 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-71787e0c-912a-8cba-80f2-80b28f29998c.dc1 127.0.0.1 -13:13.077 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17061 [Follower]" leader= -13:13.086 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-71787e0c-912a-8cba-80f2-80b28f29998c 127.0.0.1 -2021-10-31T23:13:13.086Z [INFO] agent.router: Initializing LAN area manager -13:13.086 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17056 network=udp -13:13.086 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-71787e0c-912a-8cba-80f2-80b28f29998c.dc1 area=wan -13:13.086 [INFO] TestAgent.server: Adding LAN server: server="Node-71787e0c-912a-8cba-80f2-80b28f29998c (Addr: tcp/127.0.0.1:17061) (DC: dc1)" -13:13.086 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17056 network=tcp -13:13.087 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17057 network=tcp -13:13.087 [INFO] TestAgent: started state syncer -13:13.087 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17062 network=tcp -13:13.137 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:13.149 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:13.149 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17061 [Candidate]" term=2 -13:13.150 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:13.150 [DEBUG] TestAgent.server.raft: vote granted: from=71787e0c-912a-8cba-80f2-80b28f29998c term=2 tally=1 -13:13.150 [INFO] TestAgent.server.raft: election won: tally=1 -13:13.150 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17061 [Leader]" -13:13.150 [INFO] TestAgent.server: cluster leadership acquired -13:13.150 [INFO] TestAgent.server: initializing acls -13:13.150 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:13.151 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:13.151 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:13.151 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:13.151 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-71787e0c-912a-8cba-80f2-80b28f29998c -13:13.151 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-71787e0c-912a-8cba-80f2-80b28f29998c.dc1 -13:13.152 [INFO] TestAgent.server: New leader elected: payload=Node-71787e0c-912a-8cba-80f2-80b28f29998c -13:13.152 [INFO] TestAgent.server: Updating LAN server: server="Node-71787e0c-912a-8cba-80f2-80b28f29998c (Addr: tcp/127.0.0.1:17061) (DC: dc1)" -13:13.152 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:13.152 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-71787e0c-912a-8cba-80f2-80b28f29998c.dc1 area=wan -13:13.153 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:13.153 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:13.153 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:13.153 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:13.153 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:13.153 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-71787e0c-912a-8cba-80f2-80b28f29998c -13:13.153 [INFO] TestAgent.server: member joined, marking health alive: member=Node-71787e0c-912a-8cba-80f2-80b28f29998c -13:13.158 [INFO] TestAgent.server: federation state anti-entropy synced -13:13.158 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-71787e0c-912a-8cba-80f2-80b28f29998c -13:13.184 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:13.185 [INFO] TestAgent: Synced node info -13:13.553 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:13.553 [DEBUG] TestAgent: Node info in sync -13:13.553 [DEBUG] TestAgent: Node info in sync -13:13.597 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:34398 -13:13.597 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1c700d04-6381-81b9-0ca4-e2a11a334e15.dc2 127.0.0.1 -13:13.597 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1c700d04-6381-81b9-0ca4-e2a11a334e15.dc2 area=wan -13:13.631 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=1.225299ms client=127.0.0.1:54569 client_network=udp -13:13.632 [DEBUG] TestAgent.dns: request served from client: name=b23d3889-5950-c8fe-7f02-7f9cf2308b79.query.dc2.consul. type=SRV class=IN latency=1.309006ms client=127.0.0.1:38202 client_network=udp -13:13.642 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=852.459µs client=127.0.0.1:54918 client_network=udp -13:13.643 [DEBUG] TestAgent.dns: request served from client: name=b23d3889-5950-c8fe-7f02-7f9cf2308b79.query.dc2.consul. type=A class=IN latency=334.445µs client=127.0.0.1:59623 client_network=udp -13:13.644 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=365.395µs client=127.0.0.1:43781 client_network=udp -13:13.646 [DEBUG] TestAgent.dns: request served from client: name=b23d3889-5950-c8fe-7f02-7f9cf2308b79.query.dc2.consul. type=SRV class=IN latency=1.550475ms client=127.0.0.1:51878 client_network=udp -13:13.652 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=650.952µs client=127.0.0.1:46687 client_network=udp -13:13.653 [DEBUG] TestAgent.dns: request served from client: name=b23d3889-5950-c8fe-7f02-7f9cf2308b79.query.dc2.consul. type=A class=IN latency=379.048µs client=127.0.0.1:58608 client_network=udp -13:13.657 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=761.736µs client=127.0.0.1:49605 client_network=udp -13:13.666 [DEBUG] TestAgent.dns: request served from client: name=b23d3889-5950-c8fe-7f02-7f9cf2308b79.query.dc2.consul. type=SRV class=IN latency=8.196901ms client=127.0.0.1:53506 client_network=udp -13:13.678 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=4.589552ms client=127.0.0.1:52784 client_network=udp -13:13.679 [DEBUG] TestAgent.dns: request served from client: name=b23d3889-5950-c8fe-7f02-7f9cf2308b79.query.dc2.consul. type=A class=IN latency=469.661µs client=127.0.0.1:59599 client_network=udp -13:13.680 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=412.134µs client=127.0.0.1:60350 client_network=udp -13:13.681 [DEBUG] TestAgent.dns: request served from client: name=b23d3889-5950-c8fe-7f02-7f9cf2308b79.query.dc2.consul. type=SRV class=IN latency=364.739µs client=127.0.0.1:33367 client_network=udp -13:13.681 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=393.356µs client=127.0.0.1:37560 client_network=udp -13:13.682 [DEBUG] TestAgent.dns: request served from client: name=b23d3889-5950-c8fe-7f02-7f9cf2308b79.query.dc2.consul. type=A class=IN latency=725.788µs client=127.0.0.1:41179 client_network=udp -13:14.195 [INFO] TestAgent: Requesting shutdown -13:14.195 [INFO] TestAgent.server: shutting down server -13:14.195 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:14.195 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:14.195 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.195 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.195 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.195 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.195 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.195 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.195 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.195 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:14.195 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:14.195 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.195 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.195 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:14.195 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:14.195Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.195Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.195Z [INFO] agent.router.manager: shutting down -13:14.195 [INFO] TestAgent: consul server down -13:14.195 [INFO] TestAgent: shutdown complete -13:14.195 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17056 network=tcp -13:14.195 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17056 network=udp -13:14.195 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17057 network=tcp -13:14.697 [INFO] TestAgent: Waiting for endpoints to shut down -13:14.697 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_WanTranslation (1.75s) - --- PASS: TestDNS_ServiceLookup_WanTranslation/service-addr-from-dc2 (0.01s) - --- PASS: TestDNS_ServiceLookup_WanTranslation/service-wan-from-dc2 (0.02s) - --- PASS: TestDNS_ServiceLookup_WanTranslation/node-addr-from-dc1 (0.02s) - --- PASS: TestDNS_ServiceLookup_WanTranslation/node-wan-from-dc1 (0.01s) - --- PASS: TestDNS_ServiceLookup_WanTranslation/service-addr-from-dc1 (0.03s) - --- PASS: TestDNS_ServiceLookup_WanTranslation/service-wan-from-dc1 (0.01s) - --- PASS: TestDNS_ServiceLookup_WanTranslation/node-addr-from-dc2 (0.00s) - --- PASS: TestDNS_ServiceLookup_WanTranslation/node-wan-from-dc2 (0.01s) +2022-12-04T05:49:25.612Z [WARN] agent: Node name "my.test-node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2022-12-04T05:49:25.612Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.622Z [WARN] agent.auto_config: Node name "my.test-node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2022-12-04T05:49:25.622Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.623 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3ee76408-422a-0bba-fea4-19680d674637 Address:127.0.0.1:23922}]" +49:25.623 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23922 [Follower]" leader= +49:25.623 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: my.test-node.dc1 127.0.0.1 +49:25.624 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: my.test-node 127.0.0.1 +2022-12-04T05:49:25.624Z [INFO] agent.router: Initializing LAN area manager +49:25.624 [INFO] TestAgent.server: Adding LAN server: server="my.test-node (Addr: tcp/127.0.0.1:23922) (DC: dc1)" +49:25.624 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=my.test-node.dc1 area=wan +49:25.624 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23938 network=udp +49:25.624 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23938 network=tcp +49:25.625 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23918 network=tcp +49:25.625 [INFO] TestAgent: started state syncer +49:25.625 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23923 network=tcp +49:25.691 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.691 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23922 [Candidate]" term=2 +49:25.691 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.691 [DEBUG] TestAgent.server.raft: vote granted: from=3ee76408-422a-0bba-fea4-19680d674637 term=2 tally=1 +49:25.691 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.691 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23922 [Leader]" +49:25.691 [INFO] TestAgent.server: cluster leadership acquired +49:25.691 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23922 +49:25.692 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.693 [INFO] TestAgent.server: New leader elected: payload=my.test-node +49:25.693 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.693 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.693 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.693 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.693 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.693 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=my.test-node +49:25.693 [INFO] TestAgent.server: member joined, marking health alive: member=my.test-node +49:25.694 [INFO] TestAgent.server: federation state anti-entropy synced +49:25.821 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:25.822 [INFO] TestAgent: Synced node info +49:25.822 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=132.365µs client=127.0.0.1:36751 client_network=udp +49:25.822 [INFO] TestAgent: Requesting shutdown +49:25.822 [INFO] TestAgent.server: shutting down server +49:25.822 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.822 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.822 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.822 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.822 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.822 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.822 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.822 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.822 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.822 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.822 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:25.823Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.823Z [INFO] agent.router.manager: shutting down +49:25.823 [INFO] TestAgent: consul server down +49:25.823 [INFO] TestAgent: shutdown complete +49:25.823 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23938 network=tcp +49:25.823 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23938 network=udp +49:25.823 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23918 network=tcp +49:26.323 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.323 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookupWithInternalServiceAddress (0.72s) +=== CONT TestDNS_ReverseLookup_IPV6 +2022-12-04T05:49:25.554Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.580Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.582 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:99f3aaa2-56ec-c2b4-264b-225597664fb6 Address:127.0.0.1:23880}]" +49:25.582 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23880 [Follower]" leader= +49:25.582 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-99f3aaa2-56ec-c2b4-264b-225597664fb6.dc1 127.0.0.1 +49:25.583 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-99f3aaa2-56ec-c2b4-264b-225597664fb6 127.0.0.1 +2022-12-04T05:49:25.583Z [INFO] agent.router: Initializing LAN area manager +49:25.583 [INFO] TestAgent.server: Adding LAN server: server="Node-99f3aaa2-56ec-c2b4-264b-225597664fb6 (Addr: tcp/127.0.0.1:23880) (DC: dc1)" +49:25.583 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-99f3aaa2-56ec-c2b4-264b-225597664fb6.dc1 area=wan +49:25.583 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23868 network=udp +49:25.583 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23868 network=tcp +49:25.584 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23890 network=tcp +49:25.584 [INFO] TestAgent: started state syncer +49:25.584 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23881 network=tcp +49:25.643 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.643 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23880 [Candidate]" term=2 +49:25.643 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.643 [DEBUG] TestAgent.server.raft: vote granted: from=99f3aaa2-56ec-c2b4-264b-225597664fb6 term=2 tally=1 +49:25.643 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.643 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23880 [Leader]" +49:25.643 [INFO] TestAgent.server: cluster leadership acquired +49:25.643 [INFO] TestAgent.server: New leader elected: payload=Node-99f3aaa2-56ec-c2b4-264b-225597664fb6 +49:25.643 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23880 +49:25.644 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.645 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.645 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.645 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.645 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.645 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.645 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-99f3aaa2-56ec-c2b4-264b-225597664fb6 +49:25.645 [INFO] TestAgent.server: member joined, marking health alive: member=Node-99f3aaa2-56ec-c2b4-264b-225597664fb6 +49:25.646 [INFO] TestAgent.server: federation state anti-entropy synced +49:25.704 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:25.704 [INFO] TestAgent: Synced node info +49:25.790 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:25.790 [DEBUG] TestAgent: Node info in sync +49:25.790 [DEBUG] TestAgent: Node info in sync +49:25.856 [DEBUG] TestAgent.dns: request served from client: name=api.ingress.consul. type=A class=IN latency=189.453µs client=127.0.0.1:60529 client_network=udp +49:25.856 [DEBUG] TestAgent.dns: request served from client: name=api.ingress.dc1.consul. type=A class=IN latency=105.243µs client=127.0.0.1:44399 client_network=udp +49:25.856 [DEBUG] TestAgent.dns: request served from client: name=db.ingress.consul. type=A class=IN latency=84.782µs client=127.0.0.1:41512 client_network=udp +49:25.857 [DEBUG] TestAgent.dns: request served from client: name=db.ingress.dc1.consul. type=A class=IN latency=85.596µs client=127.0.0.1:49538 client_network=udp +49:25.857 [INFO] TestAgent: Requesting shutdown +49:25.857 [INFO] TestAgent.server: shutting down server +49:25.857 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:25.857 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:25.857 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:25.857 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:25.857 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:25.857 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:25.857 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:25.857 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:25.857 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:25.857 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:25.857 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:25.857Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:25.857Z [INFO] agent.router.manager: shutting down +49:25.857 [INFO] TestAgent: consul server down +49:25.857 [INFO] TestAgent: shutdown complete +49:25.857 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23868 network=tcp +49:25.857 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23868 network=udp +49:25.857 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23890 network=tcp +49:26.357 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.357 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_IngressServiceLookup (0.82s) + --- PASS: TestDNS_IngressServiceLookup/api.ingress.consul. (0.00s) + --- PASS: TestDNS_IngressServiceLookup/api.ingress.dc1.consul. (0.00s) + --- PASS: TestDNS_IngressServiceLookup/db.ingress.consul. (0.00s) + --- PASS: TestDNS_IngressServiceLookup/db.ingress.dc1.consul. (0.00s) === CONT TestDNS_ServiceReverseLookupNodeAddress -2021-10-31T23:13:13.813Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:13.851Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:13.853 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:36d547d5-ec50-cb41-6545-d507f2adf557 Address:127.0.0.1:17145}]" -13:13.854 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17145 [Follower]" leader= -13:13.862 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-36d547d5-ec50-cb41-6545-d507f2adf557.dc1 127.0.0.1 -13:13.862 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-36d547d5-ec50-cb41-6545-d507f2adf557 127.0.0.1 -2021-10-31T23:13:13.862Z [INFO] agent.router: Initializing LAN area manager -13:13.863 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17140 network=udp -13:13.863 [INFO] TestAgent.server: Adding LAN server: server="Node-36d547d5-ec50-cb41-6545-d507f2adf557 (Addr: tcp/127.0.0.1:17145) (DC: dc1)" -13:13.863 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-36d547d5-ec50-cb41-6545-d507f2adf557.dc1 area=wan -13:13.863 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17140 network=tcp -13:13.864 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17141 network=tcp -13:13.864 [INFO] TestAgent: started state syncer -13:13.864 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17146 network=tcp -13:13.891 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:13.891 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17145 [Candidate]" term=2 -13:13.891 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:13.891 [DEBUG] TestAgent.server.raft: vote granted: from=36d547d5-ec50-cb41-6545-d507f2adf557 term=2 tally=1 -13:13.891 [INFO] TestAgent.server.raft: election won: tally=1 -13:13.891 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17145 [Leader]" -13:13.891 [INFO] TestAgent.server: cluster leadership acquired -13:13.892 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17145 -13:13.892 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:13.893 [INFO] TestAgent.server: New leader elected: payload=Node-36d547d5-ec50-cb41-6545-d507f2adf557 -13:13.895 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:13.895 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:13.895 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:13.895 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:13.895 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:13.895 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-36d547d5-ec50-cb41-6545-d507f2adf557 -13:13.895 [INFO] TestAgent.server: member joined, marking health alive: member=Node-36d547d5-ec50-cb41-6545-d507f2adf557 -13:13.895 [INFO] TestAgent.server: federation state anti-entropy synced -13:13.994 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:13.995 [INFO] TestAgent: Synced node info -13:13.995 [DEBUG] TestAgent: Node info in sync -13:14.146 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:14.146 [DEBUG] TestAgent: Node info in sync -13:14.231 [DEBUG] TestAgent.dns: request served from client: name=db.connect.consul. type=SRV class=IN latency=271.889µs client=127.0.0.1:36315 client_network=udp -13:14.238 [INFO] TestAgent: Requesting shutdown -13:14.238 [INFO] TestAgent.server: shutting down server -13:14.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.238 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.238 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.238 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.238 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.238 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.238 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:14.238 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:14.238Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.238Z [INFO] agent.router.manager: shutting down -13:14.238 [INFO] TestAgent: consul server down -13:14.238 [INFO] TestAgent: shutdown complete -13:14.238 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17140 network=tcp -13:14.238 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17140 network=udp -13:14.239 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17141 network=tcp -13:14.740 [INFO] TestAgent: Waiting for endpoints to shut down -13:14.740 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ConnectServiceLookup (0.99s) +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:26.091Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.102Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.103 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c3d9ae5b-4b34-c4a9-407f-66cd692dc891 Address:127.0.0.1:23405}]" +49:26.104 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23405 [Follower]" leader= +49:26.104 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c3d9ae5b-4b34-c4a9-407f-66cd692dc891.dc1 127.0.0.1 +49:26.104 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c3d9ae5b-4b34-c4a9-407f-66cd692dc891 127.0.0.1 +2022-12-04T05:49:26.104Z [INFO] agent.router: Initializing LAN area manager +49:26.104 [INFO] TestAgent.server: Adding LAN server: server="Node-c3d9ae5b-4b34-c4a9-407f-66cd692dc891 (Addr: tcp/127.0.0.1:23405) (DC: dc1)" +49:26.104 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c3d9ae5b-4b34-c4a9-407f-66cd692dc891.dc1 area=wan +49:26.104 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23966 network=udp +49:26.105 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23966 network=tcp +49:26.105 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23967 network=tcp +49:26.105 [INFO] TestAgent: started state syncer +49:26.105 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23406 network=tcp +49:26.160 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.160 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23405 [Candidate]" term=2 +49:26.160 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.160 [DEBUG] TestAgent.server.raft: vote granted: from=c3d9ae5b-4b34-c4a9-407f-66cd692dc891 term=2 tally=1 +49:26.160 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.160 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23405 [Leader]" +49:26.160 [INFO] TestAgent.server: cluster leadership acquired +49:26.160 [INFO] TestAgent.server: New leader elected: payload=Node-c3d9ae5b-4b34-c4a9-407f-66cd692dc891 +49:26.160 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23405 +49:26.161 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.163 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.163 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.163 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.163 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.163 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.163 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c3d9ae5b-4b34-c4a9-407f-66cd692dc891 +49:26.163 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c3d9ae5b-4b34-c4a9-407f-66cd692dc891 +49:26.164 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.191 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=167.678µs client=127.0.0.1:33485 client_network=udp +49:26.191 [INFO] TestAgent: Requesting shutdown +49:26.191 [INFO] TestAgent.server: shutting down server +49:26.191 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.191 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.191 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.191 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.191 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.191 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.191 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.191 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:26.191 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.191 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.191 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:26.191 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:26.191Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.191Z [INFO] agent.router.manager: shutting down +49:26.191 [INFO] TestAgent: consul server down +49:26.191 [INFO] TestAgent: shutdown complete +49:26.191 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23966 network=tcp +49:26.191 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23966 network=udp +49:26.191 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23967 network=tcp +49:26.691 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.691 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookupMultiAddrNoCNAME (0.61s) === CONT TestDNS_SOA_Settings -2021-10-31T23:13:14.114Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:14.166Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:14.167 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0056d565-c0af-d703-3095-87298347f70a Address:127.0.0.1:16244}]" -13:14.168 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0056d565-c0af-d703-3095-87298347f70a.dc1 127.0.0.1 -13:14.168 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0056d565-c0af-d703-3095-87298347f70a 127.0.0.1 -2021-10-31T23:13:14.168Z [INFO] agent.router: Initializing LAN area manager -13:14.169 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16225 network=udp -13:14.169 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16244 [Follower]" leader= -13:14.169 [INFO] TestAgent.server: Adding LAN server: server="Node-0056d565-c0af-d703-3095-87298347f70a (Addr: tcp/127.0.0.1:16244) (DC: dc1)" -13:14.169 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0056d565-c0af-d703-3095-87298347f70a.dc1 area=wan -13:14.169 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16225 network=tcp -13:14.179 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16226 network=tcp -13:14.179 [INFO] TestAgent: started state syncer -13:14.180 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16245 network=tcp -13:14.223 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:14.223 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16244 [Candidate]" term=2 -13:14.223 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:14.223 [DEBUG] TestAgent.server.raft: vote granted: from=0056d565-c0af-d703-3095-87298347f70a term=2 tally=1 -13:14.223 [INFO] TestAgent.server.raft: election won: tally=1 -13:14.223 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16244 [Leader]" -13:14.223 [INFO] TestAgent.server: cluster leadership acquired -13:14.223 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16244 -13:14.224 [INFO] TestAgent.server: New leader elected: payload=Node-0056d565-c0af-d703-3095-87298347f70a -13:14.224 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:14.226 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:14.226 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:14.226 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:14.226 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:14.226 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:14.226 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0056d565-c0af-d703-3095-87298347f70a -13:14.226 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0056d565-c0af-d703-3095-87298347f70a -13:14.227 [INFO] TestAgent.server: federation state anti-entropy synced -13:14.335 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=211.471µs client=127.0.0.1:55264 client_network=udp -13:14.335 [INFO] TestAgent: Requesting shutdown -13:14.335 [INFO] TestAgent.server: shutting down server -13:14.335 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.335 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.335 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.335 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.335 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.336 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.336 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.336 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:14.336 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.336 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.336 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:14.336 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:14.336Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.336Z [INFO] agent.router.manager: shutting down -13:14.336 [INFO] TestAgent: consul server down -13:14.336 [INFO] TestAgent: shutdown complete -13:14.336 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16225 network=tcp -13:14.336 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16225 network=udp -13:14.336 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16226 network=tcp -13:14.839 [INFO] TestAgent: Waiting for endpoints to shut down -13:14.839 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ExternalServiceLookup (0.76s) -=== CONT TestDNS_ReverseLookup -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:13.947Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:14.032Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:14.046 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aab2f651-eb31-9420-b000-db6ffc572cd1 Address:127.0.0.1:16132}]" -13:14.047 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aab2f651-eb31-9420-b000-db6ffc572cd1.dc1 127.0.0.1 -13:14.047 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16132 [Follower]" leader= -13:14.047 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aab2f651-eb31-9420-b000-db6ffc572cd1 127.0.0.1 -2021-10-31T23:13:14.047Z [INFO] agent.router: Initializing LAN area manager -13:14.048 [INFO] TestAgent.server: Adding LAN server: server="Node-aab2f651-eb31-9420-b000-db6ffc572cd1 (Addr: tcp/127.0.0.1:16132) (DC: dc1)" -13:14.048 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=udp -13:14.048 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aab2f651-eb31-9420-b000-db6ffc572cd1.dc1 area=wan -13:14.048 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=tcp -13:14.049 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16128 network=tcp -13:14.049 [INFO] TestAgent: started state syncer -13:14.049 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16133 network=tcp -13:14.117 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:14.117 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16132 [Candidate]" term=2 -13:14.118 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:14.118 [DEBUG] TestAgent.server.raft: vote granted: from=aab2f651-eb31-9420-b000-db6ffc572cd1 term=2 tally=1 -13:14.118 [INFO] TestAgent.server.raft: election won: tally=1 -13:14.118 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16132 [Leader]" -13:14.118 [INFO] TestAgent.server: cluster leadership acquired -13:14.118 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16132 -13:14.118 [INFO] TestAgent.server: New leader elected: payload=Node-aab2f651-eb31-9420-b000-db6ffc572cd1 -13:14.118 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:14.120 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:14.120 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:14.120 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:14.120 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:14.120 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:14.120 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aab2f651-eb31-9420-b000-db6ffc572cd1 -13:14.120 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aab2f651-eb31-9420-b000-db6ffc572cd1 -13:14.121 [INFO] TestAgent.server: federation state anti-entropy synced -13:14.277 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:14.278 [INFO] TestAgent: Synced node info -13:14.278 [DEBUG] TestAgent: Node info in sync -13:14.364 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=264.041µs client=127.0.0.1:52020 client_network=udp -13:14.364 [INFO] TestAgent: Requesting shutdown -13:14.364 [INFO] TestAgent.server: shutting down server -13:14.364 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.364 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.364 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.364 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.364 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.364 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.364 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.364 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.364 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.364 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:14.364 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:14.365Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.365Z [INFO] agent.router.manager: shutting down -13:14.365 [INFO] TestAgent: consul server down -13:14.365 [INFO] TestAgent: shutdown complete -13:14.365 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=tcp -13:14.365 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=udp -13:14.365 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16128 network=tcp -13:14.870 [INFO] TestAgent: Waiting for endpoints to shut down -13:14.870 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookupMultiAddrNoCNAME (0.97s) -=== CONT TestDNS_ServiceReverseLookup -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_IngressServiceLookup/api.ingress.consul. -=== RUN TestDNS_IngressServiceLookup/api.ingress.dc1.consul. -=== RUN TestDNS_IngressServiceLookup/db.ingress.consul. -=== RUN TestDNS_IngressServiceLookup/db.ingress.dc1.consul. -2021-10-31T23:13:13.953Z [WARN] agent: Node name "test node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2021-10-31T23:13:13.959Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:14.032Z [WARN] agent.auto_config: Node name "test node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2021-10-31T23:13:14.032Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:14.035 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d2cff269-bb7a-270c-f138-140ee658bed2 Address:127.0.0.1:16139}]" -13:14.036 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test node.dc1 127.0.0.1 -13:14.037 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test node 127.0.0.1 -2021-10-31T23:13:14.037Z [INFO] agent.router: Initializing LAN area manager -13:14.037 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16139 [Follower]" leader= -13:14.038 [INFO] TestAgent.server: Adding LAN server: server="test node (Addr: tcp/127.0.0.1:16139) (DC: dc1)" -13:14.038 [INFO] TestAgent.server: Handled event for server in area: event=member-join server="test node.dc1" area=wan -13:14.038 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=tcp -13:14.046 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=udp -13:14.047 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16135 network=tcp -13:14.047 [INFO] TestAgent: started state syncer -13:14.047 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16140 network=tcp -13:14.086 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:14.086 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16139 [Candidate]" term=2 -13:14.086 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:14.086 [DEBUG] TestAgent.server.raft: vote granted: from=d2cff269-bb7a-270c-f138-140ee658bed2 term=2 tally=1 -13:14.086 [INFO] TestAgent.server.raft: election won: tally=1 -13:14.086 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16139 [Leader]" -13:14.086 [INFO] TestAgent.server: cluster leadership acquired -13:14.086 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16139 -13:14.087 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:14.088 [INFO] TestAgent.server: New leader elected: payload="test node" -13:14.089 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:14.089 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:14.089 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:14.089 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:14.089 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:14.089 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node="test node" -13:14.089 [INFO] TestAgent.server: member joined, marking health alive: member="test node" -13:14.098 [INFO] TestAgent.server: federation state anti-entropy synced -13:14.137 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:14.138 [INFO] TestAgent: Synced node info -13:14.138 [DEBUG] TestAgent: Node info in sync -13:14.457 [ERROR] TestAgent.dns: Infinite recursion detected for name, won't perform any CNAME resolution.: name=web.service.consul. -13:14.457 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=A class=IN latency=259.574µs client=127.0.0.1:51909 client_network=udp -13:14.457 [INFO] TestAgent: Requesting shutdown -13:14.457 [INFO] TestAgent.server: shutting down server -13:14.457 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.457 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.457 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.457 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.457 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.457 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.457 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.457 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.457 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.458 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:14.458 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:14.458Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.458Z [INFO] agent.router.manager: shutting down -13:14.730 [INFO] TestAgent: consul server down -13:14.730 [INFO] TestAgent: shutdown complete -13:14.730 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=tcp -13:14.730 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=udp -13:14.730 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16135 network=tcp -13:15.231 [INFO] TestAgent: Waiting for endpoints to shut down -13:15.231 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_InifiniteRecursion (1.31s) +2022-12-04T05:49:26.087Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.099Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.100 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7f7e1c8a-378f-f130-29b2-a69d22cd6bdd Address:127.0.0.1:23964}]" +49:26.100 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23964 [Follower]" leader= +49:26.101 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7f7e1c8a-378f-f130-29b2-a69d22cd6bdd.dc1 127.0.0.1 +49:26.101 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7f7e1c8a-378f-f130-29b2-a69d22cd6bdd 127.0.0.1 +2022-12-04T05:49:26.101Z [INFO] agent.router: Initializing LAN area manager +49:26.101 [DEBUG] TestAgent.dns: recursor enabled +49:26.102 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23903 network=udp +49:26.102 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7f7e1c8a-378f-f130-29b2-a69d22cd6bdd.dc1 area=wan +49:26.102 [INFO] TestAgent.server: Adding LAN server: server="Node-7f7e1c8a-378f-f130-29b2-a69d22cd6bdd (Addr: tcp/127.0.0.1:23964) (DC: dc1)" +49:26.102 [DEBUG] TestAgent.dns: recursor enabled +49:26.102 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23903 network=tcp +49:26.102 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23904 network=tcp +49:26.102 [INFO] TestAgent: started state syncer +49:26.102 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23965 network=tcp +49:26.148 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.149 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23964 [Candidate]" term=2 +49:26.149 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.149 [DEBUG] TestAgent.server.raft: vote granted: from=7f7e1c8a-378f-f130-29b2-a69d22cd6bdd term=2 tally=1 +49:26.149 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.149 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23964 [Leader]" +49:26.149 [INFO] TestAgent.server: cluster leadership acquired +49:26.149 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23964 +49:26.149 [INFO] TestAgent.server: New leader elected: payload=Node-7f7e1c8a-378f-f130-29b2-a69d22cd6bdd +49:26.150 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.152 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.152 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.152 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.152 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.152 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.152 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7f7e1c8a-378f-f130-29b2-a69d22cd6bdd +49:26.152 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7f7e1c8a-378f-f130-29b2-a69d22cd6bdd +49:26.154 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.227 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=122.552µs +49:26.227 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=315.373µs client=127.0.0.1:47029 client_network=udp +49:26.227 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=93.965µs +49:26.228 [DEBUG] TestAgent.dns: request served from client: name=36051701-ab18-e855-88d2-8fba67024bc0.query.consul. type=SRV class=IN latency=228.642µs client=127.0.0.1:55861 client_network=udp +49:26.228 [INFO] TestAgent: Requesting shutdown +49:26.228 [INFO] TestAgent.server: shutting down server +49:26.228 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.228 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.228 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.228 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.228 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.228 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.228 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.228 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:26.228 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.228 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:26.228 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.228 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:26.228Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.228Z [INFO] agent.router.manager: shutting down +49:26.228 [INFO] TestAgent: consul server down +49:26.228 [INFO] TestAgent: shutdown complete +49:26.228 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23903 network=tcp +49:26.228 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23903 network=udp +49:26.228 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23904 network=tcp +49:26.729 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.729 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_ServiceAddress_SRV (0.66s) +=== CONT TestDNS_ReverseLookup_CustomDomain +2022-12-04T05:49:26.071Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.084Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.085 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e2147c09-e3ec-9958-a0f7-587f351b4a6f Address:127.0.0.1:23929}]" +49:26.085 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23929 [Follower]" leader= +49:26.086 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e2147c09-e3ec-9958-a0f7-587f351b4a6f.dc1 127.0.0.1 +49:26.087 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e2147c09-e3ec-9958-a0f7-587f351b4a6f 127.0.0.1 +2022-12-04T05:49:26.087Z [INFO] agent.router: Initializing LAN area manager +49:26.087 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e2147c09-e3ec-9958-a0f7-587f351b4a6f.dc1 area=wan +49:26.087 [INFO] TestAgent.server: Adding LAN server: server="Node-e2147c09-e3ec-9958-a0f7-587f351b4a6f (Addr: tcp/127.0.0.1:23929) (DC: dc1)" +49:26.087 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23917 network=udp +49:26.087 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23917 network=tcp +49:26.087 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23932 network=tcp +49:26.087 [INFO] TestAgent: started state syncer +49:26.088 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23930 network=tcp +49:26.136 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.136 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23929 [Candidate]" term=2 +49:26.136 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.136 [DEBUG] TestAgent.server.raft: vote granted: from=e2147c09-e3ec-9958-a0f7-587f351b4a6f term=2 tally=1 +49:26.136 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.136 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23929 [Leader]" +49:26.136 [INFO] TestAgent.server: cluster leadership acquired +49:26.137 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23929 +49:26.137 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.138 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.138 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.138 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.138 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.138 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.139 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e2147c09-e3ec-9958-a0f7-587f351b4a6f +49:26.139 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e2147c09-e3ec-9958-a0f7-587f351b4a6f +49:26.139 [INFO] TestAgent.server: New leader elected: payload=Node-e2147c09-e3ec-9958-a0f7-587f351b4a6f +49:26.139 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.228 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=82.177µs client=127.0.0.1:49002 client_network=udp +49:26.228 [DEBUG] TestAgent.dns: request served from client: name=0861212b-7e29-83ba-a5ac-e28f2f4557bf.query.consul. type=SRV class=IN latency=89.633µs client=127.0.0.1:50814 client_network=udp +49:26.228 [DEBUG] TestAgent.dns: request served from client: name=nodb.service.consul. type=SRV class=IN latency=63.853µs client=127.0.0.1:38850 client_network=udp +49:26.229 [DEBUG] TestAgent.dns: request served from client: name=nope.query.consul. type=SRV class=IN latency=99.366µs client=127.0.0.1:41487 client_network=udp +49:26.229 [INFO] TestAgent: Requesting shutdown +49:26.229 [INFO] TestAgent.server: shutting down server +49:26.229 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.229 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.229 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.229 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.229 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.229 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.229 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.229 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.229 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:26.229 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:26.229 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.229 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:26.229Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.229Z [INFO] agent.router.manager: shutting down +49:26.229 [INFO] TestAgent: consul server down +49:26.229 [INFO] TestAgent: shutdown complete +49:26.229 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23917 network=tcp +49:26.229 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23917 network=udp +49:26.229 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23932 network=tcp +49:26.729 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.729 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup (0.67s) === CONT TestDNS_ServiceReverseLookup_CustomDomain -2021-10-31T23:13:14.071Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:14.107Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:14.109 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:85d6c85d-6aba-0421-6982-b51ae7e70992 Address:127.0.0.1:16223}]" -13:14.110 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16223 [Follower]" leader= -13:14.115 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-85d6c85d-6aba-0421-6982-b51ae7e70992.dc1 127.0.0.1 -13:14.116 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-85d6c85d-6aba-0421-6982-b51ae7e70992 127.0.0.1 -2021-10-31T23:13:14.116Z [INFO] agent.router: Initializing LAN area manager -13:14.116 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16141 network=udp -13:14.116 [INFO] TestAgent.server: Adding LAN server: server="Node-85d6c85d-6aba-0421-6982-b51ae7e70992 (Addr: tcp/127.0.0.1:16223) (DC: dc1)" -13:14.116 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-85d6c85d-6aba-0421-6982-b51ae7e70992.dc1 area=wan -13:14.117 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16141 network=tcp -13:14.117 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16142 network=tcp -13:14.117 [INFO] TestAgent: started state syncer -13:14.117 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16224 network=tcp -13:14.156 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:14.156 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16223 [Candidate]" term=2 -13:14.156 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:14.156 [DEBUG] TestAgent.server.raft: vote granted: from=85d6c85d-6aba-0421-6982-b51ae7e70992 term=2 tally=1 -13:14.156 [INFO] TestAgent.server.raft: election won: tally=1 -13:14.156 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16223 [Leader]" -13:14.156 [INFO] TestAgent.server: cluster leadership acquired -13:14.156 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16223 -13:14.157 [INFO] TestAgent.server: New leader elected: payload=Node-85d6c85d-6aba-0421-6982-b51ae7e70992 -13:14.157 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:14.159 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:14.159 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:14.159 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:14.159 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:14.159 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:14.159 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-85d6c85d-6aba-0421-6982-b51ae7e70992 -13:14.159 [INFO] TestAgent.server: member joined, marking health alive: member=Node-85d6c85d-6aba-0421-6982-b51ae7e70992 -13:14.160 [INFO] TestAgent.server: federation state anti-entropy synced -13:14.217 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:14.231 [INFO] TestAgent: Synced node info -13:14.494 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=307.18µs client=127.0.0.1:51045 client_network=udp -13:14.494 [INFO] TestAgent: Requesting shutdown -13:14.495 [INFO] TestAgent.server: shutting down server -13:14.495 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.495 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.495 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.495 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.495 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.495 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.495 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.495 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.495 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:14.495 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.495 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:14.495Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.495Z [INFO] agent.router.manager: shutting down -13:14.758 [INFO] TestAgent: consul server down -13:14.758 [INFO] TestAgent: shutdown complete -13:14.758 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16141 network=tcp -13:14.758 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16141 network=udp -13:14.758 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16142 network=tcp -13:15.259 [INFO] TestAgent: Waiting for endpoints to shut down -13:15.259 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookupPreferNoCNAME (1.22s) +2022-12-04T05:49:26.070Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.084Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.086 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5878b181-16a8-a7a8-5d8f-b72632ddfb95 Address:127.0.0.1:23901}]" +49:26.086 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23901 [Follower]" leader= +49:26.086 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: server1.dc1 127.0.0.1 +49:26.087 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: server1 127.0.0.1 +2022-12-04T05:49:26.087Z [INFO] agent.router: Initializing LAN area manager +49:26.087 [INFO] TestAgent.server: Adding LAN server: server="server1 (Addr: tcp/127.0.0.1:23901) (DC: dc1)" +49:26.087 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23931 network=udp +49:26.087 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=server1.dc1 area=wan +49:26.087 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23931 network=tcp +49:26.088 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23911 network=tcp +49:26.088 [INFO] TestAgent: started state syncer +49:26.088 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23902 network=tcp +49:26.129 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.129 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23901 [Candidate]" term=2 +49:26.129 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.129 [DEBUG] TestAgent.server.raft: vote granted: from=5878b181-16a8-a7a8-5d8f-b72632ddfb95 term=2 tally=1 +49:26.129 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.129 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23901 [Leader]" +49:26.129 [INFO] TestAgent.server: cluster leadership acquired +49:26.129 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23901 +49:26.129 [INFO] TestAgent.server: New leader elected: payload=server1 +49:26.130 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.131 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.131 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.131 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.131 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.131 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.131 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=server1 +49:26.131 [INFO] TestAgent.server: member joined, marking health alive: member=server1 +49:26.132 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.247 [DEBUG] TestAgent.dns: request served from client: name=something.node.consul. type=NS class=IN latency=192.214µs client=127.0.0.1:44098 client_network=udp +49:26.247 [INFO] TestAgent: Requesting shutdown +49:26.247 [INFO] TestAgent.server: shutting down server +49:26.247 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.247 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.247 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.247 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.247 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.247 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.247 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:26.247 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.247 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:26.247 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:26.247Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.247Z [INFO] agent.router.manager: shutting down +49:26.248 [INFO] TestAgent: consul server down +49:26.248 [INFO] TestAgent: shutdown complete +49:26.248 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23931 network=tcp +49:26.248 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23931 network=udp +49:26.248 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23911 network=tcp +49:26.748 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.748 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NSRecords (0.69s) === CONT TestDNS_EDNS0_ECS -2021-10-31T23:13:14.152Z [WARN] agent: Node name "my.test-node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2021-10-31T23:13:14.152Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:14.203Z [WARN] agent.auto_config: Node name "my.test-node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2021-10-31T23:13:14.204Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:14.211 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:66bbdcb8-f243-1fbd-86ce-7cc0dca65925 Address:127.0.0.1:16153}]" -13:14.212 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: my.test-node.dc1 127.0.0.1 -13:14.212 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: my.test-node 127.0.0.1 -2021-10-31T23:13:14.212Z [INFO] agent.router: Initializing LAN area manager -13:14.212 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16153 [Follower]" leader= -13:14.212 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16246 network=udp -13:14.212 [INFO] TestAgent.server: Adding LAN server: server="my.test-node (Addr: tcp/127.0.0.1:16153) (DC: dc1)" -13:14.213 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=my.test-node.dc1 area=wan -13:14.213 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16246 network=tcp -13:14.213 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16247 network=tcp -13:14.218 [INFO] TestAgent: started state syncer -13:14.218 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16154 network=tcp -13:14.257 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:14.257 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16153 [Candidate]" term=2 -13:14.258 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:14.258 [DEBUG] TestAgent.server.raft: vote granted: from=66bbdcb8-f243-1fbd-86ce-7cc0dca65925 term=2 tally=1 -13:14.258 [INFO] TestAgent.server.raft: election won: tally=1 -13:14.258 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16153 [Leader]" -13:14.258 [INFO] TestAgent.server: cluster leadership acquired -13:14.258 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16153 -13:14.259 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:14.260 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:14.260 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:14.260 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:14.260 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:14.260 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:14.260 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=my.test-node -13:14.260 [INFO] TestAgent.server: member joined, marking health alive: member=my.test-node -13:14.261 [INFO] TestAgent.server: New leader elected: payload=my.test-node -13:14.261 [INFO] TestAgent.server: federation state anti-entropy synced -13:14.542 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:14.731 [INFO] TestAgent: Synced node info -13:14.734 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=220.536µs client=127.0.0.1:51502 client_network=udp -13:14.734 [INFO] TestAgent: Requesting shutdown -13:14.734 [INFO] TestAgent.server: shutting down server -13:14.734 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.734 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.734 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.734 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.734 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.734 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.734 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.734 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.734 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.734 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:14.734 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:14.735Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.735Z [INFO] agent.router.manager: shutting down -13:14.778 [INFO] TestAgent: consul server down -13:14.778 [INFO] TestAgent: shutdown complete -13:14.778 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16246 network=tcp -13:14.778 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16246 network=udp -13:14.778 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16247 network=tcp -13:15.278 [INFO] TestAgent: Waiting for endpoints to shut down -13:15.278 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookupWithInternalServiceAddress (1.17s) -=== CONT TestDNS_ReverseLookup_IPV6 -2021-10-31T23:13:14.331Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:14.399Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:14.442 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ce32e0b8-ef54-198b-baf6-aaeb16df6b13 Address:127.0.0.1:16216}]" -13:14.443 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ce32e0b8-ef54-198b-baf6-aaeb16df6b13.dc1 127.0.0.1 -13:14.443 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16216 [Follower]" leader= -13:14.444 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ce32e0b8-ef54-198b-baf6-aaeb16df6b13 127.0.0.1 -2021-10-31T23:13:14.444Z [INFO] agent.router: Initializing LAN area manager -13:14.444 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16155 network=udp -13:14.444 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ce32e0b8-ef54-198b-baf6-aaeb16df6b13.dc1 area=wan -13:14.444 [INFO] TestAgent.server: Adding LAN server: server="Node-ce32e0b8-ef54-198b-baf6-aaeb16df6b13 (Addr: tcp/127.0.0.1:16216) (DC: dc1)" -13:14.444 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16155 network=tcp -13:14.445 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16156 network=tcp -13:14.445 [INFO] TestAgent: started state syncer -13:14.445 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16217 network=tcp -13:14.481 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:14.481 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16216 [Candidate]" term=2 -13:14.482 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:14.482 [DEBUG] TestAgent.server.raft: vote granted: from=ce32e0b8-ef54-198b-baf6-aaeb16df6b13 term=2 tally=1 -13:14.482 [INFO] TestAgent.server.raft: election won: tally=1 -13:14.482 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16216 [Leader]" -13:14.482 [INFO] TestAgent.server: cluster leadership acquired -13:14.482 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16216 -13:14.482 [INFO] TestAgent.server: New leader elected: payload=Node-ce32e0b8-ef54-198b-baf6-aaeb16df6b13 -13:14.782 [INFO] TestAgent: Synced node info -13:14.782 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:14.783 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:14.783 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:14.783 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:14.783 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:14.783 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:14.783 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ce32e0b8-ef54-198b-baf6-aaeb16df6b13 -13:14.784 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ce32e0b8-ef54-198b-baf6-aaeb16df6b13 -13:14.798 [INFO] TestAgent.server: federation state anti-entropy synced -13:14.810 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=459.576µs client=127.0.0.1:57222 client_network=udp -13:14.814 [INFO] TestAgent: Requesting shutdown -13:14.814 [INFO] TestAgent.server: shutting down server -13:14.814 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.814 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.814 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.814 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.814 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.814 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.814 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.814 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.814 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.814 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:14.814 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:14.814Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.814Z [INFO] agent.router.manager: shutting down -13:14.814 [INFO] TestAgent: consul server down -13:14.814 [INFO] TestAgent: shutdown complete -13:14.814 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16155 network=tcp -13:14.814 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16155 network=udp -13:14.814 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16156 network=tcp -13:15.317 [INFO] TestAgent: Waiting for endpoints to shut down -13:15.317 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookupNoMultiCNAME (1.03s) -=== CONT TestDNS_NodeLookup_CNAME -2021-10-31T23:13:14.379Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:14.435Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:14.444 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c5a309b5-8cde-17ce-871b-26dd5aeaedac Address:127.0.0.1:16146}]" -13:14.444 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c5a309b5-8cde-17ce-871b-26dd5aeaedac.dc1 127.0.0.1 -13:14.445 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c5a309b5-8cde-17ce-871b-26dd5aeaedac 127.0.0.1 -2021-10-31T23:13:14.445Z [INFO] agent.router: Initializing LAN area manager -13:14.445 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16218 network=udp -13:14.445 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16146 [Follower]" leader= -13:14.445 [INFO] TestAgent.server: Adding LAN server: server="Node-c5a309b5-8cde-17ce-871b-26dd5aeaedac (Addr: tcp/127.0.0.1:16146) (DC: dc1)" -13:14.445 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c5a309b5-8cde-17ce-871b-26dd5aeaedac.dc1 area=wan -13:14.445 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16218 network=tcp -13:14.446 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16219 network=tcp -13:14.446 [INFO] TestAgent: started state syncer -13:14.446 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16147 network=tcp -13:14.505 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:14.506 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16146 [Candidate]" term=2 -13:14.506 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:14.506 [DEBUG] TestAgent.server.raft: vote granted: from=c5a309b5-8cde-17ce-871b-26dd5aeaedac term=2 tally=1 -13:14.506 [INFO] TestAgent.server.raft: election won: tally=1 -13:14.506 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16146 [Leader]" -13:14.506 [INFO] TestAgent.server: cluster leadership acquired -13:14.506 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16146 -13:14.507 [INFO] TestAgent.server: New leader elected: payload=Node-c5a309b5-8cde-17ce-871b-26dd5aeaedac -13:14.782 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:14.795 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:14.795 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:14.795 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:14.795 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:14.795 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:14.795 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c5a309b5-8cde-17ce-871b-26dd5aeaedac -13:14.795 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c5a309b5-8cde-17ce-871b-26dd5aeaedac -13:14.796 [INFO] TestAgent.server: federation state anti-entropy synced -13:14.801 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:14.802 [INFO] TestAgent: Synced node info -13:14.802 [DEBUG] TestAgent: Node info in sync -13:14.843 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=SRV class=IN latency=148.55µs client=127.0.0.1:51501 client_network=udp -13:14.843 [DEBUG] TestAgent.dns: request served from client: name=925b07f9-408a-6b02-8cd7-d877c5aa2a16.query.consul. type=SRV class=IN latency=116.034µs client=127.0.0.1:49245 client_network=udp -13:14.843 [DEBUG] TestAgent.dns: request served from client: name=nodb.service.consul. type=SRV class=IN latency=66.74µs client=127.0.0.1:43075 client_network=udp -13:14.843 [DEBUG] TestAgent.dns: request served from client: name=nope.query.consul. type=SRV class=IN latency=51.815µs client=127.0.0.1:39838 client_network=udp -13:14.843 [INFO] TestAgent: Requesting shutdown -13:14.843 [INFO] TestAgent.server: shutting down server -13:14.843 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.843 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.843 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.843 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.843 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.843 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.843 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.843 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.843 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.843 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:14.844 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:14.844Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.844Z [INFO] agent.router.manager: shutting down -13:14.844 [INFO] TestAgent: consul server down -13:14.844 [INFO] TestAgent: shutdown complete -13:14.844 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16218 network=tcp -13:14.844 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16218 network=udp -13:14.844 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16219 network=tcp -13:15.345 [INFO] TestAgent: Waiting for endpoints to shut down -13:15.345 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup (1.05s) +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:25.864Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.890Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.892 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:efde3eb5-294d-9989-e027-1fa1463a8c33 Address:[::1]:23908}]" +49:25.892 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: server1.dc1 ::1 +49:25.893 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: server1 ::1 +2022-12-04T05:49:25.893Z [INFO] agent.router: Initializing LAN area manager +49:25.893 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23924 network=udp +49:25.893 [INFO] TestAgent.server.raft: entering follower state: follower="Node at [::1]:23908 [Follower]" leader= +49:25.894 [INFO] TestAgent.server: Adding LAN server: server="server1 (Addr: tcp/[::1]:23908) (DC: dc1)" +49:25.894 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=server1.dc1 area=wan +49:25.894 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23924 network=tcp +49:25.894 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23946 network=tcp +49:25.894 [INFO] TestAgent: started state syncer +49:25.894 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23909 network=tcp +49:25.960 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.960 [INFO] TestAgent.server.raft: entering candidate state: node="Node at [::1]:23908 [Candidate]" term=2 +49:25.960 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.960 [DEBUG] TestAgent.server.raft: vote granted: from=efde3eb5-294d-9989-e027-1fa1463a8c33 term=2 tally=1 +49:25.960 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.960 [INFO] TestAgent.server.raft: entering leader state: leader="Node at [::1]:23908 [Leader]" +49:25.960 [INFO] TestAgent.server: cluster leadership acquired +49:25.961 [INFO] TestAgent.server: New leader elected: payload=server1 +49:25.961 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=[::1]:23908 +49:25.961 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.962 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.962 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.962 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.962 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.962 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.962 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=server1 +49:25.962 [INFO] TestAgent.server: member joined, marking health alive: member=server1 +49:25.963 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.200 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:26.201 [INFO] TestAgent: Synced node info +49:26.301 [DEBUG] TestAgent.dns: request served from client: name=server1.node.dc1.consul. type=NS class=IN latency=158.162µs client=127.0.0.1:44560 client_network=udp +49:26.301 [INFO] TestAgent: Requesting shutdown +49:26.301 [INFO] TestAgent.server: shutting down server +49:26.301 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.301 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.301 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.301 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.301 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.301 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.301 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.301 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.301 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.301 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:26.301 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:26.301Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.301Z [INFO] agent.router.manager: shutting down +49:26.302 [INFO] TestAgent: consul server down +49:26.302 [INFO] TestAgent: shutdown complete +49:26.302 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23924 network=tcp +49:26.302 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23924 network=udp +49:26.302 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23946 network=tcp +49:26.802 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.802 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NSRecords_IPV6 (0.95s) +=== CONT TestDNS_ReverseLookup +2022-12-04T05:49:26.301Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.310Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.311 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4ba34693-c534-1fab-8b83-5f9c67920745 Address:127.0.0.1:23971}]" +49:26.311 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23971 [Follower]" leader= +49:26.311 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4ba34693-c534-1fab-8b83-5f9c67920745.dc1 127.0.0.1 +49:26.312 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4ba34693-c534-1fab-8b83-5f9c67920745 127.0.0.1 +2022-12-04T05:49:26.312Z [INFO] agent.router: Initializing LAN area manager +49:26.312 [INFO] TestAgent.server: Adding LAN server: server="Node-4ba34693-c534-1fab-8b83-5f9c67920745 (Addr: tcp/127.0.0.1:23971) (DC: dc1)" +49:26.312 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4ba34693-c534-1fab-8b83-5f9c67920745.dc1 area=wan +49:26.312 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23945 network=udp +49:26.312 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23945 network=tcp +49:26.313 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23953 network=tcp +49:26.313 [INFO] TestAgent: started state syncer +49:26.313 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23972 network=tcp +49:26.371 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.371 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23971 [Candidate]" term=2 +49:26.372 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.372 [DEBUG] TestAgent.server.raft: vote granted: from=4ba34693-c534-1fab-8b83-5f9c67920745 term=2 tally=1 +49:26.372 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.372 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23971 [Leader]" +49:26.372 [INFO] TestAgent.server: cluster leadership acquired +49:26.372 [INFO] TestAgent.server: New leader elected: payload=Node-4ba34693-c534-1fab-8b83-5f9c67920745 +49:26.372 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23971 +49:26.372 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.374 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.374 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.374 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.374 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.374 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.374 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4ba34693-c534-1fab-8b83-5f9c67920745 +49:26.374 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4ba34693-c534-1fab-8b83-5f9c67920745 +49:26.374 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.382 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:26.383 [INFO] TestAgent: Synced node info +49:26.400 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=208.129µs client=127.0.0.1:45451 client_network=udp +49:26.400 [INFO] TestAgent: Requesting shutdown +49:26.400 [INFO] TestAgent.server: shutting down server +49:26.400 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.400 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.400 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.400 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.400 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.400 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.401 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.401 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.401 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.401 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:26.401 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:26.401Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.408Z [INFO] agent.router.manager: shutting down +49:26.408 [INFO] TestAgent: consul server down +49:26.408 [INFO] TestAgent: shutdown complete +49:26.408 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23945 network=tcp +49:26.408 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23945 network=udp +49:26.409 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23953 network=tcp +49:26.909 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.909 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceReverseLookup (0.62s) === CONT TestDNS_EDNS0 -2021-10-31T23:13:13.167Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:13.181Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:13.184 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:47b1785e-2b29-dd67-2721-e0a80eb9948e Address:127.0.0.1:17082}]" -13:13.185 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-47b1785e-2b29-dd67-2721-e0a80eb9948e.dc1 127.0.0.1 -13:13.185 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17082 [Follower]" leader= -13:13.186 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-47b1785e-2b29-dd67-2721-e0a80eb9948e 127.0.0.1 -2021-10-31T23:13:13.186Z [INFO] agent.router: Initializing LAN area manager -13:13.186 [INFO] TestAgent.server: Adding LAN server: server="Node-47b1785e-2b29-dd67-2721-e0a80eb9948e (Addr: tcp/127.0.0.1:17082) (DC: dc1)" -13:13.186 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-47b1785e-2b29-dd67-2721-e0a80eb9948e.dc1 area=wan -13:13.187 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17077 network=udp -13:13.187 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17077 network=tcp -13:13.188 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17078 network=tcp -13:13.188 [INFO] TestAgent: started state syncer -13:13.188 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17083 network=tcp -13:13.230 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:13.230 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17082 [Candidate]" term=2 -13:13.230 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:13.230 [DEBUG] TestAgent.server.raft: vote granted: from=47b1785e-2b29-dd67-2721-e0a80eb9948e term=2 tally=1 -13:13.230 [INFO] TestAgent.server.raft: election won: tally=1 -13:13.230 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17082 [Leader]" -13:13.230 [INFO] TestAgent.server: cluster leadership acquired -13:13.231 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17082 -13:13.231 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:13.233 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:13.233 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:13.233 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:13.233 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:13.233 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:13.233 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-47b1785e-2b29-dd67-2721-e0a80eb9948e -13:13.233 [INFO] TestAgent.server: member joined, marking health alive: member=Node-47b1785e-2b29-dd67-2721-e0a80eb9948e -13:13.233 [INFO] TestAgent.server: New leader elected: payload=Node-47b1785e-2b29-dd67-2721-e0a80eb9948e -13:13.234 [INFO] TestAgent.server: federation state anti-entropy synced -13:13.295 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:13.295 [INFO] TestAgent: Synced node info -13:13.295 [DEBUG] TestAgent: Node info in sync -13:14.400 [WARN] TestAgent.server.rpc: Attempting to apply large raft entry: size_in_bytes=4719032 -13:14.468 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:14.468 [DEBUG] TestAgent: Node info in sync -13:14.845 [INFO] TestAgent: Requesting shutdown -13:14.846 [INFO] TestAgent.server: shutting down server -13:14.846 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.846 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.846 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.846 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.846 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.846 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.846 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.846 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:14.846 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.846 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.846 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:14.846Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.846Z [INFO] agent.router.manager: shutting down -13:14.846 [INFO] TestAgent: consul server down -13:14.846 [INFO] TestAgent: shutdown complete -13:14.846 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17077 network=tcp -13:14.846 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17077 network=udp -13:14.846 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17078 network=tcp -13:15.349 [INFO] TestAgent: Waiting for endpoints to shut down -13:15.349 [INFO] TestAgent: Endpoints down -[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! ---- PASS: TestTxnEndpoint_Bad_Size_Net (10.69s) - --- PASS: TestTxnEndpoint_Bad_Size_Net/exceeds_default_limits (1.95s) - --- PASS: TestTxnEndpoint_Bad_Size_Net/exceeds_configured_max_txn_len (2.04s) - --- PASS: TestTxnEndpoint_Bad_Size_Net/exceeds_default_max_kv_value_size (2.16s) - --- PASS: TestTxnEndpoint_Bad_Size_Net/allowed (2.30s) - --- PASS: TestTxnEndpoint_Bad_Size_Net/allowed_kv_max_backward_compatible (2.22s) +2022-12-04T05:49:26.262Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.272Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.274 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bc2d3096-d18f-b42b-c12e-6b76863be55b Address:127.0.0.1:23992}]" +49:26.274 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23992 [Follower]" leader= +49:26.275 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bc2d3096-d18f-b42b-c12e-6b76863be55b.dc1 127.0.0.1 +49:26.275 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bc2d3096-d18f-b42b-c12e-6b76863be55b 127.0.0.1 +2022-12-04T05:49:26.275Z [INFO] agent.router: Initializing LAN area manager +49:26.276 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23952 network=udp +49:26.276 [INFO] TestAgent.server: Adding LAN server: server="Node-bc2d3096-d18f-b42b-c12e-6b76863be55b (Addr: tcp/127.0.0.1:23992) (DC: dc1)" +49:26.276 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bc2d3096-d18f-b42b-c12e-6b76863be55b.dc1 area=wan +49:26.276 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23952 network=tcp +49:26.277 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23939 network=tcp +49:26.277 [INFO] TestAgent: started state syncer +49:26.277 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23993 network=tcp +49:26.313 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.313 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23992 [Candidate]" term=2 +49:26.314 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.314 [DEBUG] TestAgent.server.raft: vote granted: from=bc2d3096-d18f-b42b-c12e-6b76863be55b term=2 tally=1 +49:26.314 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.314 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23992 [Leader]" +49:26.314 [INFO] TestAgent.server: cluster leadership acquired +49:26.314 [INFO] TestAgent.server: New leader elected: payload=Node-bc2d3096-d18f-b42b-c12e-6b76863be55b +49:26.314 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23992 +49:26.315 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.316 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.316 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.316 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.316 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.316 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.316 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bc2d3096-d18f-b42b-c12e-6b76863be55b +49:26.316 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bc2d3096-d18f-b42b-c12e-6b76863be55b +49:26.316 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.410 [DEBUG] TestAgent.dns: request served from client: question="{9.2.3.8.2.4.0.0.0.0.f.f.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa. 255 1}" latency=245.85µs client=127.0.0.1:37600 client_network=udp +49:26.410 [INFO] TestAgent: Requesting shutdown +49:26.410 [INFO] TestAgent.server: shutting down server +49:26.410 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.410 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.410 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.410 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.410 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.410 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.410 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.410 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:26.410 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.410 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.410 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:26.410 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:26.410Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.410Z [INFO] agent.router.manager: shutting down +49:26.411 [INFO] TestAgent: consul server down +49:26.411 [INFO] TestAgent: shutdown complete +49:26.411 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23952 network=tcp +49:26.411 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23952 network=udp +49:26.411 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23939 network=tcp +49:26.911 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.911 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:26.209Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.220Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.222 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ffd02138-6535-1dad-38a6-5726115ec41a Address:127.0.0.1:23342}]" +49:26.222 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23342 [Follower]" leader= +49:26.222 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ffd02138-6535-1dad-38a6-5726115ec41a.dc1 127.0.0.1 +49:26.223 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ffd02138-6535-1dad-38a6-5726115ec41a 127.0.0.1 +2022-12-04T05:49:26.223Z [INFO] agent.router: Initializing LAN area manager +49:26.223 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23414 network=udp +49:26.223 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ffd02138-6535-1dad-38a6-5726115ec41a.dc1 area=wan +49:26.223 [INFO] TestAgent.server: Adding LAN server: server="Node-ffd02138-6535-1dad-38a6-5726115ec41a (Addr: tcp/127.0.0.1:23342) (DC: dc1)" +49:26.223 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23414 network=tcp +49:26.224 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23415 network=tcp +49:26.224 [INFO] TestAgent: started state syncer +49:26.224 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23343 network=tcp +49:26.275 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.275 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23342 [Candidate]" term=2 +49:26.275 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.275 [DEBUG] TestAgent.server.raft: vote granted: from=ffd02138-6535-1dad-38a6-5726115ec41a term=2 tally=1 +49:26.275 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.275 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23342 [Leader]" +49:26.275 [INFO] TestAgent.server: cluster leadership acquired +49:26.275 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23342 +49:26.276 [INFO] TestAgent.server: New leader elected: payload=Node-ffd02138-6535-1dad-38a6-5726115ec41a +49:26.276 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.282 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.282 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.282 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.282 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.282 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.283 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ffd02138-6535-1dad-38a6-5726115ec41a +49:26.283 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ffd02138-6535-1dad-38a6-5726115ec41a +49:26.284 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.410 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=300.192µs client=127.0.0.1:41498 client_network=udp +49:26.410 [INFO] TestAgent: Requesting shutdown +49:26.410 [INFO] TestAgent.server: shutting down server +49:26.410 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.410 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.410 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.410 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.410 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.410 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.410 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.410 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:26.410 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.410 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.410 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:26.410 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:26.411Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.411Z [INFO] agent.router.manager: shutting down +49:26.411 [INFO] TestAgent: consul server down +49:26.411 [INFO] TestAgent: shutdown complete +49:26.411 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23414 network=tcp +49:26.411 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23414 network=udp +49:26.411 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23415 network=tcp +49:26.911 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.911 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceReverseLookup_IPV6 (0.66s) +=== CONT TestDNS_NodeLookup_CNAME +--- PASS: TestDNS_ServiceLookupPreferNoCNAME (0.71s) === CONT TestDNS_NodeLookup_AAAA -2021-10-31T23:13:14.468Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:14.504Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:14.784 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:799eeec7-98f2-204f-0a27-9a9de583ed9d Address:127.0.0.1:16237}]" -13:14.785 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-799eeec7-98f2-204f-0a27-9a9de583ed9d.dc1 127.0.0.1 -13:14.785 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-799eeec7-98f2-204f-0a27-9a9de583ed9d 127.0.0.1 -2021-10-31T23:13:14.785Z [INFO] agent.router: Initializing LAN area manager -13:14.786 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16204 network=udp -13:14.786 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16237 [Follower]" leader= -13:14.786 [INFO] TestAgent.server: Adding LAN server: server="Node-799eeec7-98f2-204f-0a27-9a9de583ed9d (Addr: tcp/127.0.0.1:16237) (DC: dc1)" -13:14.786 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-799eeec7-98f2-204f-0a27-9a9de583ed9d.dc1 area=wan -13:14.786 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16204 network=tcp -13:14.787 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16205 network=tcp -13:14.787 [INFO] TestAgent: started state syncer -13:14.787 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16238 network=tcp -13:14.841 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:14.841 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16237 [Candidate]" term=2 -13:14.842 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:14.842 [DEBUG] TestAgent.server.raft: vote granted: from=799eeec7-98f2-204f-0a27-9a9de583ed9d term=2 tally=1 -13:14.842 [INFO] TestAgent.server.raft: election won: tally=1 -13:14.842 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16237 [Leader]" -13:14.842 [INFO] TestAgent.server: cluster leadership acquired -13:14.842 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16237 -13:14.843 [INFO] TestAgent.server: New leader elected: payload=Node-799eeec7-98f2-204f-0a27-9a9de583ed9d -13:14.844 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:14.846 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:14.846 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:14.846 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:14.846 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:14.846 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:14.846 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-799eeec7-98f2-204f-0a27-9a9de583ed9d -13:14.846 [INFO] TestAgent.server: member joined, marking health alive: member=Node-799eeec7-98f2-204f-0a27-9a9de583ed9d -13:14.870 [INFO] TestAgent.server: federation state anti-entropy synced -13:14.871 [DEBUG] TestAgent.dns: request served from client: question="{9.2.3.8.2.4.0.0.0.0.f.f.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa. 255 1}" latency=243.34µs client=127.0.0.1:48756 client_network=udp -13:14.871 [INFO] TestAgent: Requesting shutdown -13:14.871 [INFO] TestAgent.server: shutting down server -13:14.871 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.871 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.871 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.871 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.871 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.871 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.871 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.871 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:14.871 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.871 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.871 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:14.871 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:14.872Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.872Z [INFO] agent.router.manager: shutting down -13:14.872 [INFO] TestAgent: consul server down -13:14.872 [INFO] TestAgent: shutdown complete -13:14.872 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16204 network=tcp -13:14.872 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16204 network=udp -13:14.872 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16205 network=tcp -13:15.382 [INFO] TestAgent: Waiting for endpoints to shut down -13:15.383 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceReverseLookup_IPV6 (0.94s) -=== CONT TestDNSCycleRecursorCheck -2021-10-31T23:13:14.729Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:14.750Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:14.781 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2b1718ea-391e-a6e4-2127-b30da9d8a0c6 Address:127.0.0.1:16209}]" -13:14.781 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2b1718ea-391e-a6e4-2127-b30da9d8a0c6.dc1 127.0.0.1 -13:14.782 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2b1718ea-391e-a6e4-2127-b30da9d8a0c6 127.0.0.1 -2021-10-31T23:13:14.782Z [INFO] agent.router: Initializing LAN area manager -13:14.782 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16232 network=udp -13:14.782 [INFO] TestAgent.server: Adding LAN server: server="Node-2b1718ea-391e-a6e4-2127-b30da9d8a0c6 (Addr: tcp/127.0.0.1:16209) (DC: dc1)" -13:14.782 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2b1718ea-391e-a6e4-2127-b30da9d8a0c6.dc1 area=wan -13:14.782 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16209 [Follower]" leader= -13:14.782 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16232 network=tcp -13:14.783 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16233 network=tcp -13:14.783 [INFO] TestAgent: started state syncer -13:14.783 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16210 network=tcp -13:14.842 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:14.842 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16209 [Candidate]" term=2 -13:14.842 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:14.842 [DEBUG] TestAgent.server.raft: vote granted: from=2b1718ea-391e-a6e4-2127-b30da9d8a0c6 term=2 tally=1 -13:14.842 [INFO] TestAgent.server.raft: election won: tally=1 -13:14.842 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16209 [Leader]" -13:14.842 [INFO] TestAgent.server: cluster leadership acquired -13:14.842 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16209 -13:14.843 [INFO] TestAgent.server: New leader elected: payload=Node-2b1718ea-391e-a6e4-2127-b30da9d8a0c6 -13:14.844 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:14.850 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:14.850 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:14.850 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:14.850 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:14.850 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:14.850 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2b1718ea-391e-a6e4-2127-b30da9d8a0c6 -13:14.850 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2b1718ea-391e-a6e4-2127-b30da9d8a0c6 -13:14.852 [INFO] TestAgent.server: federation state anti-entropy synced -13:14.870 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:14.870 [INFO] TestAgent: Synced node info -13:14.870 [DEBUG] TestAgent: Node info in sync -13:14.879 [DEBUG] TestAgent.dns: request served from client: question="{1.0.0.127.in-addr.arpa. 255 1}" latency=97.29µs client=127.0.0.1:54000 client_network=udp -13:14.879 [INFO] TestAgent: Requesting shutdown -13:14.879 [INFO] TestAgent.server: shutting down server -13:14.879 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.879 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.879 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.879 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.879 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.879 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.879 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.879 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.879 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.879 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:14.880 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:14.880Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.880Z [INFO] agent.router.manager: shutting down -13:14.880 [INFO] TestAgent: consul server down -13:14.880 [INFO] TestAgent: shutdown complete -13:14.880 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16232 network=tcp -13:14.880 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16232 network=udp -13:14.880 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16233 network=tcp -13:15.386 [INFO] TestAgent: Waiting for endpoints to shut down -13:15.387 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceReverseLookupNodeAddress (0.69s) -=== CONT TestDNS_CaseInsensitiveNodeLookup === RUN TestDNS_EDNS0_ECS/global === RUN TestDNS_EDNS0_ECS/query === RUN TestDNS_EDNS0_ECS/query-subnet -2021-10-31T23:13:14.604Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:14.640Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:14.794 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:477dbf9a-d328-dec2-65bf-9b25b75b06a2 Address:127.0.0.1:16230}]" -13:14.795 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-477dbf9a-d328-dec2-65bf-9b25b75b06a2.dc1 127.0.0.1 -13:14.796 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-477dbf9a-d328-dec2-65bf-9b25b75b06a2 127.0.0.1 -2021-10-31T23:13:14.796Z [INFO] agent.router: Initializing LAN area manager -13:14.796 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16239 network=udp -13:14.796 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16230 [Follower]" leader= -13:14.796 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-477dbf9a-d328-dec2-65bf-9b25b75b06a2.dc1 area=wan -13:14.796 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16239 network=tcp -13:14.796 [INFO] TestAgent.server: Adding LAN server: server="Node-477dbf9a-d328-dec2-65bf-9b25b75b06a2 (Addr: tcp/127.0.0.1:16230) (DC: dc1)" -13:14.797 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16240 network=tcp -13:14.797 [INFO] TestAgent: started state syncer -13:14.797 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16231 network=tcp -13:14.847 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:14.847 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16230 [Candidate]" term=2 -13:14.847 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:14.847 [DEBUG] TestAgent.server.raft: vote granted: from=477dbf9a-d328-dec2-65bf-9b25b75b06a2 term=2 tally=1 -13:14.847 [INFO] TestAgent.server.raft: election won: tally=1 -13:14.847 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16230 [Leader]" -13:14.847 [INFO] TestAgent.server: cluster leadership acquired -13:14.847 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16230 -13:14.848 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:14.849 [INFO] TestAgent.server: New leader elected: payload=Node-477dbf9a-d328-dec2-65bf-9b25b75b06a2 -13:14.850 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:14.850 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:14.850 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:14.850 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:14.850 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:14.851 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-477dbf9a-d328-dec2-65bf-9b25b75b06a2 -13:14.851 [INFO] TestAgent.server: member joined, marking health alive: member=Node-477dbf9a-d328-dec2-65bf-9b25b75b06a2 -13:14.852 [INFO] TestAgent.server: federation state anti-entropy synced -13:14.990 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=188.1µs client=127.0.0.1:35254 client_network=udp -13:14.990 [INFO] TestAgent: Requesting shutdown -13:14.990 [INFO] TestAgent.server: shutting down server -13:14.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.990 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.990 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.990 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.990 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.990 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:14.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.990 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.990 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:14.990 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:14.990Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.990Z [INFO] agent.router.manager: shutting down -13:14.990 [INFO] TestAgent: consul server down -13:14.990 [INFO] TestAgent: shutdown complete -13:14.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16239 network=tcp -13:14.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16239 network=udp -13:14.991 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16240 network=tcp -13:15.497 [INFO] TestAgent: Waiting for endpoints to shut down -13:15.497 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ReverseLookup_CustomDomain (0.92s) +2022-12-04T05:49:26.334Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.345Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.346 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c852a7b4-f78c-f334-c671-bfd706fed3ef Address:127.0.0.1:23985}]" +49:26.346 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23985 [Follower]" leader= +49:26.347 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c852a7b4-f78c-f334-c671-bfd706fed3ef.dc1 127.0.0.1 +49:26.347 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c852a7b4-f78c-f334-c671-bfd706fed3ef 127.0.0.1 +2022-12-04T05:49:26.347Z [INFO] agent.router: Initializing LAN area manager +49:26.347 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23861 network=udp +49:26.347 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c852a7b4-f78c-f334-c671-bfd706fed3ef.dc1 area=wan +49:26.347 [INFO] TestAgent.server: Adding LAN server: server="Node-c852a7b4-f78c-f334-c671-bfd706fed3ef (Addr: tcp/127.0.0.1:23985) (DC: dc1)" +49:26.347 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23861 network=tcp +49:26.348 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23876 network=tcp +49:26.348 [INFO] TestAgent: started state syncer +49:26.348 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23986 network=tcp +49:26.409 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.409 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23985 [Candidate]" term=2 +49:26.410 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.410 [DEBUG] TestAgent.server.raft: vote granted: from=c852a7b4-f78c-f334-c671-bfd706fed3ef term=2 tally=1 +49:26.410 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.410 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23985 [Leader]" +49:26.410 [INFO] TestAgent.server: cluster leadership acquired +49:26.410 [INFO] TestAgent.server: New leader elected: payload=Node-c852a7b4-f78c-f334-c671-bfd706fed3ef +49:26.410 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23985 +49:26.411 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.413 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.413 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.413 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.413 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.413 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.414 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c852a7b4-f78c-f334-c671-bfd706fed3ef +49:26.414 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c852a7b4-f78c-f334-c671-bfd706fed3ef +49:26.414 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.494 [DEBUG] TestAgent.dns: request served from client: question="{2.4.2.4.2.4.2.4.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa. 255 1}" latency=140.051µs client=127.0.0.1:33943 client_network=udp +49:26.494 [INFO] TestAgent: Requesting shutdown +49:26.494 [INFO] TestAgent.server: shutting down server +49:26.494 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.494 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.494 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.494 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.494 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.494 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:26.494 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.494 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.494 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:26.494 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:26.494Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.494Z [INFO] agent.router.manager: shutting down +49:26.494 [INFO] TestAgent: consul server down +49:26.494 [INFO] TestAgent: shutdown complete +49:26.494 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23861 network=tcp +49:26.494 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23861 network=udp +49:26.494 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23876 network=tcp +49:26.994 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.994 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ReverseLookup_IPV6 (0.67s) +=== CONT TestDNSCycleRecursorCheck +2022-12-04T05:49:26.272Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.293Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.295 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:15641238-a849-0aca-4549-d09ea763828e Address:127.0.0.1:23943}]" +49:26.295 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23943 [Follower]" leader= +49:26.296 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-15641238-a849-0aca-4549-d09ea763828e.dc1 127.0.0.1 +49:26.297 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-15641238-a849-0aca-4549-d09ea763828e 127.0.0.1 +2022-12-04T05:49:26.297Z [INFO] agent.router: Initializing LAN area manager +49:26.297 [INFO] TestAgent.server: Adding LAN server: server="Node-15641238-a849-0aca-4549-d09ea763828e (Addr: tcp/127.0.0.1:23943) (DC: dc1)" +49:26.297 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-15641238-a849-0aca-4549-d09ea763828e.dc1 area=wan +49:26.297 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23994 network=udp +49:26.297 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23994 network=tcp +49:26.298 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23981 network=tcp +49:26.298 [INFO] TestAgent: started state syncer +49:26.298 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23944 network=tcp +49:26.331 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.331 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23943 [Candidate]" term=2 +49:26.332 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.332 [DEBUG] TestAgent.server.raft: vote granted: from=15641238-a849-0aca-4549-d09ea763828e term=2 tally=1 +49:26.332 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.332 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23943 [Leader]" +49:26.332 [INFO] TestAgent.server: cluster leadership acquired +49:26.332 [INFO] TestAgent.server: New leader elected: payload=Node-15641238-a849-0aca-4549-d09ea763828e +49:26.332 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23943 +49:26.332 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.333 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.333 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.333 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.333 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.333 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.334 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-15641238-a849-0aca-4549-d09ea763828e +49:26.334 [INFO] TestAgent.server: member joined, marking health alive: member=Node-15641238-a849-0aca-4549-d09ea763828e +49:26.334 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.411 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:26.412 [INFO] TestAgent: Synced node info +49:26.412 [DEBUG] TestAgent: Node info in sync +49:26.546 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=ANY class=IN latency=249.972µs client=127.0.0.1:36216 client_network=udp +49:26.546 [INFO] TestAgent: Requesting shutdown +49:26.546 [INFO] TestAgent.server: shutting down server +49:26.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.546 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.546 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.546 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.546 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.546 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.546 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:26.546 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:26.546Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.546Z [INFO] agent.router.manager: shutting down +49:26.546 [INFO] TestAgent: consul server down +49:26.546 [INFO] TestAgent: shutdown complete +49:26.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23994 network=tcp +49:26.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23994 network=udp +49:26.546 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23981 network=tcp +49:27.047 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.047 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookupNoMultiCNAME (0.79s) +=== CONT TestDNS_NodeLookup_PeriodName +2022-12-04T05:49:26.261Z [WARN] agent: Node name "test node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2022-12-04T05:49:26.261Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.275Z [WARN] agent.auto_config: Node name "test node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2022-12-04T05:49:26.275Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.276 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ad53e471-427b-e24f-c25a-d54536d3a467 Address:127.0.0.1:23950}]" +49:26.276 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23950 [Follower]" leader= +49:26.279 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test node.dc1 127.0.0.1 +49:26.282 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test node 127.0.0.1 +2022-12-04T05:49:26.282Z [INFO] agent.router: Initializing LAN area manager +49:26.283 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23358 network=udp +49:26.284 [INFO] TestAgent.server: Adding LAN server: server="test node (Addr: tcp/127.0.0.1:23950) (DC: dc1)" +49:26.284 [INFO] TestAgent.server: Handled event for server in area: event=member-join server="test node.dc1" area=wan +49:26.285 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23358 network=tcp +49:26.286 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23359 network=tcp +49:26.286 [INFO] TestAgent: started state syncer +49:26.286 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23951 network=tcp +49:26.317 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.317 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23950 [Candidate]" term=2 +49:26.317 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.317 [DEBUG] TestAgent.server.raft: vote granted: from=ad53e471-427b-e24f-c25a-d54536d3a467 term=2 tally=1 +49:26.317 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.317 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23950 [Leader]" +49:26.318 [INFO] TestAgent.server: cluster leadership acquired +49:26.318 [INFO] TestAgent.server: New leader elected: payload="test node" +49:26.318 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23950 +49:26.318 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.319 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.319 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.319 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.319 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.319 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.319 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node="test node" +49:26.319 [INFO] TestAgent.server: member joined, marking health alive: member="test node" +49:26.320 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.429 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:26.430 [INFO] TestAgent: Synced node info +49:26.547 [DEBUG] TestAgent.dns: request served from client: name=alias.service.consul. type=SRV class=IN latency=266.482µs client=127.0.0.1:33315 client_network=udp +49:26.547 [DEBUG] TestAgent.dns: request served from client: name=alias.service.CoNsUl. type=SRV class=IN latency=134.219µs client=127.0.0.1:50656 client_network=udp +49:26.547 [INFO] TestAgent: Requesting shutdown +49:26.547 [INFO] TestAgent.server: shutting down server +49:26.547 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.547 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.547 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.547 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.547 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.547 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.547 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.547 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.547 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.547 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:26.547 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:26.547Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.547Z [INFO] agent.router.manager: shutting down +49:26.547 [INFO] TestAgent: consul server down +49:26.547 [INFO] TestAgent: shutdown complete +49:26.547 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23358 network=tcp +49:26.547 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23358 network=udp +49:26.547 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23359 network=tcp +49:27.048 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.048 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ExternalServiceToConsulCNAMELookup (0.80s) === CONT TestDNS_EmptyAltDomain -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:14.470Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:14.493Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:14.797 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0a3245c8-cc53-6404-7f72-471c5b94e7cc Address:127.0.0.1:16202}]" -13:14.798 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16202 [Follower]" leader= -13:14.798 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0a3245c8-cc53-6404-7f72-471c5b94e7cc.dc1 127.0.0.1 -13:14.799 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0a3245c8-cc53-6404-7f72-471c5b94e7cc 127.0.0.1 -2021-10-31T23:13:14.799Z [INFO] agent.router: Initializing LAN area manager -13:14.799 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16148 network=udp -13:14.799 [INFO] TestAgent.server: Adding LAN server: server="Node-0a3245c8-cc53-6404-7f72-471c5b94e7cc (Addr: tcp/127.0.0.1:16202) (DC: dc1)" -13:14.799 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0a3245c8-cc53-6404-7f72-471c5b94e7cc.dc1 area=wan -13:14.799 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16148 network=tcp -13:14.800 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16149 network=tcp -13:14.800 [INFO] TestAgent: started state syncer -13:14.800 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16203 network=tcp -13:14.870 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:14.870 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16202 [Candidate]" term=2 -13:14.870 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:14.870 [DEBUG] TestAgent.server.raft: vote granted: from=0a3245c8-cc53-6404-7f72-471c5b94e7cc term=2 tally=1 -13:14.870 [INFO] TestAgent.server.raft: election won: tally=1 -13:14.870 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16202 [Leader]" -13:14.870 [INFO] TestAgent.server: cluster leadership acquired -13:14.870 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16202 -13:14.870 [INFO] TestAgent.server: New leader elected: payload=Node-0a3245c8-cc53-6404-7f72-471c5b94e7cc -13:14.880 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:14.889 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:14.889 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:14.889 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:14.889 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:14.889 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:14.889 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a3245c8-cc53-6404-7f72-471c5b94e7cc -13:14.889 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0a3245c8-cc53-6404-7f72-471c5b94e7cc -13:14.890 [INFO] TestAgent.server: federation state anti-entropy synced -13:15.118 [DEBUG] TestAgent.dns: request served from client: name=api.ingress.consul. type=A class=IN latency=318.431µs client=127.0.0.1:45596 client_network=udp -13:15.119 [DEBUG] TestAgent.dns: request served from client: name=api.ingress.dc1.consul. type=A class=IN latency=122.316µs client=127.0.0.1:56929 client_network=udp -13:15.119 [DEBUG] TestAgent.dns: request served from client: name=db.ingress.consul. type=A class=IN latency=100.881µs client=127.0.0.1:57374 client_network=udp -13:15.119 [DEBUG] TestAgent.dns: request served from client: name=db.ingress.dc1.consul. type=A class=IN latency=98.197µs client=127.0.0.1:46087 client_network=udp -13:15.119 [INFO] TestAgent: Requesting shutdown -13:15.119 [INFO] TestAgent.server: shutting down server -13:15.119 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:15.119 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:15.119 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:15.119 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:15.119 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:15.119 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:15.119 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:15.120 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:15.120 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:15.120 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:15.120 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:15.120 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:15.120Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:15.120Z [INFO] agent.router.manager: shutting down -13:15.120 [INFO] TestAgent: consul server down -13:15.120 [INFO] TestAgent: shutdown complete -13:15.120 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16148 network=tcp -13:15.120 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16148 network=udp -13:15.120 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16149 network=tcp -13:15.621 [INFO] TestAgent: Waiting for endpoints to shut down -13:15.621 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_IngressServiceLookup (1.18s) - --- PASS: TestDNS_IngressServiceLookup/api.ingress.consul. (0.00s) - --- PASS: TestDNS_IngressServiceLookup/api.ingress.dc1.consul. (0.00s) - --- PASS: TestDNS_IngressServiceLookup/db.ingress.consul. (0.00s) - --- PASS: TestDNS_IngressServiceLookup/db.ingress.dc1.consul. (0.00s) -=== CONT TestParseConsistency -2021-10-31T23:13:14.910Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:14.968Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:14.974 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a0964bea-74b9-51f8-d18e-74f5f1a29059 Address:127.0.0.1:16167}]" -13:14.975 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a0964bea-74b9-51f8-d18e-74f5f1a29059.dc1 127.0.0.1 -13:14.975 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a0964bea-74b9-51f8-d18e-74f5f1a29059 127.0.0.1 -2021-10-31T23:13:14.975Z [INFO] agent.router: Initializing LAN area manager -13:14.976 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16176 network=udp -13:14.976 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16167 [Follower]" leader= -13:14.976 [INFO] TestAgent.server: Adding LAN server: server="Node-a0964bea-74b9-51f8-d18e-74f5f1a29059 (Addr: tcp/127.0.0.1:16167) (DC: dc1)" -13:14.976 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a0964bea-74b9-51f8-d18e-74f5f1a29059.dc1 area=wan -13:14.976 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16176 network=tcp -13:14.977 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16177 network=tcp -13:14.977 [INFO] TestAgent: started state syncer -13:14.977 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16168 network=tcp -13:15.040 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:15.040 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16167 [Candidate]" term=2 -13:15.040 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:15.040 [DEBUG] TestAgent.server.raft: vote granted: from=a0964bea-74b9-51f8-d18e-74f5f1a29059 term=2 tally=1 -13:15.041 [INFO] TestAgent.server.raft: election won: tally=1 -13:15.041 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16167 [Leader]" -13:15.041 [INFO] TestAgent.server: cluster leadership acquired -13:15.041 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16167 -13:15.042 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:15.043 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:15.043 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:15.043 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:15.043 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:15.043 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:15.043 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a0964bea-74b9-51f8-d18e-74f5f1a29059 -13:15.043 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a0964bea-74b9-51f8-d18e-74f5f1a29059 -13:15.044 [INFO] TestAgent.server: New leader elected: payload=Node-a0964bea-74b9-51f8-d18e-74f5f1a29059 -13:15.044 [INFO] TestAgent.server: federation state anti-entropy synced -13:15.190 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=149.535µs client=127.0.0.1:51692 client_network=udp -13:15.193 [INFO] TestAgent: Requesting shutdown -13:15.193 [INFO] TestAgent.server: shutting down server -13:15.193 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:15.193 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:15.193 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:15.193 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:15.193 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:15.193 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:15.193 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:15.193 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:15.193 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:15.193 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:15.193 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:15.193 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:15.193Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:15.193Z [INFO] agent.router.manager: shutting down -13:15.193 [INFO] TestAgent: consul server down -13:15.193 [INFO] TestAgent: shutdown complete -13:15.193 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16176 network=tcp -13:15.193 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16176 network=udp -13:15.193 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16177 network=tcp -13:15.693 [INFO] TestAgent: Waiting for endpoints to shut down -13:15.693 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ReverseLookup (0.85s) -=== CONT TestDNS_NodeLookup -2021-10-31T23:13:14.910Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:14.950Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:14.952 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e5941b98-0467-6c8b-1bfe-0a4498a0db46 Address:127.0.0.1:16195}]" -13:14.952 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e5941b98-0467-6c8b-1bfe-0a4498a0db46.dc1 127.0.0.1 -13:14.953 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e5941b98-0467-6c8b-1bfe-0a4498a0db46 127.0.0.1 -2021-10-31T23:13:14.953Z [INFO] agent.router: Initializing LAN area manager -13:14.953 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16183 network=udp -13:14.953 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16195 [Follower]" leader= -13:14.954 [INFO] TestAgent.server: Adding LAN server: server="Node-e5941b98-0467-6c8b-1bfe-0a4498a0db46 (Addr: tcp/127.0.0.1:16195) (DC: dc1)" -13:14.954 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e5941b98-0467-6c8b-1bfe-0a4498a0db46.dc1 area=wan -13:14.954 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16183 network=tcp -13:14.955 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16184 network=tcp -13:14.955 [INFO] TestAgent: started state syncer -13:14.955 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16196 network=tcp -13:15.021 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:15.021 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16195 [Candidate]" term=2 -13:15.022 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:15.022 [DEBUG] TestAgent.server.raft: vote granted: from=e5941b98-0467-6c8b-1bfe-0a4498a0db46 term=2 tally=1 -13:15.022 [INFO] TestAgent.server.raft: election won: tally=1 -13:15.022 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16195 [Leader]" -13:15.022 [INFO] TestAgent.server: cluster leadership acquired -13:15.022 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16195 -13:15.022 [INFO] TestAgent.server: New leader elected: payload=Node-e5941b98-0467-6c8b-1bfe-0a4498a0db46 -13:15.023 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:15.024 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:15.024 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:15.024 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:15.024 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:15.024 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:15.024 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e5941b98-0467-6c8b-1bfe-0a4498a0db46 -13:15.024 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e5941b98-0467-6c8b-1bfe-0a4498a0db46 -13:15.025 [INFO] TestAgent.server: federation state anti-entropy synced -13:15.290 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:15.290 [INFO] TestAgent: Synced node info -13:15.307 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=116.492µs client=127.0.0.1:50474 client_network=udp -13:15.307 [INFO] TestAgent: Requesting shutdown -13:15.307 [INFO] TestAgent.server: shutting down server -13:15.307 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:15.307 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:15.307 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:15.307 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:15.307 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:15.307 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:15.307 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:15.307 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:15.307 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:15.307 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:15.307 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:15.307Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:15.307Z [INFO] agent.router.manager: shutting down -13:15.307 [INFO] TestAgent: consul server down -13:15.307 [INFO] TestAgent: shutdown complete -13:15.307 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16183 network=tcp -13:15.307 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16183 network=udp -13:15.307 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16184 network=tcp -13:15.808 [INFO] TestAgent: Waiting for endpoints to shut down -13:15.808 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceReverseLookup (0.94s) -=== CONT TestParseWait_InvalidIndex ---- PASS: TestParseWait_InvalidIndex (0.00s) +[WARN] freeport: 10 out of 59 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:25.929Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:25.929Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.939Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:25.939Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.941 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d6504248-5d2e-ee55-bb51-e8fa2ec1085d Address:127.0.0.1:23915}]" +49:25.941 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23915 [Follower]" leader= +49:25.941 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d6504248-5d2e-ee55-bb51-e8fa2ec1085d.dc2 127.0.0.1 +49:25.942 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d6504248-5d2e-ee55-bb51-e8fa2ec1085d 127.0.0.1 +2022-12-04T05:49:25.942Z [INFO] agent.router: Initializing LAN area manager +49:25.942 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d6504248-5d2e-ee55-bb51-e8fa2ec1085d.dc2 area=wan +49:25.942 [INFO] TestAgent.server: Adding LAN server: server="Node-d6504248-5d2e-ee55-bb51-e8fa2ec1085d (Addr: tcp/127.0.0.1:23915) (DC: dc2)" +49:25.942 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23910 network=udp +49:25.942 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23910 network=tcp +49:25.943 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23925 network=tcp +49:25.943 [INFO] TestAgent: started state syncer +49:25.943 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23916 network=tcp +49:25.987 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.987 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23915 [Candidate]" term=2 +49:25.987 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.987 [DEBUG] TestAgent.server.raft: vote granted: from=d6504248-5d2e-ee55-bb51-e8fa2ec1085d term=2 tally=1 +49:25.987 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.987 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23915 [Leader]" +49:25.987 [INFO] TestAgent.server: cluster leadership acquired +49:25.987 [INFO] TestAgent.server: initializing acls +49:25.987 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:25.988 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:25.988 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:25.988 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:25.988 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d6504248-5d2e-ee55-bb51-e8fa2ec1085d +49:25.988 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d6504248-5d2e-ee55-bb51-e8fa2ec1085d.dc2 +49:25.988 [INFO] TestAgent.server: New leader elected: payload=Node-d6504248-5d2e-ee55-bb51-e8fa2ec1085d +49:25.988 [INFO] TestAgent.server: Updating LAN server: server="Node-d6504248-5d2e-ee55-bb51-e8fa2ec1085d (Addr: tcp/127.0.0.1:23915) (DC: dc2)" +49:25.988 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d6504248-5d2e-ee55-bb51-e8fa2ec1085d.dc2 area=wan +49:25.988 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.989 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.989 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.989 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.989 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.989 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.989 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d6504248-5d2e-ee55-bb51-e8fa2ec1085d +49:25.989 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d6504248-5d2e-ee55-bb51-e8fa2ec1085d +49:25.990 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d6504248-5d2e-ee55-bb51-e8fa2ec1085d +49:25.990 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.117 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:23865] +49:26.117 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:23865 +49:26.118 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-754a4f8c-c63b-1f90-145b-cc7e64f738fa.dc1 127.0.0.1 +49:26.118 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +49:26.118 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-754a4f8c-c63b-1f90-145b-cc7e64f738fa.dc1 area=wan +49:26.138 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=69.239µs client=127.0.0.1:60892 client_network=udp +49:26.138 [DEBUG] TestAgent.dns: request served from client: name=e5177ca6-28dd-ae46-1c66-2a546bb4af5a.query.dc2.consul. type=SRV class=IN latency=67.522µs client=127.0.0.1:58819 client_network=udp +49:26.139 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=64.056µs client=127.0.0.1:46012 client_network=udp +49:26.139 [DEBUG] TestAgent.dns: request served from client: name=e5177ca6-28dd-ae46-1c66-2a546bb4af5a.query.dc2.consul. type=A class=IN latency=60.649µs client=127.0.0.1:52849 client_network=udp +49:26.139 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=70.488µs client=127.0.0.1:54678 client_network=udp +49:26.139 [DEBUG] TestAgent.dns: request served from client: name=e5177ca6-28dd-ae46-1c66-2a546bb4af5a.query.dc2.consul. type=SRV class=IN latency=76.772µs client=127.0.0.1:52219 client_network=udp +49:26.140 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=200.436µs client=127.0.0.1:53853 client_network=udp +49:26.141 [DEBUG] TestAgent.dns: request served from client: name=e5177ca6-28dd-ae46-1c66-2a546bb4af5a.query.dc2.consul. type=A class=IN latency=93.782µs client=127.0.0.1:45871 client_network=udp +49:26.148 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=344.995µs client=127.0.0.1:45854 client_network=udp +49:26.149 [DEBUG] TestAgent.dns: request served from client: name=e5177ca6-28dd-ae46-1c66-2a546bb4af5a.query.dc2.consul. type=SRV class=IN latency=235.358µs client=127.0.0.1:40402 client_network=udp +49:26.149 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=138.954µs client=127.0.0.1:38203 client_network=udp +49:26.150 [DEBUG] TestAgent.dns: request served from client: name=e5177ca6-28dd-ae46-1c66-2a546bb4af5a.query.dc2.consul. type=A class=IN latency=112.041µs client=127.0.0.1:41737 client_network=udp +49:26.151 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=256.025µs client=127.0.0.1:41549 client_network=udp +49:26.152 [DEBUG] TestAgent.dns: request served from client: name=e5177ca6-28dd-ae46-1c66-2a546bb4af5a.query.dc2.consul. type=SRV class=IN latency=350.999µs client=127.0.0.1:35629 client_network=udp +49:26.153 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=159.19µs client=127.0.0.1:40219 client_network=udp +49:26.153 [DEBUG] TestAgent.dns: request served from client: name=e5177ca6-28dd-ae46-1c66-2a546bb4af5a.query.dc2.consul. type=A class=IN latency=132.593µs client=127.0.0.1:60231 client_network=udp +49:26.155 [INFO] TestAgent: Requesting shutdown +49:26.155 [INFO] TestAgent.server: shutting down server +49:26.155 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.155 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.155 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.155 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.155 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:26.155 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:26.155 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.155 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.155 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:26.155 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:26.155 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.155 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.155 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:26.155Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.155Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.155Z [INFO] agent.router.manager: shutting down +49:26.156 [INFO] TestAgent: consul server down +49:26.156 [INFO] TestAgent: shutdown complete +49:26.156 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23910 network=tcp +49:26.157 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:26.155 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.155 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:26.157 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23910 network=udp +49:26.157 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23925 network=tcp +49:26.660 [INFO] TestAgent: Waiting for endpoints to shut down +49:26.660 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:25.517Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:25.517Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.529Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:25.529Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.531 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:754a4f8c-c63b-1f90-145b-cc7e64f738fa Address:127.0.0.1:23866}]" +49:25.531 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23866 [Follower]" leader= +49:25.531 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-754a4f8c-c63b-1f90-145b-cc7e64f738fa.dc1 127.0.0.1 +49:25.532 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-754a4f8c-c63b-1f90-145b-cc7e64f738fa 127.0.0.1 +2022-12-04T05:49:25.532Z [INFO] agent.router: Initializing LAN area manager +49:25.532 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-754a4f8c-c63b-1f90-145b-cc7e64f738fa.dc1 area=wan +49:25.532 [INFO] TestAgent.server: Adding LAN server: server="Node-754a4f8c-c63b-1f90-145b-cc7e64f738fa (Addr: tcp/127.0.0.1:23866) (DC: dc1)" +49:25.532 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23875 network=udp +49:25.532 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23875 network=tcp +49:25.533 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23855 network=tcp +49:25.533 [INFO] TestAgent: started state syncer +49:25.533 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23867 network=tcp +49:25.582 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:25.591 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.591 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23866 [Candidate]" term=2 +49:25.591 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.591 [DEBUG] TestAgent.server.raft: vote granted: from=754a4f8c-c63b-1f90-145b-cc7e64f738fa term=2 tally=1 +49:25.591 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.591 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23866 [Leader]" +49:25.591 [INFO] TestAgent.server: cluster leadership acquired +49:25.591 [INFO] TestAgent.server: initializing acls +49:25.591 [INFO] TestAgent.server: New leader elected: payload=Node-754a4f8c-c63b-1f90-145b-cc7e64f738fa +49:25.591 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:25.592 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:25.592 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:25.592 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:25.592 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-754a4f8c-c63b-1f90-145b-cc7e64f738fa +49:25.592 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-754a4f8c-c63b-1f90-145b-cc7e64f738fa.dc1 +49:25.592 [INFO] TestAgent.server: Updating LAN server: server="Node-754a4f8c-c63b-1f90-145b-cc7e64f738fa (Addr: tcp/127.0.0.1:23866) (DC: dc1)" +49:25.592 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-754a4f8c-c63b-1f90-145b-cc7e64f738fa.dc1 area=wan +49:25.592 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.594 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.594 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.594 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.594 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.594 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.594 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-754a4f8c-c63b-1f90-145b-cc7e64f738fa +49:25.594 [INFO] TestAgent.server: member joined, marking health alive: member=Node-754a4f8c-c63b-1f90-145b-cc7e64f738fa +49:25.594 [INFO] TestAgent.server: federation state anti-entropy synced +49:25.594 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-754a4f8c-c63b-1f90-145b-cc7e64f738fa +49:25.931 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:25.932 [INFO] TestAgent: Synced node info +49:25.932 [DEBUG] TestAgent: Node info in sync +49:26.100 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:26.100 [DEBUG] TestAgent: Node info in sync +49:26.117 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:46296 +49:26.118 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d6504248-5d2e-ee55-bb51-e8fa2ec1085d.dc2 127.0.0.1 +49:26.118 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d6504248-5d2e-ee55-bb51-e8fa2ec1085d.dc2 area=wan +49:26.120 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=923.123µs client=127.0.0.1:47835 client_network=udp +49:26.121 [DEBUG] TestAgent.dns: request served from client: name=e5177ca6-28dd-ae46-1c66-2a546bb4af5a.query.dc2.consul. type=SRV class=IN latency=591.236µs client=127.0.0.1:36317 client_network=udp +49:26.121 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=388.635µs client=127.0.0.1:60287 client_network=udp +49:26.122 [DEBUG] TestAgent.dns: request served from client: name=e5177ca6-28dd-ae46-1c66-2a546bb4af5a.query.dc2.consul. type=A class=IN latency=474.237µs client=127.0.0.1:45859 client_network=udp +49:26.130 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=1.180008ms client=127.0.0.1:49466 client_network=udp +49:26.130 [DEBUG] TestAgent.dns: request served from client: name=e5177ca6-28dd-ae46-1c66-2a546bb4af5a.query.dc2.consul. type=SRV class=IN latency=448.635µs client=127.0.0.1:50932 client_network=udp +49:26.131 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=338.114µs client=127.0.0.1:40192 client_network=udp +49:26.131 [DEBUG] TestAgent.dns: request served from client: name=e5177ca6-28dd-ae46-1c66-2a546bb4af5a.query.dc2.consul. type=A class=IN latency=333.597µs client=127.0.0.1:34486 client_network=udp +49:26.133 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=466.79µs client=127.0.0.1:49877 client_network=udp +49:26.133 [DEBUG] TestAgent.dns: request served from client: name=e5177ca6-28dd-ae46-1c66-2a546bb4af5a.query.dc2.consul. type=SRV class=IN latency=427.748µs client=127.0.0.1:46403 client_network=udp +49:26.134 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=346.658µs client=127.0.0.1:48999 client_network=udp +49:26.134 [DEBUG] TestAgent.dns: request served from client: name=e5177ca6-28dd-ae46-1c66-2a546bb4af5a.query.dc2.consul. type=A class=IN latency=359.603µs client=127.0.0.1:55004 client_network=udp +49:26.136 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=SRV class=IN latency=282.359µs client=127.0.0.1:44421 client_network=udp +49:26.137 [DEBUG] TestAgent.dns: request served from client: name=e5177ca6-28dd-ae46-1c66-2a546bb4af5a.query.dc2.consul. type=SRV class=IN latency=261.581µs client=127.0.0.1:33130 client_network=udp +49:26.137 [DEBUG] TestAgent.dns: request served from client: name=db.service.dc2.consul. type=A class=IN latency=250.249µs client=127.0.0.1:47839 client_network=udp +49:26.138 [DEBUG] TestAgent.dns: request served from client: name=e5177ca6-28dd-ae46-1c66-2a546bb4af5a.query.dc2.consul. type=A class=IN latency=259.897µs client=127.0.0.1:34845 client_network=udp +49:26.660 [INFO] TestAgent: Requesting shutdown +49:26.660 [INFO] TestAgent.server: shutting down server +49:26.660 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.660 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.660 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.660 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:26.660 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:26.660 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.660 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.660 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.660 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:26.660 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.660 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.660 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.660 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:26.660 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:26.660 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:26.661Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.661Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.661Z [INFO] agent.router.manager: shutting down +49:26.661 [INFO] TestAgent: consul server down +49:26.661 [INFO] TestAgent: shutdown complete +49:26.661 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23875 network=tcp +49:26.661 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23875 network=udp +49:26.661 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23855 network=tcp +49:27.161 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.161 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_WanTranslation (1.66s) + --- PASS: TestDNS_ServiceLookup_WanTranslation/node-addr-from-dc1 (0.01s) + --- PASS: TestDNS_ServiceLookup_WanTranslation/node-wan-from-dc1 (0.00s) + --- PASS: TestDNS_ServiceLookup_WanTranslation/service-addr-from-dc1 (0.00s) + --- PASS: TestDNS_ServiceLookup_WanTranslation/service-wan-from-dc1 (0.00s) + --- PASS: TestDNS_ServiceLookup_WanTranslation/node-addr-from-dc2 (0.00s) + --- PASS: TestDNS_ServiceLookup_WanTranslation/node-wan-from-dc2 (0.00s) + --- PASS: TestDNS_ServiceLookup_WanTranslation/service-addr-from-dc2 (0.01s) + --- PASS: TestDNS_ServiceLookup_WanTranslation/service-wan-from-dc2 (0.00s) === CONT TestDNS_Over_TCP -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:15.293Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:15.321Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:15.327 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9a12c46b-40dd-df4d-a61d-d33ee199bf62 Address:127.0.0.1:16279}]" -13:15.327 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16279 [Follower]" leader= -13:15.331 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9a12c46b-40dd-df4d-a61d-d33ee199bf62.dc1 127.0.0.1 -13:15.332 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9a12c46b-40dd-df4d-a61d-d33ee199bf62 127.0.0.1 -2021-10-31T23:13:15.332Z [INFO] agent.router: Initializing LAN area manager -13:15.333 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16190 network=udp -13:15.333 [INFO] TestAgent.server: Adding LAN server: server="Node-9a12c46b-40dd-df4d-a61d-d33ee199bf62 (Addr: tcp/127.0.0.1:16279) (DC: dc1)" -13:15.333 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9a12c46b-40dd-df4d-a61d-d33ee199bf62.dc1 area=wan -13:15.333 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16190 network=tcp -13:15.333 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16191 network=tcp -13:15.333 [INFO] TestAgent: started state syncer -13:15.334 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16280 network=tcp -13:15.370 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:15.370 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16279 [Candidate]" term=2 -13:15.370 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:15.370 [DEBUG] TestAgent.server.raft: vote granted: from=9a12c46b-40dd-df4d-a61d-d33ee199bf62 term=2 tally=1 -13:15.370 [INFO] TestAgent.server.raft: election won: tally=1 -13:15.370 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16279 [Leader]" -13:15.370 [INFO] TestAgent.server: cluster leadership acquired -13:15.370 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16279 -13:15.371 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:15.373 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:15.373 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:15.373 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:15.373 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:15.373 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:15.373 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9a12c46b-40dd-df4d-a61d-d33ee199bf62 -13:15.373 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9a12c46b-40dd-df4d-a61d-d33ee199bf62 -13:15.379 [INFO] TestAgent.server: New leader elected: payload=Node-9a12c46b-40dd-df4d-a61d-d33ee199bf62 -13:15.382 [INFO] TestAgent.server: federation state anti-entropy synced -13:15.475 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=106.558µs client=127.0.0.1:49629 client_network=udp -13:15.475 [DEBUG] TestAgent.dns: request served from client: name=test.query.consul. type=A class=IN latency=85.554µs client=127.0.0.1:49764 client_network=udp -13:15.476 [DEBUG] TestAgent.dns: request served from client: name=test.query.consul. type=A class=IN latency=70.194µs client=127.0.0.1:43996 client_network=udp -13:15.476 [INFO] TestAgent: Requesting shutdown -13:15.476 [INFO] TestAgent.server: shutting down server -13:15.476 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:15.476 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:15.476 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:15.476 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:15.476 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:15.476 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:15.476 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:15.476 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:15.476 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:15.476 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:15.476 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:15.476 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:15.476Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:15.476Z [INFO] agent.router.manager: shutting down -13:15.476 [INFO] TestAgent: consul server down -13:15.476 [INFO] TestAgent: shutdown complete -13:15.476 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16190 network=tcp -13:15.476 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16190 network=udp -13:15.476 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16191 network=tcp -13:15.977 [INFO] TestAgent: Waiting for endpoints to shut down -13:15.977 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_EDNS0_ECS (0.72s) - --- PASS: TestDNS_EDNS0_ECS/global (0.00s) - --- PASS: TestDNS_EDNS0_ECS/query (0.00s) - --- PASS: TestDNS_EDNS0_ECS/query-subnet (0.00s) -=== CONT TestParseWait_InvalidTime ---- PASS: TestParseWait_InvalidTime (0.00s) +2022-12-04T05:49:26.368Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.379Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.380 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:999dad12-88e4-d143-c4a2-a82b2aa64b90 Address:127.0.0.1:24006}]" +49:26.380 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24006 [Follower]" leader= +49:26.381 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-999dad12-88e4-d143-c4a2-a82b2aa64b90.dc1 127.0.0.1 +49:26.381 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-999dad12-88e4-d143-c4a2-a82b2aa64b90 127.0.0.1 +2022-12-04T05:49:26.381Z [INFO] agent.router: Initializing LAN area manager +49:26.381 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23987 network=udp +49:26.382 [INFO] TestAgent.server: Adding LAN server: server="Node-999dad12-88e4-d143-c4a2-a82b2aa64b90 (Addr: tcp/127.0.0.1:24006) (DC: dc1)" +49:26.382 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-999dad12-88e4-d143-c4a2-a82b2aa64b90.dc1 area=wan +49:26.382 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23987 network=tcp +49:26.382 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24002 network=tcp +49:26.382 [INFO] TestAgent: started state syncer +49:26.382 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24007 network=tcp +49:26.445 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.445 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24006 [Candidate]" term=2 +49:26.445 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.445 [DEBUG] TestAgent.server.raft: vote granted: from=999dad12-88e4-d143-c4a2-a82b2aa64b90 term=2 tally=1 +49:26.445 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.445 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24006 [Leader]" +49:26.445 [INFO] TestAgent.server: cluster leadership acquired +49:26.445 [INFO] TestAgent.server: New leader elected: payload=Node-999dad12-88e4-d143-c4a2-a82b2aa64b90 +49:26.445 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24006 +49:26.446 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.447 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.447 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.447 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.447 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.447 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.447 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-999dad12-88e4-d143-c4a2-a82b2aa64b90 +49:26.447 [INFO] TestAgent.server: member joined, marking health alive: member=Node-999dad12-88e4-d143-c4a2-a82b2aa64b90 +49:26.448 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.676 [DEBUG] TestAgent.dns: request served from client: question="{1.0.0.127.in-addr.arpa. 255 1}" latency=131.336µs client=127.0.0.1:47839 client_network=udp +49:26.676 [INFO] TestAgent: Requesting shutdown +49:26.676 [INFO] TestAgent.server: shutting down server +49:26.676 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.676 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.676 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.676 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.676 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.676 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:26.676 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.676 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:26.676 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.676 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.676 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.676 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:26.676Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.676Z [INFO] agent.router.manager: shutting down +49:26.676 [INFO] TestAgent: consul server down +49:26.677 [INFO] TestAgent: shutdown complete +49:26.677 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23987 network=tcp +49:26.677 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23987 network=udp +49:26.677 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24002 network=tcp +49:27.177 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.177 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceReverseLookupNodeAddress (0.82s) +=== CONT TestDNS_CaseInsensitiveNodeLookup +2022-12-04T05:49:26.327Z [WARN] agent: Node name "test node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2022-12-04T05:49:26.327Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.337Z [WARN] agent.auto_config: Node name "test node" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2022-12-04T05:49:26.337Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.339 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:57f8d94f-37e5-db7d-5091-70e3f7c31b09 Address:127.0.0.1:23859}]" +49:26.339 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23859 [Follower]" leader= +49:26.339 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test node.dc1 127.0.0.1 +49:26.340 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test node 127.0.0.1 +2022-12-04T05:49:26.340Z [INFO] agent.router: Initializing LAN area manager +49:26.340 [INFO] TestAgent.server: Adding LAN server: server="test node (Addr: tcp/127.0.0.1:23859) (DC: dc1)" +49:26.340 [INFO] TestAgent.server: Handled event for server in area: event=member-join server="test node.dc1" area=wan +49:26.340 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23973 network=tcp +49:26.340 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23973 network=udp +49:26.340 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23974 network=tcp +49:26.341 [INFO] TestAgent: started state syncer +49:26.341 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23860 network=tcp +49:26.408 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.408 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23859 [Candidate]" term=2 +49:26.408 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.408 [DEBUG] TestAgent.server.raft: vote granted: from=57f8d94f-37e5-db7d-5091-70e3f7c31b09 term=2 tally=1 +49:26.408 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.408 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23859 [Leader]" +49:26.408 [INFO] TestAgent.server: cluster leadership acquired +49:26.409 [INFO] TestAgent.server: New leader elected: payload="test node" +49:26.409 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23859 +49:26.410 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.412 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.412 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.412 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.412 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.412 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.412 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node="test node" +49:26.412 [INFO] TestAgent.server: member joined, marking health alive: member="test node" +49:26.414 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.760 [ERROR] TestAgent.dns: Infinite recursion detected for name, won't perform any CNAME resolution.: name=web.service.consul. +49:26.760 [DEBUG] TestAgent.dns: request served from client: name=web.service.consul. type=A class=IN latency=381.591µs client=127.0.0.1:59466 client_network=udp +49:26.760 [INFO] TestAgent: Requesting shutdown +49:26.760 [INFO] TestAgent.server: shutting down server +49:26.760 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.760 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.760 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.760 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.760 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.761 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.761 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.761 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:26.761 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.761 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:26.761 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.761 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:26.761Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.761Z [INFO] agent.router.manager: shutting down +49:26.761 [INFO] TestAgent: consul server down +49:26.761 [INFO] TestAgent: shutdown complete +49:26.761 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23973 network=tcp +49:26.761 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23973 network=udp +49:26.761 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23974 network=tcp +49:27.261 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.261 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_InifiniteRecursion (0.95s) === CONT TestDenylist === RUN TestDenylist/nothing_blocked_root === RUN TestDenylist/nothing_blocked_path @@ -31759,310 +37580,486 @@ --- PASS: TestDenylist/no_match (0.00s) === CONT TestRecursorAddr --- PASS: TestRecursorAddr (0.00s) -=== CONT TestPProfHandlers_ACLs -2021-10-31T23:13:15.248Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:15.274Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:15.275 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:89770413-9448-51ac-776b-b04964152b8d Address:127.0.0.1:16181}]" -13:15.275 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16181 [Follower]" leader= -13:15.276 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-89770413-9448-51ac-776b-b04964152b8d.dc1 127.0.0.1 -13:15.276 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-89770413-9448-51ac-776b-b04964152b8d 127.0.0.1 -2021-10-31T23:13:15.276Z [INFO] agent.router: Initializing LAN area manager -13:15.277 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16197 network=udp -13:15.277 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-89770413-9448-51ac-776b-b04964152b8d.dc1 area=wan -13:15.277 [INFO] TestAgent.server: Adding LAN server: server="Node-89770413-9448-51ac-776b-b04964152b8d (Addr: tcp/127.0.0.1:16181) (DC: dc1)" -13:15.277 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16197 network=tcp -13:15.277 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16198 network=tcp -13:15.278 [INFO] TestAgent: started state syncer -13:15.278 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16182 network=tcp -13:15.321 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:15.321 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16181 [Candidate]" term=2 -13:15.322 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:15.322 [DEBUG] TestAgent.server.raft: vote granted: from=89770413-9448-51ac-776b-b04964152b8d term=2 tally=1 -13:15.322 [INFO] TestAgent.server.raft: election won: tally=1 -13:15.322 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16181 [Leader]" -13:15.322 [INFO] TestAgent.server: cluster leadership acquired -13:15.322 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16181 -13:15.323 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:15.323 [INFO] TestAgent.server: New leader elected: payload=Node-89770413-9448-51ac-776b-b04964152b8d -13:15.324 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:15.324 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:15.324 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:15.324 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:15.324 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:15.324 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-89770413-9448-51ac-776b-b04964152b8d -13:15.324 [INFO] TestAgent.server: member joined, marking health alive: member=Node-89770413-9448-51ac-776b-b04964152b8d -13:15.325 [INFO] TestAgent.server: federation state anti-entropy synced -13:15.490 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=221.517µs client=127.0.0.1:46514 client_network=udp -13:15.490 [INFO] TestAgent: Requesting shutdown -13:15.490 [INFO] TestAgent.server: shutting down server -13:15.490 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:15.490 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:15.490 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:15.490 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:15.490 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:15.490 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:15.490 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:15.490 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:15.490 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:15.490 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:15.490 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:15.490 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:15.490Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:15.490Z [INFO] agent.router.manager: shutting down -13:15.490 [INFO] TestAgent: consul server down -13:15.490 [INFO] TestAgent: shutdown complete -13:15.490 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16197 network=tcp -13:15.490 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16197 network=udp -13:15.490 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16198 network=tcp -13:15.992 [INFO] TestAgent: Waiting for endpoints to shut down -13:15.992 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceReverseLookup_CustomDomain (0.76s) -=== CONT TestDiscoveryChainRead -2021-10-31T23:13:12.237Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:12.367Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:12.369 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:61e6a96e-6e7f-1f30-77be-99f2976b52c2 Address:127.0.0.1:16998}]" -13:12.369 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-61e6a96e-6e7f-1f30-77be-99f2976b52c2.dc1 127.0.0.1 -13:12.369 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16998 [Follower]" leader= -13:12.378 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-61e6a96e-6e7f-1f30-77be-99f2976b52c2 127.0.0.1 -2021-10-31T23:13:12.378Z [INFO] agent.router: Initializing LAN area manager -13:12.378 [DEBUG] TestAgent.dns: recursor enabled -13:12.379 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16993 network=udp -13:12.379 [INFO] TestAgent.server: Adding LAN server: server="Node-61e6a96e-6e7f-1f30-77be-99f2976b52c2 (Addr: tcp/127.0.0.1:16998) (DC: dc1)" -13:12.379 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-61e6a96e-6e7f-1f30-77be-99f2976b52c2.dc1 area=wan -13:12.379 [DEBUG] TestAgent.dns: recursor enabled -13:12.379 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16993 network=tcp -13:12.379 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16994 network=tcp -13:12.379 [INFO] TestAgent: started state syncer -13:12.380 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16999 network=tcp -13:12.409 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:12.409 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16998 [Candidate]" term=2 -13:12.410 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:12.410 [DEBUG] TestAgent.server.raft: vote granted: from=61e6a96e-6e7f-1f30-77be-99f2976b52c2 term=2 tally=1 -13:12.410 [INFO] TestAgent.server.raft: election won: tally=1 -13:12.410 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16998 [Leader]" -13:12.410 [INFO] TestAgent.server: cluster leadership acquired -13:12.410 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16998 -13:12.410 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:12.412 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:12.412 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:12.412 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:12.412 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:12.412 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:12.412 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-61e6a96e-6e7f-1f30-77be-99f2976b52c2 -13:12.412 [INFO] TestAgent.server: member joined, marking health alive: member=Node-61e6a96e-6e7f-1f30-77be-99f2976b52c2 -13:12.412 [INFO] TestAgent.server: New leader elected: payload=Node-61e6a96e-6e7f-1f30-77be-99f2976b52c2 -13:12.412 [INFO] TestAgent.server: federation state anti-entropy synced -13:12.422 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:12.422 [INFO] TestAgent: Synced node info -13:12.890 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:12.890 [DEBUG] TestAgent: Node info in sync -13:12.890 [DEBUG] TestAgent: Node info in sync -13:15.545 [ERROR] TestAgent.dns: recurse failed: error="read udp 127.0.0.1:54755->127.0.0.1:38010: i/o timeout" -13:15.546 [ERROR] TestAgent.dns: all resolvers failed for question from client: question="{apple.com. 255 1}" client=127.0.0.1:33975 client_network=udp -13:15.546 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=3.002221306s client=127.0.0.1:33975 client_network=udp -13:15.546 [INFO] TestAgent: Requesting shutdown -13:15.546 [INFO] TestAgent.server: shutting down server -13:15.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:15.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:15.546 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:15.546 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:15.546 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:15.546 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:15.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:15.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:15.546 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:15.546 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:15.546 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:15.546Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:15.546Z [INFO] agent.router.manager: shutting down -13:15.546 [INFO] TestAgent: consul server down -13:15.546 [INFO] TestAgent: shutdown complete -13:15.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16993 network=tcp -13:15.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16993 network=udp -13:15.546 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16994 network=tcp -13:16.049 [INFO] TestAgent: Waiting for endpoints to shut down -13:16.049 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_RecursorTimeout (3.86s) === CONT TestCoordinate_Update_ACLDeny +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:26.743Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.757Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.759 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:413d6c3a-dc92-e13b-b271-c2a967a209bf Address:127.0.0.1:23999}]" +49:26.759 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23999 [Follower]" leader= +49:26.759 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-413d6c3a-dc92-e13b-b271-c2a967a209bf.dc1 127.0.0.1 +49:26.760 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-413d6c3a-dc92-e13b-b271-c2a967a209bf 127.0.0.1 +2022-12-04T05:49:26.760Z [INFO] agent.router: Initializing LAN area manager +49:26.760 [INFO] TestAgent.server: Adding LAN server: server="Node-413d6c3a-dc92-e13b-b271-c2a967a209bf (Addr: tcp/127.0.0.1:23999) (DC: dc1)" +49:26.760 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-413d6c3a-dc92-e13b-b271-c2a967a209bf.dc1 area=wan +49:26.760 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23980 network=udp +49:26.760 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23980 network=tcp +49:26.761 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23988 network=tcp +49:26.761 [INFO] TestAgent: started state syncer +49:26.761 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24000 network=tcp +49:26.817 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.817 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23999 [Candidate]" term=2 +49:26.817 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.817 [DEBUG] TestAgent.server.raft: vote granted: from=413d6c3a-dc92-e13b-b271-c2a967a209bf term=2 tally=1 +49:26.817 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.817 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23999 [Leader]" +49:26.817 [INFO] TestAgent.server: cluster leadership acquired +49:26.817 [INFO] TestAgent.server: New leader elected: payload=Node-413d6c3a-dc92-e13b-b271-c2a967a209bf +49:26.817 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23999 +49:26.818 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.820 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.820 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.820 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.820 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.820 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.820 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-413d6c3a-dc92-e13b-b271-c2a967a209bf +49:26.820 [INFO] TestAgent.server: member joined, marking health alive: member=Node-413d6c3a-dc92-e13b-b271-c2a967a209bf +49:26.820 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.821 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=130.155µs client=127.0.0.1:48635 client_network=udp +49:26.821 [INFO] TestAgent: Requesting shutdown +49:26.821 [INFO] TestAgent.server: shutting down server +49:26.821 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.821 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.821 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.821 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.821 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.821 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:26.821 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.821 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.821 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.821 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.821 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:26.821 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:26.822Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.822Z [INFO] agent.router.manager: shutting down +49:26.822 [INFO] TestAgent: consul server down +49:26.822 [INFO] TestAgent: shutdown complete +49:26.822 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23980 network=tcp +49:26.822 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23980 network=udp +49:26.822 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23988 network=tcp +49:27.322 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.322 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ReverseLookup_CustomDomain (0.59s) +=== CONT TestDNS_NodeLookup +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=true +2022-12-04T05:49:26.758Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.768Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.769 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:71cbd705-589b-ee2f-1e85-f67ec74847c4 Address:127.0.0.1:24048}]" +49:26.770 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24048 [Follower]" leader= +49:26.770 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-71cbd705-589b-ee2f-1e85-f67ec74847c4.dc1 127.0.0.1 +49:26.770 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-71cbd705-589b-ee2f-1e85-f67ec74847c4 127.0.0.1 +2022-12-04T05:49:26.770Z [INFO] agent.router: Initializing LAN area manager +49:26.770 [INFO] TestAgent.server: Adding LAN server: server="Node-71cbd705-589b-ee2f-1e85-f67ec74847c4 (Addr: tcp/127.0.0.1:24048) (DC: dc1)" +49:26.770 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-71cbd705-589b-ee2f-1e85-f67ec74847c4.dc1 area=wan +49:26.770 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24022 network=udp +49:26.771 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24022 network=tcp +49:26.771 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24009 network=tcp +49:26.771 [INFO] TestAgent: started state syncer +49:26.771 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24049 network=tcp +49:26.835 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.835 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24048 [Candidate]" term=2 +49:26.836 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.836 [DEBUG] TestAgent.server.raft: vote granted: from=71cbd705-589b-ee2f-1e85-f67ec74847c4 term=2 tally=1 +49:26.836 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.836 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24048 [Leader]" +49:26.836 [INFO] TestAgent.server: cluster leadership acquired +49:26.836 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24048 +49:26.836 [INFO] TestAgent.server: New leader elected: payload=Node-71cbd705-589b-ee2f-1e85-f67ec74847c4 +49:26.837 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.838 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.838 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.838 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.838 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.838 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.838 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-71cbd705-589b-ee2f-1e85-f67ec74847c4 +49:26.838 [INFO] TestAgent.server: member joined, marking health alive: member=Node-71cbd705-589b-ee2f-1e85-f67ec74847c4 +49:26.839 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.910 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:26.910 [INFO] TestAgent: Synced node info +49:26.946 [DEBUG] TestAgent.dns: request served from client: name=db.service.consul. type=A class=IN latency=122.309µs client=127.0.0.1:52040 client_network=udp +49:26.947 [DEBUG] TestAgent.dns: request served from client: name=test.query.consul. type=A class=IN latency=94.768µs client=127.0.0.1:49374 client_network=udp +49:26.947 [DEBUG] TestAgent.dns: request served from client: name=test.query.consul. type=A class=IN latency=171.045µs client=127.0.0.1:49013 client_network=udp +49:26.948 [INFO] TestAgent: Requesting shutdown +49:26.948 [INFO] TestAgent.server: shutting down server +49:26.948 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.948 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.948 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.948 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.948 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.948 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.948 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.948 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.948 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:26.948 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.948 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:26.948Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.948Z [INFO] agent.router.manager: shutting down +49:26.949 [INFO] TestAgent: consul server down +49:26.949 [INFO] TestAgent: shutdown complete +49:26.949 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24022 network=tcp +49:26.949 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24022 network=udp +49:26.949 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24009 network=tcp +49:27.450 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.450 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_EDNS0_ECS (0.70s) + --- PASS: TestDNS_EDNS0_ECS/global (0.00s) + --- PASS: TestDNS_EDNS0_ECS/query (0.00s) + --- PASS: TestDNS_EDNS0_ECS/query-subnet (0.00s) +=== CONT TestDiscoveryChainRead +2022-12-04T05:49:26.919Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.928Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.930 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bcca09dd-6d95-63d9-006d-0cfb57bfb460 Address:127.0.0.1:24013}]" +49:26.930 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24013 [Follower]" leader= +49:26.930 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bcca09dd-6d95-63d9-006d-0cfb57bfb460.dc1 127.0.0.1 +49:26.931 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bcca09dd-6d95-63d9-006d-0cfb57bfb460 127.0.0.1 +2022-12-04T05:49:26.931Z [INFO] agent.router: Initializing LAN area manager +49:26.931 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24036 network=udp +49:26.931 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bcca09dd-6d95-63d9-006d-0cfb57bfb460.dc1 area=wan +49:26.931 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24036 network=tcp +49:26.931 [INFO] TestAgent.server: Adding LAN server: server="Node-bcca09dd-6d95-63d9-006d-0cfb57bfb460 (Addr: tcp/127.0.0.1:24013) (DC: dc1)" +49:26.931 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24044 network=tcp +49:26.931 [INFO] TestAgent: started state syncer +49:26.932 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24014 network=tcp +49:26.968 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.968 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24013 [Candidate]" term=2 +49:26.968 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.968 [DEBUG] TestAgent.server.raft: vote granted: from=bcca09dd-6d95-63d9-006d-0cfb57bfb460 term=2 tally=1 +49:26.968 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.968 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24013 [Leader]" +49:26.968 [INFO] TestAgent.server: cluster leadership acquired +49:26.969 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24013 +49:26.969 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.970 [INFO] TestAgent.server: New leader elected: payload=Node-bcca09dd-6d95-63d9-006d-0cfb57bfb460 +49:26.970 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.970 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.970 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.970 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.970 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.970 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bcca09dd-6d95-63d9-006d-0cfb57bfb460 +49:26.970 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bcca09dd-6d95-63d9-006d-0cfb57bfb460 +49:26.971 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.032 [DEBUG] TestAgent.dns: request served from client: name=foo.node.dc1.consul. type=ANY class=IN latency=118.209µs client=127.0.0.1:52795 client_network=udp +49:27.032 [INFO] TestAgent: Requesting shutdown +49:27.032 [INFO] TestAgent.server: shutting down server +49:27.032 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.032 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.032 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.032 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.032 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.032 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.032 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.032 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:27.032 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.032 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.032 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:27.032 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:27.032Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.032Z [INFO] agent.router.manager: shutting down +49:27.032 [INFO] TestAgent: consul server down +49:27.032 [INFO] TestAgent: shutdown complete +49:27.032 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24036 network=tcp +49:27.033 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24036 network=udp +49:27.033 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24044 network=tcp +49:27.533 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.533 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_EDNS0 (0.62s) +=== CONT TestCoordinate_Nodes [WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:15.446Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:15.474Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:15.476 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aa247f68-30b8-4ac3-b21e-2bf2c21ad0c5 Address:127.0.0.1:16265}]" -13:15.476 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aa247f68-30b8-4ac3-b21e-2bf2c21ad0c5.dc1 127.0.0.1 -13:15.476 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16265 [Follower]" leader= -13:15.477 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aa247f68-30b8-4ac3-b21e-2bf2c21ad0c5 127.0.0.1 -2021-10-31T23:13:15.477Z [INFO] agent.router: Initializing LAN area manager -13:15.477 [INFO] TestAgent.server: Adding LAN server: server="Node-aa247f68-30b8-4ac3-b21e-2bf2c21ad0c5 (Addr: tcp/127.0.0.1:16265) (DC: dc1)" -13:15.477 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aa247f68-30b8-4ac3-b21e-2bf2c21ad0c5.dc1 area=wan -13:15.477 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16295 network=tcp -13:15.477 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16295 network=udp -13:15.490 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16296 network=tcp -13:15.490 [INFO] TestAgent: started state syncer -13:15.490 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16266 network=tcp -13:15.533 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:15.533 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16265 [Candidate]" term=2 -13:15.534 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:15.534 [DEBUG] TestAgent.server.raft: vote granted: from=aa247f68-30b8-4ac3-b21e-2bf2c21ad0c5 term=2 tally=1 -13:15.534 [INFO] TestAgent.server.raft: election won: tally=1 -13:15.534 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16265 [Leader]" -13:15.534 [INFO] TestAgent.server: cluster leadership acquired -13:15.534 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16265 -13:15.534 [INFO] TestAgent.server: New leader elected: payload=Node-aa247f68-30b8-4ac3-b21e-2bf2c21ad0c5 -13:15.550 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:15.551 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:15.551 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:15.551 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:15.551 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:15.551 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:15.552 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aa247f68-30b8-4ac3-b21e-2bf2c21ad0c5 -13:15.552 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aa247f68-30b8-4ac3-b21e-2bf2c21ad0c5 -13:15.552 [INFO] TestAgent.server: federation state anti-entropy synced -13:15.653 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=AAAA class=IN latency=105.92µs client=127.0.0.1:47987 client_network=udp -13:15.653 [INFO] TestAgent: Requesting shutdown -13:15.653 [INFO] TestAgent.server: shutting down server -13:15.653 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:15.653 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:15.653 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:15.653 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:15.653 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:15.653 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:15.653 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:15.653 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:15.654 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:15.654 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:15.654 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:15.654 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:15.654Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:15.654Z [INFO] agent.router.manager: shutting down -13:15.654 [INFO] TestAgent: consul server down -13:15.654 [INFO] TestAgent: shutdown complete -13:15.654 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16295 network=tcp -13:15.654 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16295 network=udp -13:15.654 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16296 network=tcp -13:16.155 [INFO] TestAgent: Waiting for endpoints to shut down -13:16.155 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_AAAA (0.78s) +2022-12-04T05:49:26.927Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.964Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.965 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4cf690e3-e877-91ac-4444-7df737e758ef Address:127.0.0.1:23957}]" +49:26.966 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4cf690e3-e877-91ac-4444-7df737e758ef.dc1 127.0.0.1 +49:26.966 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4cf690e3-e877-91ac-4444-7df737e758ef 127.0.0.1 +2022-12-04T05:49:26.966Z [INFO] agent.router: Initializing LAN area manager +49:26.966 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24029 network=udp +49:26.967 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23957 [Follower]" leader= +49:26.967 [INFO] TestAgent.server: Adding LAN server: server="Node-4cf690e3-e877-91ac-4444-7df737e758ef (Addr: tcp/127.0.0.1:23957) (DC: dc1)" +49:26.967 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4cf690e3-e877-91ac-4444-7df737e758ef.dc1 area=wan +49:26.967 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24029 network=tcp +49:26.968 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24030 network=tcp +49:26.968 [INFO] TestAgent: started state syncer +49:26.968 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23958 network=tcp +49:27.037 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.037 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23957 [Candidate]" term=2 +49:27.037 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.037 [DEBUG] TestAgent.server.raft: vote granted: from=4cf690e3-e877-91ac-4444-7df737e758ef term=2 tally=1 +49:27.037 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.037 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23957 [Leader]" +49:27.037 [INFO] TestAgent.server: cluster leadership acquired +49:27.037 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23957 +49:27.037 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.038 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.038 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.038 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.038 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.038 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.038 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4cf690e3-e877-91ac-4444-7df737e758ef +49:27.038 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4cf690e3-e877-91ac-4444-7df737e758ef +49:27.039 [INFO] TestAgent.server: New leader elected: payload=Node-4cf690e3-e877-91ac-4444-7df737e758ef +49:27.039 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.056 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:27.056 [INFO] TestAgent: Synced node info +49:27.056 [DEBUG] TestAgent: Node info in sync +49:27.098 [DEBUG] TestAgent.dns: request served from client: name=bar.node.consul. type=AAAA class=IN latency=142.813µs client=127.0.0.1:60476 client_network=udp +49:27.098 [INFO] TestAgent: Requesting shutdown +49:27.099 [INFO] TestAgent.server: shutting down server +49:27.099 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.099 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.099 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.099 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.099 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.099 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.099 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.099 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.099 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.099 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:27.099 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:27.099Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.099Z [INFO] agent.router.manager: shutting down +49:27.099 [INFO] TestAgent: consul server down +49:27.099 [INFO] TestAgent: shutdown complete +49:27.099 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24029 network=tcp +49:27.099 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24029 network=udp +49:27.099 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24030 network=tcp +49:27.600 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.600 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_AAAA (0.69s) === CONT TestCoordinate_Update -2021-10-31T23:13:15.492Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:15.536Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:15.538 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9c14e033-f533-5068-33d2-f55d3bb32e1a Address:127.0.0.1:16160}]" -13:15.539 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9c14e033-f533-5068-33d2-f55d3bb32e1a.dc1 127.0.0.1 -13:15.539 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16160 [Follower]" leader= -13:15.539 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9c14e033-f533-5068-33d2-f55d3bb32e1a 127.0.0.1 -2021-10-31T23:13:15.539Z [INFO] agent.router: Initializing LAN area manager -13:15.539 [DEBUG] TestAgent.dns: recursor enabled -13:15.539 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16267 network=udp -13:15.539 [INFO] TestAgent.server: Adding LAN server: server="Node-9c14e033-f533-5068-33d2-f55d3bb32e1a (Addr: tcp/127.0.0.1:16160) (DC: dc1)" -13:15.540 [DEBUG] TestAgent.dns: recursor enabled -13:15.540 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16267 network=tcp -13:15.540 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16268 network=tcp -13:15.540 [INFO] TestAgent: started state syncer -13:15.540 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16161 network=tcp -13:15.554 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9c14e033-f533-5068-33d2-f55d3bb32e1a.dc1 area=wan -13:15.609 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:15.609 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16160 [Candidate]" term=2 -13:15.610 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:15.610 [DEBUG] TestAgent.server.raft: vote granted: from=9c14e033-f533-5068-33d2-f55d3bb32e1a term=2 tally=1 -13:15.610 [INFO] TestAgent.server.raft: election won: tally=1 -13:15.610 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16160 [Leader]" -13:15.610 [INFO] TestAgent.server: cluster leadership acquired -13:15.610 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16160 -13:15.611 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:15.612 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:15.612 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:15.612 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:15.613 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:15.613 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:15.613 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9c14e033-f533-5068-33d2-f55d3bb32e1a -13:15.613 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9c14e033-f533-5068-33d2-f55d3bb32e1a -13:15.613 [INFO] TestAgent.server: New leader elected: payload=Node-9c14e033-f533-5068-33d2-f55d3bb32e1a -13:15.613 [INFO] TestAgent.server: federation state anti-entropy synced -13:15.662 [DEBUG] TestAgent.dns: recurse failed for question: question="{google.com. 1 1}" rtt=73.069µs recursor=127.0.0.1:48291 rcode=SERVFAIL -13:15.662 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{google.com. 1 1}" rtt=87.644µs recursor=127.0.0.1:36195 -13:15.662 [DEBUG] TestAgent.dns: request served from client: question="{google.com. 1 1}" network=udp latency=356.285µs client=127.0.0.1:46907 client_network=udp -13:15.662 [INFO] TestAgent: Requesting shutdown -13:15.662 [INFO] TestAgent.server: shutting down server -13:15.662 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:15.663 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:15.663 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:15.663 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:15.663 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:15.663 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:15.663 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:15.663 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:15.663 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:15.663 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:15.663 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:15.663 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:15.663Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:15.663Z [INFO] agent.router.manager: shutting down -13:15.663 [INFO] TestAgent: consul server down -13:15.663 [INFO] TestAgent: shutdown complete -13:15.663 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16267 network=tcp -13:15.663 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16267 network=udp -13:15.663 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16268 network=tcp -13:16.166 [INFO] TestAgent: Waiting for endpoints to shut down -13:16.166 [INFO] TestAgent: Endpoints down ---- PASS: TestDNSCycleRecursorCheck (0.78s) -=== CONT TestPProfHandlers_DisableDebugNoACLs -2021-10-31T23:13:15.307Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:15.344Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:15.374 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0f1b5c28-5355-0b7e-3928-cc020f59d558 Address:127.0.0.1:16272}]" -13:15.375 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0f1b5c28-5355-0b7e-3928-cc020f59d558.dc1 127.0.0.1 -13:15.376 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0f1b5c28-5355-0b7e-3928-cc020f59d558 127.0.0.1 -2021-10-31T23:13:15.376Z [INFO] agent.router: Initializing LAN area manager -13:15.376 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16281 network=udp -13:15.376 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16272 [Follower]" leader= -13:15.377 [INFO] TestAgent.server: Adding LAN server: server="Node-0f1b5c28-5355-0b7e-3928-cc020f59d558 (Addr: tcp/127.0.0.1:16272) (DC: dc1)" -13:15.377 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0f1b5c28-5355-0b7e-3928-cc020f59d558.dc1 area=wan -13:15.377 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16281 network=tcp -13:15.386 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16282 network=tcp -13:15.386 [INFO] TestAgent: started state syncer -13:15.386 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16273 network=tcp -13:15.421 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:15.421 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16272 [Candidate]" term=2 -13:15.465 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:15.466 [DEBUG] TestAgent.server.raft: vote granted: from=0f1b5c28-5355-0b7e-3928-cc020f59d558 term=2 tally=1 -13:15.466 [INFO] TestAgent.server.raft: election won: tally=1 -13:15.466 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16272 [Leader]" -13:15.467 [INFO] TestAgent.server: cluster leadership acquired -13:15.467 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16272 -13:15.468 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:15.469 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:15.469 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:15.469 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:15.469 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:15.469 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:15.469 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0f1b5c28-5355-0b7e-3928-cc020f59d558 -13:15.469 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0f1b5c28-5355-0b7e-3928-cc020f59d558 -13:15.470 [INFO] TestAgent.server: New leader elected: payload=Node-0f1b5c28-5355-0b7e-3928-cc020f59d558 -13:15.470 [INFO] TestAgent.server: federation state anti-entropy synced -13:15.671 [DEBUG] TestAgent.dns: request served from client: question="{2.4.2.4.2.4.2.4.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa. 255 1}" latency=111.495µs client=127.0.0.1:58399 client_network=udp -13:15.671 [INFO] TestAgent: Requesting shutdown -13:15.671 [INFO] TestAgent.server: shutting down server -13:15.671 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:15.671 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:15.671 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:15.671 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:15.671 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:15.671 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:15.671 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:15.671 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:15.671 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:15.671 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:15.671 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:15.671 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:15.671Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:15.671Z [INFO] agent.router.manager: shutting down -13:15.671 [INFO] TestAgent: consul server down -13:15.671 [INFO] TestAgent: shutdown complete -13:15.671 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16281 network=tcp -13:15.671 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16281 network=udp -13:15.671 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16282 network=tcp -13:16.172 [INFO] TestAgent: Waiting for endpoints to shut down -13:16.172 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ReverseLookup_IPV6 (0.89s) -=== CONT TestCoordinate_Nodes +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=false +2022-12-04T05:49:26.814Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.825Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.826 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef68762c-2af9-8b5a-91fe-78cf4d74a12a Address:127.0.0.1:24034}]" +49:26.827 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24034 [Follower]" leader= +49:26.827 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef68762c-2af9-8b5a-91fe-78cf4d74a12a.dc1 127.0.0.1 +49:26.828 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef68762c-2af9-8b5a-91fe-78cf4d74a12a 127.0.0.1 +2022-12-04T05:49:26.828Z [INFO] agent.router: Initializing LAN area manager +49:26.828 [INFO] TestAgent.server: Adding LAN server: server="Node-ef68762c-2af9-8b5a-91fe-78cf4d74a12a (Addr: tcp/127.0.0.1:24034) (DC: dc1)" +49:26.828 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef68762c-2af9-8b5a-91fe-78cf4d74a12a.dc1 area=wan +49:26.828 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24050 network=udp +49:26.828 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24050 network=tcp +49:26.828 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24037 network=tcp +49:26.828 [INFO] TestAgent: started state syncer +49:26.828 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24035 network=tcp +49:26.875 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.875 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24034 [Candidate]" term=2 +49:26.875 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.875 [DEBUG] TestAgent.server.raft: vote granted: from=ef68762c-2af9-8b5a-91fe-78cf4d74a12a term=2 tally=1 +49:26.875 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.875 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24034 [Leader]" +49:26.875 [INFO] TestAgent.server: cluster leadership acquired +49:26.876 [INFO] TestAgent.server: New leader elected: payload=Node-ef68762c-2af9-8b5a-91fe-78cf4d74a12a +49:26.876 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24034 +49:26.876 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.877 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.877 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.877 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.877 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.877 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.878 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef68762c-2af9-8b5a-91fe-78cf4d74a12a +49:26.878 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef68762c-2af9-8b5a-91fe-78cf4d74a12a +49:26.878 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.156 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=114.435µs client=127.0.0.1:49699 client_network=udp +49:27.156 [INFO] TestAgent: Requesting shutdown +49:27.156 [INFO] TestAgent.server: shutting down server +49:27.156 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.156 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.156 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.156 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.156 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.156 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:27.156 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.156 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.156 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:27.156 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.156 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.156 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:27.156Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.156Z [INFO] agent.router.manager: shutting down +49:27.156 [INFO] TestAgent: consul server down +49:27.156 [INFO] TestAgent: shutdown complete +49:27.156 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24050 network=tcp +49:27.156 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24050 network=udp +49:27.156 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24037 network=tcp +49:27.657 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.657 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ReverseLookup (0.86s) +=== CONT TestConnectCAConfig +=== RUN TestConnectCAConfig/basic +2022-12-04T05:49:26.746Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.759Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.760 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f0413559-a351-1425-51f8-2cd231719388 Address:127.0.0.1:24020}]" +49:26.760 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24020 [Follower]" leader= +49:26.761 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f0413559-a351-1425-51f8-2cd231719388.dc1 127.0.0.1 +49:26.761 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f0413559-a351-1425-51f8-2cd231719388 127.0.0.1 +2022-12-04T05:49:26.761Z [INFO] agent.router: Initializing LAN area manager +49:26.761 [INFO] TestAgent.server: Adding LAN server: server="Node-f0413559-a351-1425-51f8-2cd231719388 (Addr: tcp/127.0.0.1:24020) (DC: dc1)" +49:26.761 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f0413559-a351-1425-51f8-2cd231719388.dc1 area=wan +49:26.761 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24001 network=udp +49:26.761 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24001 network=tcp +49:26.762 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24016 network=tcp +49:26.762 [INFO] TestAgent: started state syncer +49:26.762 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24021 network=tcp +49:26.823 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.823 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24020 [Candidate]" term=2 +49:26.823 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.823 [DEBUG] TestAgent.server.raft: vote granted: from=f0413559-a351-1425-51f8-2cd231719388 term=2 tally=1 +49:26.823 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.823 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24020 [Leader]" +49:26.823 [INFO] TestAgent.server: cluster leadership acquired +49:26.824 [INFO] TestAgent.server: New leader elected: payload=Node-f0413559-a351-1425-51f8-2cd231719388 +49:26.824 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24020 +49:26.824 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.826 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.826 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.826 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.826 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.826 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.826 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f0413559-a351-1425-51f8-2cd231719388 +49:26.826 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f0413559-a351-1425-51f8-2cd231719388 +49:26.826 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.019 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:27.019 [INFO] TestAgent: Synced node info +49:27.171 [DEBUG] TestAgent.dns: request served from client: question="{2.0.0.127.in-addr.arpa. 255 1}" latency=101.522µs client=127.0.0.1:55786 client_network=udp +49:27.171 [INFO] TestAgent: Requesting shutdown +49:27.171 [INFO] TestAgent.server: shutting down server +49:27.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.171 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.171 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.171 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.171 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.171 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.171 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:27.171 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:27.171Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.171Z [INFO] agent.router.manager: shutting down +49:27.171 [INFO] TestAgent: consul server down +49:27.171 [INFO] TestAgent: shutdown complete +49:27.171 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24001 network=tcp +49:27.171 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24001 network=udp +49:27.171 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24016 network=tcp +49:27.671 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.671 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceReverseLookup_CustomDomain (0.94s) +=== CONT TestCoordinate_Node +=== RUN TestCoordinate_Update_ACLDeny/no_token +=== RUN TestCoordinate_Update_ACLDeny/valid_token +2022-12-04T05:49:27.059Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.069Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.071 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bf8fa731-f4d0-b918-7abc-5e0ab84cf840 Address:127.0.0.1:24062}]" +49:27.071 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24062 [Follower]" leader= +49:27.071 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bf8fa731-f4d0-b918-7abc-5e0ab84cf840.dc1 127.0.0.1 +49:27.072 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bf8fa731-f4d0-b918-7abc-5e0ab84cf840 127.0.0.1 +2022-12-04T05:49:27.072Z [INFO] agent.router: Initializing LAN area manager +49:27.072 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bf8fa731-f4d0-b918-7abc-5e0ab84cf840.dc1 area=wan +49:27.072 [INFO] TestAgent.server: Adding LAN server: server="Node-bf8fa731-f4d0-b918-7abc-5e0ab84cf840 (Addr: tcp/127.0.0.1:24062) (DC: dc1)" +49:27.072 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24078 network=udp +49:27.072 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24078 network=tcp +49:27.072 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24065 network=tcp +49:27.072 [INFO] TestAgent: started state syncer +49:27.072 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24063 network=tcp +49:27.111 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.112 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24062 [Candidate]" term=2 +49:27.112 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.112 [DEBUG] TestAgent.server.raft: vote granted: from=bf8fa731-f4d0-b918-7abc-5e0ab84cf840 term=2 tally=1 +49:27.112 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.112 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24062 [Leader]" +49:27.112 [INFO] TestAgent.server: cluster leadership acquired +49:27.112 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24062 +49:27.113 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.114 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.114 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.114 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.114 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.114 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.114 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bf8fa731-f4d0-b918-7abc-5e0ab84cf840 +49:27.114 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bf8fa731-f4d0-b918-7abc-5e0ab84cf840 +49:27.114 [INFO] TestAgent.server: New leader elected: payload=Node-bf8fa731-f4d0-b918-7abc-5e0ab84cf840 +49:27.115 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.250 [INFO] TestAgent: Requesting shutdown +49:27.250 [INFO] TestAgent.server: shutting down server +49:27.250 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.250 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.250 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.250 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.250 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.250 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:27.250 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.250 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.250 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.250 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:27.250 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.250 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:27.250Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.250Z [INFO] agent.router.manager: shutting down +49:27.250 [INFO] TestAgent: consul server down +49:27.250 [INFO] TestAgent: shutdown complete +49:27.250 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24078 network=tcp +49:27.250 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24078 network=udp +49:27.250 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24065 network=tcp +49:27.751 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.751 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_EmptyAltDomain (0.70s) +=== CONT TestConnectCARoots_list +2022-12-04T05:49:27.171Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.181Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.182 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:47ad1b19-229b-67aa-7bfe-3f44d26fbc39 Address:127.0.0.1:24055}]" +49:27.182 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24055 [Follower]" leader= +49:27.183 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-47ad1b19-229b-67aa-7bfe-3f44d26fbc39.dc1 127.0.0.1 +49:27.183 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-47ad1b19-229b-67aa-7bfe-3f44d26fbc39 127.0.0.1 +2022-12-04T05:49:27.183Z [INFO] agent.router: Initializing LAN area manager +49:27.183 [INFO] TestAgent.server: Adding LAN server: server="Node-47ad1b19-229b-67aa-7bfe-3f44d26fbc39 (Addr: tcp/127.0.0.1:24055) (DC: dc1)" +49:27.183 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-47ad1b19-229b-67aa-7bfe-3f44d26fbc39.dc1 area=wan +49:27.183 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24064 network=udp +49:27.183 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24064 network=tcp +49:27.184 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24051 network=tcp +49:27.184 [INFO] TestAgent: started state syncer +49:27.184 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24056 network=tcp +49:27.240 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.240 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24055 [Candidate]" term=2 +49:27.240 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.240 [DEBUG] TestAgent.server.raft: vote granted: from=47ad1b19-229b-67aa-7bfe-3f44d26fbc39 term=2 tally=1 +49:27.240 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.240 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24055 [Leader]" +49:27.240 [INFO] TestAgent.server: cluster leadership acquired +49:27.240 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24055 +49:27.240 [INFO] TestAgent.server: New leader elected: payload=Node-47ad1b19-229b-67aa-7bfe-3f44d26fbc39 +49:27.241 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.242 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.242 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.242 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.242 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.242 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.242 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-47ad1b19-229b-67aa-7bfe-3f44d26fbc39 +49:27.242 [INFO] TestAgent.server: member joined, marking health alive: member=Node-47ad1b19-229b-67aa-7bfe-3f44d26fbc39 +49:27.242 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.252 [DEBUG] TestAgent.dns: request served from client: name=foo.node.dc1.consul. type=ANY class=IN latency=68.003µs client=127.0.0.1:55518 client_network=tcp +49:27.252 [INFO] TestAgent: Requesting shutdown +49:27.252 [INFO] TestAgent.server: shutting down server +49:27.252 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.252 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.252 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.252 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.252 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.252 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.252 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.253 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:27.253 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.253 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.253 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:27.253 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:27.253Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.253Z [INFO] agent.router.manager: shutting down +49:27.253 [INFO] TestAgent: consul server down +49:27.253 [INFO] TestAgent: shutdown complete +49:27.253 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24064 network=tcp +49:27.253 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24064 network=udp +49:27.253 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24051 network=tcp +49:27.755 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.755 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_Over_TCP (0.59s) +=== CONT TestCoordinate_Datacenters === RUN TestDiscoveryChainRead/GET:_error_on_no_service_name === RUN TestDiscoveryChainRead/GET:_read_default_chain === RUN TestDiscoveryChainRead/GET:_read_default_chain;_evaluate_in_dc2 @@ -32071,498 +38068,460 @@ === RUN TestDiscoveryChainRead/POST:_read_default_chain === RUN TestDiscoveryChainRead/POST:_read_default_chain;_evaluate_in_dc2 === RUN TestDiscoveryChainRead/POST:_read_default_chain_with_cache -2021-10-31T23:13:15.417Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:15.473Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:15.475 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2d14495e-1a63-fdbd-9b8b-c6cb4edcd77e Address:127.0.0.1:16293}]" -13:15.475 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2d14495e-1a63-fdbd-9b8b-c6cb4edcd77e.dc1 127.0.0.1 -13:15.476 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2d14495e-1a63-fdbd-9b8b-c6cb4edcd77e 127.0.0.1 -2021-10-31T23:13:15.476Z [INFO] agent.router: Initializing LAN area manager -13:15.476 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16293 [Follower]" leader= -13:15.476 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16260 network=udp -13:15.476 [INFO] TestAgent.server: Adding LAN server: server="Node-2d14495e-1a63-fdbd-9b8b-c6cb4edcd77e (Addr: tcp/127.0.0.1:16293) (DC: dc1)" -13:15.476 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2d14495e-1a63-fdbd-9b8b-c6cb4edcd77e.dc1 area=wan -13:15.476 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16260 network=tcp -13:15.477 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16261 network=tcp -13:15.477 [INFO] TestAgent: started state syncer -13:15.477 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16294 network=tcp -13:15.545 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:15.545 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16293 [Candidate]" term=2 -13:15.546 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:15.546 [DEBUG] TestAgent.server.raft: vote granted: from=2d14495e-1a63-fdbd-9b8b-c6cb4edcd77e term=2 tally=1 -13:15.546 [INFO] TestAgent.server.raft: election won: tally=1 -13:15.546 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16293 [Leader]" -13:15.546 [INFO] TestAgent.server: cluster leadership acquired -13:15.546 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16293 -13:15.546 [INFO] TestAgent.server: New leader elected: payload=Node-2d14495e-1a63-fdbd-9b8b-c6cb4edcd77e -13:15.547 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:15.548 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:15.548 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:15.548 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:15.548 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:15.548 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:15.548 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2d14495e-1a63-fdbd-9b8b-c6cb4edcd77e -13:15.548 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2d14495e-1a63-fdbd-9b8b-c6cb4edcd77e -13:15.549 [INFO] TestAgent.server: federation state anti-entropy synced -13:15.695 [DEBUG] TestAgent.dns: request served from client: name=foo.node.dc1.consul. type=ANY class=IN latency=88.727µs client=127.0.0.1:53528 client_network=udp -13:15.695 [INFO] TestAgent: Requesting shutdown -13:15.695 [INFO] TestAgent.server: shutting down server -13:15.695 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:15.695 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:15.695 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:15.695 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:15.695 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:15.695 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:15.695 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:15.695 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:15.695 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:15.696 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:15.696 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:15.696 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:15.696Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:15.696Z [INFO] agent.router.manager: shutting down -13:15.696 [INFO] TestAgent: consul server down -13:15.696 [INFO] TestAgent: shutdown complete -13:15.696 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16260 network=tcp -13:15.696 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16260 network=udp -13:15.696 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16261 network=tcp -13:16.197 [INFO] TestAgent: Waiting for endpoints to shut down -13:16.197 [INFO] TestAgent: Endpoints down === RUN TestDiscoveryChainRead/GET:_read_modified_chain ---- PASS: TestDNS_EDNS0 (0.85s) -=== CONT TestPProfHandlers_EnableDebug +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:27.058Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.071Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.072 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5d357f84-20b2-02a0-777b-469ac7a8a324 Address:127.0.0.1:24076}]" +49:27.073 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24076 [Follower]" leader= +49:27.073 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5d357f84-20b2-02a0-777b-469ac7a8a324.dc1 127.0.0.1 +49:27.073 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5d357f84-20b2-02a0-777b-469ac7a8a324 127.0.0.1 +2022-12-04T05:49:27.073Z [INFO] agent.router: Initializing LAN area manager +49:27.074 [INFO] TestAgent.server: Adding LAN server: server="Node-5d357f84-20b2-02a0-777b-469ac7a8a324 (Addr: tcp/127.0.0.1:24076) (DC: dc1)" +49:27.074 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5d357f84-20b2-02a0-777b-469ac7a8a324.dc1 area=wan +49:27.074 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24145 network=udp +49:27.074 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24145 network=tcp +49:27.074 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24146 network=tcp +49:27.074 [INFO] TestAgent: started state syncer +49:27.075 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24077 network=tcp +49:27.120 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.120 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24076 [Candidate]" term=2 +49:27.120 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.120 [DEBUG] TestAgent.server.raft: vote granted: from=5d357f84-20b2-02a0-777b-469ac7a8a324 term=2 tally=1 +49:27.120 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.120 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24076 [Leader]" +49:27.120 [INFO] TestAgent.server: cluster leadership acquired +49:27.120 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24076 +49:27.121 [INFO] TestAgent.server: New leader elected: payload=Node-5d357f84-20b2-02a0-777b-469ac7a8a324 +49:27.121 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.122 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.122 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.122 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.122 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.122 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.122 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5d357f84-20b2-02a0-777b-469ac7a8a324 +49:27.122 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5d357f84-20b2-02a0-777b-469ac7a8a324 +49:27.123 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.123 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:27.123 [INFO] TestAgent: Synced node info +49:27.293 [DEBUG] TestAgent.dns: request served from client: name=foo.bar.node.consul. type=ANY class=IN latency=189.925µs client=127.0.0.1:52714 client_network=udp +49:27.293 [INFO] TestAgent: Requesting shutdown +49:27.293 [INFO] TestAgent.server: shutting down server +49:27.293 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.293 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.293 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:27.294 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.294 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.294 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.294 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.294 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.294 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:27.294 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" +49:27.294 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +2022-12-04T05:49:27.294Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.294Z [INFO] agent.router.manager: shutting down +49:27.295 [INFO] TestAgent: consul server down +49:27.295 [INFO] TestAgent: shutdown complete +49:27.295 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24145 network=tcp +49:27.295 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24145 network=udp +49:27.295 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24146 network=tcp +49:27.803 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.803 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_PeriodName (0.76s) +=== CONT TestConnectCARoots_empty === RUN TestDiscoveryChainRead/POST:_read_modified_chain_with_overrides_(camel_case) === RUN TestDiscoveryChainRead/POST:_read_modified_chain_with_overrides_(snake_case) -=== RUN TestPProfHandlers_ACLs/case_0_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:200,_token:"master",_endpoint:"/debug/pprof/heap",_nilResponse:false}) -2021-10-31T23:13:15.382Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:15.454Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:15.456 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:763e2db7-0c60-1585-b0b1-0ea335642720 Address:127.0.0.1:16258}]" -13:15.457 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-763e2db7-0c60-1585-b0b1-0ea335642720.dc1 127.0.0.1 -13:15.457 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-763e2db7-0c60-1585-b0b1-0ea335642720 127.0.0.1 -13:15.458 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16258 [Follower]" leader= -2021-10-31T23:13:15.461Z [INFO] agent.router: Initializing LAN area manager -13:15.462 [DEBUG] TestAgent.dns: recursor enabled -13:15.462 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16274 network=udp -13:15.462 [INFO] TestAgent.server: Adding LAN server: server="Node-763e2db7-0c60-1585-b0b1-0ea335642720 (Addr: tcp/127.0.0.1:16258) (DC: dc1)" -13:15.462 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-763e2db7-0c60-1585-b0b1-0ea335642720.dc1 area=wan -13:15.462 [DEBUG] TestAgent.dns: recursor enabled -13:15.462 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16274 network=tcp -13:15.462 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16275 network=tcp -13:15.463 [INFO] TestAgent: started state syncer -13:15.463 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16259 network=tcp -13:15.505 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:15.505 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16258 [Candidate]" term=2 -13:15.506 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:15.506 [DEBUG] TestAgent.server.raft: vote granted: from=763e2db7-0c60-1585-b0b1-0ea335642720 term=2 tally=1 -13:15.506 [INFO] TestAgent.server.raft: election won: tally=1 -13:15.506 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16258 [Leader]" -13:15.506 [INFO] TestAgent.server: cluster leadership acquired -13:15.506 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16258 -13:15.506 [INFO] TestAgent.server: New leader elected: payload=Node-763e2db7-0c60-1585-b0b1-0ea335642720 -13:15.507 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:15.508 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:15.508 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:15.508 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:15.508 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:15.508 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:15.508 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-763e2db7-0c60-1585-b0b1-0ea335642720 -13:15.508 [INFO] TestAgent.server: member joined, marking health alive: member=Node-763e2db7-0c60-1585-b0b1-0ea335642720 -13:15.509 [INFO] TestAgent.server: federation state anti-entropy synced -13:15.786 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:15.786 [INFO] TestAgent: Synced node info -13:15.827 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=146.272µs -13:15.827 [DEBUG] TestAgent.dns: request served from client: name=google.node.consul. type=ANY class=IN latency=474.782µs client=127.0.0.1:57405 client_network=udp -13:15.830 [INFO] TestAgent: Requesting shutdown -13:15.830 [INFO] TestAgent.server: shutting down server -13:15.830 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:15.830 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:15.830 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:15.830 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:15.830 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:15.830 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:15.830 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:15.830 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:15.830 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:15.830 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:15.830 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:15.830Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:15.830Z [INFO] agent.router.manager: shutting down -13:15.830 [INFO] TestAgent: consul server down -13:15.830 [INFO] TestAgent: shutdown complete -13:15.830 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16274 network=tcp -13:15.830 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16274 network=udp -13:15.830 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16275 network=tcp -13:16.332 [INFO] TestAgent: Waiting for endpoints to shut down -13:16.332 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_CNAME (1.02s) -=== CONT TestCoordinate_Datacenters -[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:15.547Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:15.568Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:15.582 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3d4ffb35-6819-2f26-55e6-9e25f8d82f34 Address:127.0.0.1:16328}]" -13:15.583 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3d4ffb35-6819-2f26-55e6-9e25f8d82f34.dc1 127.0.0.1 -13:15.583 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3d4ffb35-6819-2f26-55e6-9e25f8d82f34 127.0.0.1 -2021-10-31T23:13:15.583Z [INFO] agent.router: Initializing LAN area manager -13:15.584 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16288 network=udp -13:15.584 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16328 [Follower]" leader= -13:15.584 [INFO] TestAgent.server: Adding LAN server: server="Node-3d4ffb35-6819-2f26-55e6-9e25f8d82f34 (Addr: tcp/127.0.0.1:16328) (DC: dc1)" -13:15.584 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3d4ffb35-6819-2f26-55e6-9e25f8d82f34.dc1 area=wan -13:15.584 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16288 network=tcp -13:15.585 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16289 network=tcp -13:15.585 [INFO] TestAgent: started state syncer -13:15.585 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16329 network=tcp -13:15.621 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:15.621 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16328 [Candidate]" term=2 -13:15.621 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:15.621 [DEBUG] TestAgent.server.raft: vote granted: from=3d4ffb35-6819-2f26-55e6-9e25f8d82f34 term=2 tally=1 -13:15.621 [INFO] TestAgent.server.raft: election won: tally=1 -13:15.621 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16328 [Leader]" -13:15.621 [INFO] TestAgent.server: cluster leadership acquired -13:15.622 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16328 -13:15.622 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:15.624 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:15.624 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:15.624 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:15.624 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:15.624 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:15.624 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3d4ffb35-6819-2f26-55e6-9e25f8d82f34 -13:15.624 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3d4ffb35-6819-2f26-55e6-9e25f8d82f34 -13:15.624 [INFO] TestAgent.server: New leader elected: payload=Node-3d4ffb35-6819-2f26-55e6-9e25f8d82f34 -13:15.625 [INFO] TestAgent.server: federation state anti-entropy synced -13:15.866 [INFO] TestAgent: Requesting shutdown -13:15.866 [INFO] TestAgent.server: shutting down server -13:15.866 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:15.866 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:15.866 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:15.866 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:15.866 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:15.866 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:15.866 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:15.866 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:15.866 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:15.866 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:15.866 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:15.866 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:15.867Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:15.867Z [INFO] agent.router.manager: shutting down -13:15.867 [INFO] TestAgent: consul server down -13:15.867 [INFO] TestAgent: shutdown complete -13:15.867 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16288 network=tcp -13:15.867 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16288 network=udp -13:15.867 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16289 network=tcp -13:16.370 [INFO] TestAgent: Waiting for endpoints to shut down -13:16.370 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_EmptyAltDomain (0.87s) -=== CONT TestParseWait ---- PASS: TestParseWait (0.00s) -=== CONT TestCoordinate_Node -=== RUN TestPProfHandlers_ACLs/case_1_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"agent",_endpoint:"/debug/pprof/heap",_nilResponse:true}) -=== RUN TestPProfHandlers_ACLs/case_2_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"agent",_endpoint:"/debug/pprof/",_nilResponse:true}) -=== RUN TestPProfHandlers_ACLs/case_3_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"",_endpoint:"/debug/pprof/",_nilResponse:true}) -=== RUN TestPProfHandlers_ACLs/case_4_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:200,_token:"master",_endpoint:"/debug/pprof/heap",_nilResponse:false}) -2021-10-31T23:13:15.766Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:15.815Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:15.817 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:feeebdce-0146-265b-ea74-bd185d0be484 Address:127.0.0.1:16342}]" -13:15.817 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16342 [Follower]" leader= -13:15.825 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-feeebdce-0146-265b-ea74-bd185d0be484.dc1 127.0.0.1 -13:15.826 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-feeebdce-0146-265b-ea74-bd185d0be484 127.0.0.1 -2021-10-31T23:13:15.826Z [INFO] agent.router: Initializing LAN area manager -13:15.826 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16316 network=udp -13:15.826 [INFO] TestAgent.server: Adding LAN server: server="Node-feeebdce-0146-265b-ea74-bd185d0be484 (Addr: tcp/127.0.0.1:16342) (DC: dc1)" -13:15.826 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-feeebdce-0146-265b-ea74-bd185d0be484.dc1 area=wan -13:15.826 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16316 network=tcp -13:15.827 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16317 network=tcp -13:15.827 [INFO] TestAgent: started state syncer -13:15.827 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16343 network=tcp -13:15.889 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:15.889 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16342 [Candidate]" term=2 -13:15.890 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:15.890 [DEBUG] TestAgent.server.raft: vote granted: from=feeebdce-0146-265b-ea74-bd185d0be484 term=2 tally=1 -13:15.890 [INFO] TestAgent.server.raft: election won: tally=1 -13:15.890 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16342 [Leader]" -13:15.890 [INFO] TestAgent.server: cluster leadership acquired -13:15.890 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16342 -13:15.891 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:15.891 [INFO] TestAgent.server: New leader elected: payload=Node-feeebdce-0146-265b-ea74-bd185d0be484 -13:15.892 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:15.892 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:15.892 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:15.892 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:15.892 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:15.892 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-feeebdce-0146-265b-ea74-bd185d0be484 -13:15.892 [INFO] TestAgent.server: member joined, marking health alive: member=Node-feeebdce-0146-265b-ea74-bd185d0be484 -13:15.893 [INFO] TestAgent.server: federation state anti-entropy synced -13:15.926 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=ANY class=IN latency=132.437µs client=127.0.0.1:36757 client_network=udp -13:15.927 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=77.5µs client=127.0.0.1:39602 client_network=udp -13:15.927 [DEBUG] TestAgent.dns: request served from client: name=foo.node.dc1.consul. type=ANY class=IN latency=80.438µs client=127.0.0.1:47028 client_network=udp -13:15.927 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=68.303µs client=127.0.0.1:37046 client_network=udp -13:15.927 [INFO] TestAgent: Requesting shutdown -13:15.927 [INFO] TestAgent.server: shutting down server -13:15.927 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:15.927 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:15.927 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:15.927 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:15.927 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:15.927 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:15.927 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:15.927 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:15.927 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:15.927 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:15.927 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:15.928 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:15.928Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:15.928Z [INFO] agent.router.manager: shutting down -13:15.928 [INFO] TestAgent: consul server down -13:15.928 [INFO] TestAgent: shutdown complete -13:15.928 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16316 network=tcp -13:15.928 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16316 network=udp -13:15.928 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16317 network=tcp -13:16.429 [INFO] TestAgent: Waiting for endpoints to shut down -13:16.429 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup (0.74s) +2022-12-04T05:49:27.008Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.019Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.021 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:991e2ec1-f0a1-881f-705c-37383bbd73db Address:127.0.0.1:24143}]" +49:27.021 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24143 [Follower]" leader= +49:27.021 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-991e2ec1-f0a1-881f-705c-37383bbd73db.dc1 127.0.0.1 +49:27.022 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-991e2ec1-f0a1-881f-705c-37383bbd73db 127.0.0.1 +2022-12-04T05:49:27.022Z [INFO] agent.router: Initializing LAN area manager +49:27.022 [INFO] TestAgent.server: Adding LAN server: server="Node-991e2ec1-f0a1-881f-705c-37383bbd73db (Addr: tcp/127.0.0.1:24143) (DC: dc1)" +49:27.022 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-991e2ec1-f0a1-881f-705c-37383bbd73db.dc1 area=wan +49:27.022 [DEBUG] TestAgent.dns: recursor enabled +49:27.022 [DEBUG] TestAgent.dns: recursor enabled +49:27.022 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23959 network=udp +49:27.022 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23959 network=tcp +49:27.022 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23960 network=tcp +49:27.022 [INFO] TestAgent: started state syncer +49:27.022 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24144 network=tcp +49:27.062 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.062 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24143 [Candidate]" term=2 +49:27.062 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.062 [DEBUG] TestAgent.server.raft: vote granted: from=991e2ec1-f0a1-881f-705c-37383bbd73db term=2 tally=1 +49:27.062 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.062 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24143 [Leader]" +49:27.063 [INFO] TestAgent.server: cluster leadership acquired +49:27.063 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24143 +49:27.063 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.064 [INFO] TestAgent.server: New leader elected: payload=Node-991e2ec1-f0a1-881f-705c-37383bbd73db +49:27.064 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.064 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.064 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.064 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.064 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.064 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-991e2ec1-f0a1-881f-705c-37383bbd73db +49:27.065 [INFO] TestAgent.server: member joined, marking health alive: member=Node-991e2ec1-f0a1-881f-705c-37383bbd73db +49:27.065 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.349 [DEBUG] TestAgent.dns: recurse failed for question: question="{google.com. 1 1}" rtt=82.915µs recursor=127.0.0.1:52906 rcode=SERVFAIL +49:27.349 [DEBUG] TestAgent.dns: recurse succeeded for question: question="{google.com. 1 1}" rtt=79.921µs recursor=127.0.0.1:36028 +49:27.349 [DEBUG] TestAgent.dns: request served from client: question="{google.com. 1 1}" network=udp latency=325.776µs client=127.0.0.1:54999 client_network=udp +49:27.349 [INFO] TestAgent: Requesting shutdown +49:27.349 [INFO] TestAgent.server: shutting down server +49:27.349 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.349 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.349 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.349 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.349 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.349 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.349 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.349 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:27.349 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.349 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.349 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:27.349 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:27.349Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.349Z [INFO] agent.router.manager: shutting down +49:27.350 [INFO] TestAgent: consul server down +49:27.350 [INFO] TestAgent: shutdown complete +49:27.350 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23959 network=tcp +49:27.350 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23959 network=udp +49:27.350 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23960 network=tcp +49:27.850 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.850 [INFO] TestAgent: Endpoints down +--- PASS: TestDNSCycleRecursorCheck (0.86s) === CONT TestCoordinate_Disabled_Response -2021-10-31T23:13:15.456Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:15.518Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:15.519 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e78c3301-ce0b-33c2-f90d-50c4bd41d271 Address:127.0.0.1:16321}]" -13:15.520 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e78c3301-ce0b-33c2-f90d-50c4bd41d271.dc1 127.0.0.1 -13:15.521 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e78c3301-ce0b-33c2-f90d-50c4bd41d271 127.0.0.1 -2021-10-31T23:13:15.521Z [INFO] agent.router: Initializing LAN area manager -13:15.521 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16169 network=udp -13:15.521 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16321 [Follower]" leader= -13:15.521 [INFO] TestAgent.server: Adding LAN server: server="Node-e78c3301-ce0b-33c2-f90d-50c4bd41d271 (Addr: tcp/127.0.0.1:16321) (DC: dc1)" -13:15.521 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e78c3301-ce0b-33c2-f90d-50c4bd41d271.dc1 area=wan -13:15.533 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16169 network=tcp -13:15.534 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16170 network=tcp -13:15.534 [INFO] TestAgent: started state syncer -13:15.534 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16322 network=tcp -13:15.589 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:15.590 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16321 [Candidate]" term=2 -13:15.590 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:15.590 [DEBUG] TestAgent.server.raft: vote granted: from=e78c3301-ce0b-33c2-f90d-50c4bd41d271 term=2 tally=1 -13:15.590 [INFO] TestAgent.server.raft: election won: tally=1 -13:15.590 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16321 [Leader]" -13:15.590 [INFO] TestAgent.server: cluster leadership acquired -13:15.590 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16321 -13:15.590 [INFO] TestAgent.server: New leader elected: payload=Node-e78c3301-ce0b-33c2-f90d-50c4bd41d271 -13:15.591 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:15.592 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:15.592 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:15.592 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:15.592 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:15.592 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:15.592 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e78c3301-ce0b-33c2-f90d-50c4bd41d271 -13:15.592 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e78c3301-ce0b-33c2-f90d-50c4bd41d271 -13:15.593 [INFO] TestAgent.server: federation state anti-entropy synced -13:15.618 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:15.618 [INFO] TestAgent: Synced node info -13:15.942 [DEBUG] TestAgent.dns: request served from client: name=fOO.node.dc1.consul. type=ANY class=IN latency=224.337µs client=127.0.0.1:60605 client_network=udp -13:15.945 [INFO] TestAgent: Requesting shutdown -13:15.946 [INFO] TestAgent.server: shutting down server -13:15.946 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:15.946 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:15.946 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:15.946 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:15.946 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:15.946 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:15.946 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:15.946 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:15.946 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:15.946 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:15.946 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:15.946Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:15.946Z [INFO] agent.router.manager: shutting down -13:15.946 [INFO] TestAgent: consul server down -13:15.946 [INFO] TestAgent: shutdown complete -13:15.946 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16169 network=tcp -13:15.946 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16169 network=udp -13:15.946 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16170 network=tcp -13:16.446 [INFO] TestAgent: Waiting for endpoints to shut down -13:16.446 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_CaseInsensitiveNodeLookup (1.08s) -=== CONT TestParseSource -=== RUN TestPProfHandlers_ACLs/case_5_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"towel",_endpoint:"/debug/pprof/heap",_nilResponse:true}) -=== RUN TestCoordinate_Update_ACLDeny/no_token -=== RUN TestCoordinate_Update_ACLDeny/valid_token -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:15.658Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:15.751Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:15.753 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:da547a46-c4b7-69b9-bdc5-976b421dd7d0 Address:127.0.0.1:16314}]" -13:15.753 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-da547a46-c4b7-69b9-bdc5-976b421dd7d0.dc1 127.0.0.1 -13:15.757 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16314 [Follower]" leader= -13:15.762 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-da547a46-c4b7-69b9-bdc5-976b421dd7d0 127.0.0.1 -2021-10-31T23:13:15.762Z [INFO] agent.router: Initializing LAN area manager -13:15.762 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16330 network=udp -13:15.762 [INFO] TestAgent.server: Adding LAN server: server="Node-da547a46-c4b7-69b9-bdc5-976b421dd7d0 (Addr: tcp/127.0.0.1:16314) (DC: dc1)" -13:15.763 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-da547a46-c4b7-69b9-bdc5-976b421dd7d0.dc1 area=wan -13:15.763 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16330 network=tcp -13:15.763 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16331 network=tcp -13:15.763 [INFO] TestAgent: started state syncer -13:15.763 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16315 network=tcp -13:15.805 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:15.805 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16314 [Candidate]" term=2 -13:15.806 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:15.806 [DEBUG] TestAgent.server.raft: vote granted: from=da547a46-c4b7-69b9-bdc5-976b421dd7d0 term=2 tally=1 -13:15.806 [INFO] TestAgent.server.raft: election won: tally=1 -13:15.806 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16314 [Leader]" -13:15.806 [INFO] TestAgent.server: cluster leadership acquired -13:15.806 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16314 -13:15.806 [INFO] TestAgent.server: New leader elected: payload=Node-da547a46-c4b7-69b9-bdc5-976b421dd7d0 -13:15.807 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:15.808 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:15.808 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:15.808 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:15.808 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:15.808 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:15.808 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-da547a46-c4b7-69b9-bdc5-976b421dd7d0 -13:15.808 [INFO] TestAgent.server: member joined, marking health alive: member=Node-da547a46-c4b7-69b9-bdc5-976b421dd7d0 -13:15.809 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.103 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:16.103 [INFO] TestAgent: Synced node info -13:16.103 [DEBUG] TestAgent: Node info in sync -13:16.186 [INFO] TestAgent: Requesting shutdown -13:16.186 [INFO] TestAgent.server: shutting down server -13:16.186 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:16.186 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:16.186 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:16.186 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:16.186 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:16.186 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:16.186 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:16.186 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:16.186 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:16.187 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:16.187 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:16.187Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:16.187Z [INFO] agent.router.manager: shutting down -13:16.187 [INFO] TestAgent: consul server down -13:16.187 [INFO] TestAgent: shutdown complete -13:16.187 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16330 network=tcp -13:16.187 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16330 network=udp -13:16.187 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16331 network=tcp -13:16.687 [INFO] TestAgent: Waiting for endpoints to shut down -13:16.687 [INFO] TestAgent: Endpoints down ---- PASS: TestParseConsistency (1.07s) -=== CONT TestConnectCARoots_list -2021-10-31T23:13:15.824Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:15.850Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:15.852 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a0293a5c-0664-1e06-3c43-a588e55ae244 Address:127.0.0.1:16307}]" -13:15.853 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a0293a5c-0664-1e06-3c43-a588e55ae244.dc1 127.0.0.1 -13:15.853 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16307 [Follower]" leader= -13:15.858 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a0293a5c-0664-1e06-3c43-a588e55ae244 127.0.0.1 -2021-10-31T23:13:15.858Z [INFO] agent.router: Initializing LAN area manager -13:15.859 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16344 network=udp -13:15.859 [INFO] TestAgent.server: Adding LAN server: server="Node-a0293a5c-0664-1e06-3c43-a588e55ae244 (Addr: tcp/127.0.0.1:16307) (DC: dc1)" -13:15.859 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a0293a5c-0664-1e06-3c43-a588e55ae244.dc1 area=wan -13:15.859 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16344 network=tcp -13:15.860 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16345 network=tcp -13:15.860 [INFO] TestAgent: started state syncer -13:15.860 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16308 network=tcp -13:15.905 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:15.905 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16307 [Candidate]" term=2 -13:15.905 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:15.905 [DEBUG] TestAgent.server.raft: vote granted: from=a0293a5c-0664-1e06-3c43-a588e55ae244 term=2 tally=1 -13:15.905 [INFO] TestAgent.server.raft: election won: tally=1 -13:15.905 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16307 [Leader]" -13:15.905 [INFO] TestAgent.server: cluster leadership acquired -13:15.905 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16307 -13:15.905 [INFO] TestAgent.server: New leader elected: payload=Node-a0293a5c-0664-1e06-3c43-a588e55ae244 -13:15.906 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:15.908 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:15.908 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:15.908 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:15.908 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:15.908 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:15.908 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a0293a5c-0664-1e06-3c43-a588e55ae244 -13:15.908 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a0293a5c-0664-1e06-3c43-a588e55ae244 -13:15.909 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.177 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:16.178 [INFO] TestAgent: Synced node info -13:16.178 [DEBUG] TestAgent: Node info in sync -13:16.182 [DEBUG] TestAgent.dns: request served from client: name=foo.node.dc1.consul. type=ANY class=IN latency=176.535µs client=127.0.0.1:41570 client_network=tcp -13:16.186 [INFO] TestAgent: Requesting shutdown -13:16.186 [INFO] TestAgent.server: shutting down server -13:16.186 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:16.186 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:16.186 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:16.186 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:16.186 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:16.186 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:16.186 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:16.186 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:16.186 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:16.186 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:16.186 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:16.186Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:16.186Z [INFO] agent.router.manager: shutting down -13:16.186 [INFO] TestAgent: consul server down -13:16.186 [INFO] TestAgent: shutdown complete -13:16.186 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16344 network=tcp -13:16.186 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16344 network=udp -13:16.186 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16345 network=tcp -13:16.688 [INFO] TestAgent: Waiting for endpoints to shut down -13:16.688 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_Over_TCP (0.88s) -=== CONT TestConnectCAConfig -=== RUN TestConnectCAConfig/basic +2022-12-04T05:49:26.923Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.944Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.949 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:00374d90-2147-18c7-9b79-c83df57ae59f Address:127.0.0.1:24027}]" +49:26.949 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24027 [Follower]" leader= +49:26.951 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-00374d90-2147-18c7-9b79-c83df57ae59f.dc1 127.0.0.1 +49:26.962 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-00374d90-2147-18c7-9b79-c83df57ae59f 127.0.0.1 +2022-12-04T05:49:26.962Z [INFO] agent.router: Initializing LAN area manager +49:26.962 [DEBUG] TestAgent.dns: recursor enabled +49:26.962 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24015 network=udp +49:26.966 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-00374d90-2147-18c7-9b79-c83df57ae59f.dc1 area=wan +49:26.966 [DEBUG] TestAgent.dns: recursor enabled +49:26.966 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24015 network=tcp +49:26.967 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24023 network=tcp +49:26.967 [INFO] TestAgent: started state syncer +49:26.968 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24028 network=tcp +49:26.968 [INFO] TestAgent.server: Adding LAN server: server="Node-00374d90-2147-18c7-9b79-c83df57ae59f (Addr: tcp/127.0.0.1:24027) (DC: dc1)" +49:27.006 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.006 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24027 [Candidate]" term=2 +49:27.006 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.006 [DEBUG] TestAgent.server.raft: vote granted: from=00374d90-2147-18c7-9b79-c83df57ae59f term=2 tally=1 +49:27.006 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.006 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24027 [Leader]" +49:27.006 [INFO] TestAgent.server: cluster leadership acquired +49:27.006 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24027 +49:27.007 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.008 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.008 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.008 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.008 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.008 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.008 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-00374d90-2147-18c7-9b79-c83df57ae59f +49:27.008 [INFO] TestAgent.server: member joined, marking health alive: member=Node-00374d90-2147-18c7-9b79-c83df57ae59f +49:27.008 [INFO] TestAgent.server: New leader elected: payload=Node-00374d90-2147-18c7-9b79-c83df57ae59f +49:27.009 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.115 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:27.116 [INFO] TestAgent: Synced node info +49:27.376 [DEBUG] TestAgent.dns: cname recurse RTT for name: name=www.google.com. rtt=187.994µs +49:27.376 [DEBUG] TestAgent.dns: request served from client: name=google.node.consul. type=ANY class=IN latency=431.65µs client=127.0.0.1:57682 client_network=udp +49:27.376 [INFO] TestAgent: Requesting shutdown +49:27.376 [INFO] TestAgent.server: shutting down server +49:27.376 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.376 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.376 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.376 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.376 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.376 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.376 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.377 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.377 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.377 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:27.377 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:27.377Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.377Z [INFO] agent.router.manager: shutting down +49:27.377 [INFO] TestAgent: consul server down +49:27.377 [INFO] TestAgent: shutdown complete +49:27.377 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24015 network=tcp +49:27.377 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24015 network=udp +49:27.377 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24023 network=tcp +49:27.880 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.880 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup_CNAME (0.97s) +=== CONT TestConfig_Apply_ProxyDefaultsExpose +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=true +2022-12-04T05:49:27.334Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.347Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.349 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:610017a4-fbf0-de92-f2cc-76c31e3e19be Address:127.0.0.1:24069}]" +49:27.349 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-610017a4-fbf0-de92-f2cc-76c31e3e19be.dc1 127.0.0.1 +49:27.350 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-610017a4-fbf0-de92-f2cc-76c31e3e19be 127.0.0.1 +2022-12-04T05:49:27.350Z [INFO] agent.router: Initializing LAN area manager +49:27.350 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24069 [Follower]" leader= +49:27.350 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24082 network=udp +49:27.350 [INFO] TestAgent.server: Adding LAN server: server="Node-610017a4-fbf0-de92-f2cc-76c31e3e19be (Addr: tcp/127.0.0.1:24069) (DC: dc1)" +49:27.350 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-610017a4-fbf0-de92-f2cc-76c31e3e19be.dc1 area=wan +49:27.350 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24082 network=tcp +49:27.351 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24083 network=tcp +49:27.351 [INFO] TestAgent: started state syncer +49:27.351 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24070 network=tcp +49:27.392 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.392 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24069 [Candidate]" term=2 +49:27.392 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.392 [DEBUG] TestAgent.server.raft: vote granted: from=610017a4-fbf0-de92-f2cc-76c31e3e19be term=2 tally=1 +49:27.392 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.392 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24069 [Leader]" +49:27.392 [INFO] TestAgent.server: cluster leadership acquired +49:27.392 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24069 +49:27.393 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.394 [INFO] TestAgent.server: New leader elected: payload=Node-610017a4-fbf0-de92-f2cc-76c31e3e19be +49:27.394 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.394 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.394 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.394 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.394 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.394 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-610017a4-fbf0-de92-f2cc-76c31e3e19be +49:27.394 [INFO] TestAgent.server: member joined, marking health alive: member=Node-610017a4-fbf0-de92-f2cc-76c31e3e19be +49:27.395 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.440 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=ANY class=IN latency=137.111µs client=127.0.0.1:54670 client_network=udp +49:27.440 [DEBUG] TestAgent.dns: request served from client: name=foo.node.consul. type=A class=IN latency=75.799µs client=127.0.0.1:54668 client_network=udp +49:27.440 [DEBUG] TestAgent.dns: request served from client: name=foo.node.dc1.consul. type=ANY class=IN latency=50.026µs client=127.0.0.1:43101 client_network=udp +49:27.441 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=45.893µs client=127.0.0.1:36290 client_network=udp +49:27.441 [INFO] TestAgent: Requesting shutdown +49:27.441 [INFO] TestAgent.server: shutting down server +49:27.441 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.441 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.441 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.441 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.441 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.441 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.441 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.441 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:27.441 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:27.441 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.441 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.441 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:27.441Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.441Z [INFO] agent.router.manager: shutting down +49:27.442 [INFO] TestAgent: consul server down +49:27.442 [INFO] TestAgent: shutdown complete +49:27.442 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24082 network=tcp +49:27.442 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24082 network=udp +49:27.442 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24083 network=tcp +49:27.942 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.942 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_NodeLookup (0.62s) +=== CONT TestConfig_Apply +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=255)_compressed=true +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:27.189Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.201Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.203 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f18fdcf7-e886-b3dc-5c37-4052d68a0f03 Address:127.0.0.1:24108}]" +49:27.203 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24108 [Follower]" leader= +49:27.203 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f18fdcf7-e886-b3dc-5c37-4052d68a0f03.dc1 127.0.0.1 +49:27.204 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f18fdcf7-e886-b3dc-5c37-4052d68a0f03 127.0.0.1 +2022-12-04T05:49:27.204Z [INFO] agent.router: Initializing LAN area manager +49:27.204 [INFO] TestAgent.server: Adding LAN server: server="Node-f18fdcf7-e886-b3dc-5c37-4052d68a0f03 (Addr: tcp/127.0.0.1:24108) (DC: dc1)" +49:27.204 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f18fdcf7-e886-b3dc-5c37-4052d68a0f03.dc1 area=wan +49:27.204 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24057 network=udp +49:27.204 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24057 network=tcp +49:27.205 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24072 network=tcp +49:27.205 [INFO] TestAgent: started state syncer +49:27.205 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24109 network=tcp +49:27.268 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.268 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24108 [Candidate]" term=2 +49:27.268 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.268 [DEBUG] TestAgent.server.raft: vote granted: from=f18fdcf7-e886-b3dc-5c37-4052d68a0f03 term=2 tally=1 +49:27.268 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.268 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24108 [Leader]" +49:27.268 [INFO] TestAgent.server: cluster leadership acquired +49:27.268 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24108 +49:27.269 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.269 [INFO] TestAgent.server: New leader elected: payload=Node-f18fdcf7-e886-b3dc-5c37-4052d68a0f03 +49:27.270 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.270 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.270 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.270 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.270 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.270 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f18fdcf7-e886-b3dc-5c37-4052d68a0f03 +49:27.270 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f18fdcf7-e886-b3dc-5c37-4052d68a0f03 +49:27.271 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.574 [DEBUG] TestAgent.dns: request served from client: name=fOO.node.dc1.consul. type=ANY class=IN latency=231.071µs client=127.0.0.1:54717 client_network=udp +49:27.574 [INFO] TestAgent: Requesting shutdown +49:27.574 [INFO] TestAgent.server: shutting down server +49:27.574 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.574 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.574 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.574 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.574 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.574 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.574 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.574 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:27.575 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.575 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.575 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:27.575 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:27.575Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.575Z [INFO] agent.router.manager: shutting down +49:27.575 [INFO] TestAgent: consul server down +49:27.575 [INFO] TestAgent: shutdown complete +49:27.575 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24057 network=tcp +49:27.575 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24057 network=udp +49:27.575 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24072 network=tcp +49:28.076 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.076 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_CaseInsensitiveNodeLookup (0.90s) +=== CONT TestConfig_Apply_ProxyDefaultsMeshGateway +2022-12-04T05:49:27.271Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:27.271Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.298Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:27.298Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.299 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ad024246-f055-3eaf-7bf7-96843c378da3 Address:127.0.0.1:24080}]" +49:27.299 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24080 [Follower]" leader= +49:27.300 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ad024246-f055-3eaf-7bf7-96843c378da3.dc1 127.0.0.1 +49:27.300 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ad024246-f055-3eaf-7bf7-96843c378da3 127.0.0.1 +2022-12-04T05:49:27.300Z [INFO] agent.router: Initializing LAN area manager +49:27.300 [INFO] TestAgent.server: Adding LAN server: server="Node-ad024246-f055-3eaf-7bf7-96843c378da3 (Addr: tcp/127.0.0.1:24080) (DC: dc1)" +49:27.300 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24110 network=udp +49:27.300 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ad024246-f055-3eaf-7bf7-96843c378da3.dc1 area=wan +49:27.300 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24110 network=tcp +49:27.301 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24111 network=tcp +49:27.301 [INFO] TestAgent: started state syncer +49:27.301 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24081 network=tcp +49:27.340 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.340 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24080 [Candidate]" term=2 +49:27.340 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.340 [DEBUG] TestAgent.server.raft: vote granted: from=ad024246-f055-3eaf-7bf7-96843c378da3 term=2 tally=1 +49:27.340 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.340 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24080 [Leader]" +49:27.340 [INFO] TestAgent.server: cluster leadership acquired +49:27.340 [INFO] TestAgent.server: New leader elected: payload=Node-ad024246-f055-3eaf-7bf7-96843c378da3 +49:27.340 [INFO] TestAgent.server: initializing acls +49:27.341 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:27.341 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:27.341 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:27.341 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:27.341 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:27.341 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:27.341 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ad024246-f055-3eaf-7bf7-96843c378da3 +49:27.341 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ad024246-f055-3eaf-7bf7-96843c378da3.dc1 +49:27.341 [INFO] TestAgent.server: Updating LAN server: server="Node-ad024246-f055-3eaf-7bf7-96843c378da3 (Addr: tcp/127.0.0.1:24080) (DC: dc1)" +49:27.342 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ad024246-f055-3eaf-7bf7-96843c378da3.dc1 area=wan +49:27.342 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.343 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.343 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.343 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.343 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.343 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.343 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ad024246-f055-3eaf-7bf7-96843c378da3 +49:27.343 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ad024246-f055-3eaf-7bf7-96843c378da3 +49:27.344 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.344 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ad024246-f055-3eaf-7bf7-96843c378da3 +49:27.369 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:27.369 [INFO] TestAgent: Synced node info +49:27.673 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ad024246-f055-3eaf-7bf7-96843c378da3 +49:27.673 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ad024246-f055-3eaf-7bf7-96843c378da3 +49:27.673 [INFO] TestAgent: Requesting shutdown +49:27.673 [INFO] TestAgent.server: shutting down server +49:27.673 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.673 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.673 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.673 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:27.673 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:27.673 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.673 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.673 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.673 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.673 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.673 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.673 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:27.673 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:27.673 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:27.673 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +2022-12-04T05:49:27.674Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.674Z [INFO] agent.router.manager: shutting down +49:27.674 [INFO] TestAgent: consul server down +49:27.674 [INFO] TestAgent: shutdown complete +49:27.674 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24110 network=tcp +49:27.674 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24110 network=udp +49:27.674 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24111 network=tcp +49:28.176 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.176 [INFO] TestAgent: Endpoints down +--- PASS: TestCoordinate_Update_ACLDeny (0.91s) + --- PASS: TestCoordinate_Update_ACLDeny/no_token (0.00s) + --- PASS: TestCoordinate_Update_ACLDeny/valid_token (0.00s) +=== CONT TestConfig_Apply_IngressGateway === RUN TestCoordinate_Disabled_Response/0 === RUN TestCoordinate_Disabled_Response/1 === RUN TestCoordinate_Disabled_Response/2 === RUN TestCoordinate_Disabled_Response/3 -2021-10-31T23:13:16.012Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:16.067Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:16.069 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a20dbbcc-0fc4-184b-80af-c8a07068b033 Address:127.0.0.1:16300}]" -13:16.069 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16300 [Follower]" leader= -13:16.079 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a20dbbcc-0fc4-184b-80af-c8a07068b033.dc1 127.0.0.1 -13:16.080 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a20dbbcc-0fc4-184b-80af-c8a07068b033 127.0.0.1 -2021-10-31T23:13:16.080Z [INFO] agent.router: Initializing LAN area manager -13:16.081 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16337 network=udp -13:16.081 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a20dbbcc-0fc4-184b-80af-c8a07068b033.dc1 area=wan -13:16.081 [INFO] TestAgent.server: Adding LAN server: server="Node-a20dbbcc-0fc4-184b-80af-c8a07068b033 (Addr: tcp/127.0.0.1:16300) (DC: dc1)" -13:16.081 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16337 network=tcp -13:16.102 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16338 network=tcp -13:16.102 [INFO] TestAgent: started state syncer -13:16.102 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16301 network=tcp -13:16.138 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:16.138 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16300 [Candidate]" term=2 -13:16.139 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:16.139 [DEBUG] TestAgent.server.raft: vote granted: from=a20dbbcc-0fc4-184b-80af-c8a07068b033 term=2 tally=1 -13:16.139 [INFO] TestAgent.server.raft: election won: tally=1 -13:16.139 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16300 [Leader]" -13:16.139 [INFO] TestAgent.server: cluster leadership acquired -13:16.139 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16300 -13:16.140 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:16.141 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:16.141 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:16.141 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:16.141 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:16.141 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:16.141 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a20dbbcc-0fc4-184b-80af-c8a07068b033 -13:16.141 [INFO] TestAgent.server: New leader elected: payload=Node-a20dbbcc-0fc4-184b-80af-c8a07068b033 -13:16.141 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a20dbbcc-0fc4-184b-80af-c8a07068b033 -13:16.142 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.226 [INFO] TestAgent: Requesting shutdown -13:16.226 [INFO] TestAgent.server: shutting down server -13:16.226 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:16.226 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:16.226 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:16.226 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:16.226 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:16.226 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:16.226 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:16.226 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:16.226 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:16.226 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:16.226 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:16.226 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2021-10-31T23:13:16.226Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:16.226Z [INFO] agent.router.manager: shutting down -13:16.226 [INFO] TestAgent: consul server down -13:16.226 [INFO] TestAgent: shutdown complete -13:16.226 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16337 network=tcp -13:16.227 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16337 network=udp -13:16.227 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16338 network=tcp -13:16.729 [INFO] TestAgent: Waiting for endpoints to shut down -13:16.729 [INFO] TestAgent: Endpoints down ---- PASS: TestDiscoveryChainRead (0.74s) +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:27.473Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.483Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.484 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:599f74c6-7dab-1812-8bc8-36132fb7e0c5 Address:127.0.0.1:24094}]" +49:27.484 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24094 [Follower]" leader= +49:27.485 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-599f74c6-7dab-1812-8bc8-36132fb7e0c5.dc1 127.0.0.1 +49:27.485 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-599f74c6-7dab-1812-8bc8-36132fb7e0c5 127.0.0.1 +2022-12-04T05:49:27.485Z [INFO] agent.router: Initializing LAN area manager +49:27.485 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24071 network=udp +49:27.485 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-599f74c6-7dab-1812-8bc8-36132fb7e0c5.dc1 area=wan +49:27.485 [INFO] TestAgent.server: Adding LAN server: server="Node-599f74c6-7dab-1812-8bc8-36132fb7e0c5 (Addr: tcp/127.0.0.1:24094) (DC: dc1)" +49:27.485 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24071 network=tcp +49:27.487 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24088 network=tcp +49:27.488 [INFO] TestAgent: started state syncer +49:27.488 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24095 network=tcp +49:27.544 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.544 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24094 [Candidate]" term=2 +49:27.544 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.544 [DEBUG] TestAgent.server.raft: vote granted: from=599f74c6-7dab-1812-8bc8-36132fb7e0c5 term=2 tally=1 +49:27.544 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.544 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24094 [Leader]" +49:27.544 [INFO] TestAgent.server: cluster leadership acquired +49:27.545 [INFO] TestAgent.server: New leader elected: payload=Node-599f74c6-7dab-1812-8bc8-36132fb7e0c5 +49:27.545 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24094 +49:27.546 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.548 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.548 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.548 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.548 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.548 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.548 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-599f74c6-7dab-1812-8bc8-36132fb7e0c5 +49:27.548 [INFO] TestAgent.server: member joined, marking health alive: member=Node-599f74c6-7dab-1812-8bc8-36132fb7e0c5 +49:27.549 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.620 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:27.621 [INFO] TestAgent: Synced node info +49:27.621 [DEBUG] TestAgent: Node info in sync +49:27.658 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:27.658 [DEBUG] TestAgent: Node info in sync +49:27.814 [INFO] TestAgent: Requesting shutdown +49:27.814 [INFO] TestAgent.server: shutting down server +49:27.814 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.814 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.814 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.814 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.814 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.814 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.814 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.814 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.814 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.814 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:27.814 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:27.814Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.814Z [INFO] agent.router.manager: shutting down +49:27.814 [INFO] TestAgent: consul server down +49:27.814 [INFO] TestAgent: shutdown complete +49:27.814 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24071 network=tcp +49:27.814 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24071 network=udp +49:27.814 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24088 network=tcp +49:28.315 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.315 [INFO] TestAgent: Endpoints down +--- PASS: TestDiscoveryChainRead (0.86s) --- PASS: TestDiscoveryChainRead/GET:_error_on_no_service_name (0.00s) --- PASS: TestDiscoveryChainRead/GET:_read_default_chain (0.00s) --- PASS: TestDiscoveryChainRead/GET:_read_default_chain;_evaluate_in_dc2 (0.00s) @@ -32574,2377 +38533,7527 @@ --- PASS: TestDiscoveryChainRead/GET:_read_modified_chain (0.03s) --- PASS: TestDiscoveryChainRead/POST:_read_modified_chain_with_overrides_(camel_case) (0.00s) --- PASS: TestDiscoveryChainRead/POST:_read_modified_chain_with_overrides_(snake_case) (0.00s) -=== CONT TestPrettyPrintBare -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:16.198Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:16.227Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:16.229 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5da4241e-35da-30d1-027d-120da1f87fa2 Address:127.0.0.1:16398}]" -13:16.229 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16398 [Follower]" leader= -13:16.238 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5da4241e-35da-30d1-027d-120da1f87fa2.dc1 127.0.0.1 -13:16.239 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5da4241e-35da-30d1-027d-120da1f87fa2 127.0.0.1 -2021-10-31T23:13:16.239Z [INFO] agent.router: Initializing LAN area manager -13:16.239 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16358 network=udp -13:16.239 [INFO] TestAgent.server: Adding LAN server: server="Node-5da4241e-35da-30d1-027d-120da1f87fa2 (Addr: tcp/127.0.0.1:16398) (DC: dc1)" -13:16.239 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5da4241e-35da-30d1-027d-120da1f87fa2.dc1 area=wan -13:16.239 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16358 network=tcp -13:16.240 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16359 network=tcp -13:16.240 [INFO] TestAgent: started state syncer -13:16.240 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16399 network=tcp -13:16.305 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:16.306 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16398 [Candidate]" term=2 -13:16.306 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:16.306 [DEBUG] TestAgent.server.raft: vote granted: from=5da4241e-35da-30d1-027d-120da1f87fa2 term=2 tally=1 -13:16.306 [INFO] TestAgent.server.raft: election won: tally=1 -13:16.306 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16398 [Leader]" -13:16.306 [INFO] TestAgent.server: cluster leadership acquired -13:16.306 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16398 -13:16.307 [INFO] TestAgent.server: New leader elected: payload=Node-5da4241e-35da-30d1-027d-120da1f87fa2 -13:16.307 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:16.314 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:16.314 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:16.314 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:16.314 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:16.314 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:16.314 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5da4241e-35da-30d1-027d-120da1f87fa2 -13:16.314 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5da4241e-35da-30d1-027d-120da1f87fa2 -13:16.315 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.459 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:16.459 [INFO] TestAgent: Synced node info -13:16.459 [DEBUG] TestAgent: Node info in sync -13:16.461 [INFO] TestAgent: Requesting shutdown -13:16.461 [INFO] TestAgent.server: shutting down server -13:16.461 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:16.461 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:16.461 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:16.461 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:16.461 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:16.461 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:16.462 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:16.462 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:16.462 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:16.462 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:16.462 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:16.462Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:16.462Z [INFO] agent.router.manager: shutting down -13:16.462 [INFO] TestAgent: consul server down -13:16.462 [INFO] TestAgent: shutdown complete -13:16.462 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16358 network=tcp -13:16.462 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16358 network=udp -13:16.462 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16359 network=tcp -13:16.965 [INFO] TestAgent: Waiting for endpoints to shut down -13:16.965 [INFO] TestAgent: Endpoints down ---- PASS: TestPProfHandlers_DisableDebugNoACLs (0.80s) -=== CONT TestConnectCARoots_empty -2021-10-31T23:13:16.094Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:16.094Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:16.137Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:16.137Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:16.164 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:95ddba2d-619d-f9cd-e62a-1e28ffc5a348 Address:127.0.0.1:16349}]" -13:16.165 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-95ddba2d-619d-f9cd-e62a-1e28ffc5a348.dc1 127.0.0.1 -13:16.165 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16349 [Follower]" leader= -13:16.166 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-95ddba2d-619d-f9cd-e62a-1e28ffc5a348 127.0.0.1 -2021-10-31T23:13:16.166Z [INFO] agent.router: Initializing LAN area manager -13:16.166 [INFO] TestAgent.server: Adding LAN server: server="Node-95ddba2d-619d-f9cd-e62a-1e28ffc5a348 (Addr: tcp/127.0.0.1:16349) (DC: dc1)" -13:16.166 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16302 network=udp -13:16.166 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-95ddba2d-619d-f9cd-e62a-1e28ffc5a348.dc1 area=wan -13:16.166 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16302 network=tcp -13:16.167 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16303 network=tcp -13:16.167 [INFO] TestAgent: started state syncer -13:16.167 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16350 network=tcp -13:16.222 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:16.236 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:16.236 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16349 [Candidate]" term=2 -13:16.236 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:16.236 [DEBUG] TestAgent.server.raft: vote granted: from=95ddba2d-619d-f9cd-e62a-1e28ffc5a348 term=2 tally=1 -13:16.236 [INFO] TestAgent.server.raft: election won: tally=1 -13:16.236 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16349 [Leader]" -13:16.236 [INFO] TestAgent.server: cluster leadership acquired -13:16.236 [INFO] TestAgent.server: initializing acls -13:16.237 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:16.237 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:16.237 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:16.237 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:16.238 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:16.238 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:16.238 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-95ddba2d-619d-f9cd-e62a-1e28ffc5a348 -13:16.238 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-95ddba2d-619d-f9cd-e62a-1e28ffc5a348.dc1 -13:16.238 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:16.240 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:16.240 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:16.240 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:16.240 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:16.240 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:16.240 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-95ddba2d-619d-f9cd-e62a-1e28ffc5a348 -13:16.240 [INFO] TestAgent.server: member joined, marking health alive: member=Node-95ddba2d-619d-f9cd-e62a-1e28ffc5a348 -13:16.240 [INFO] TestAgent.server: New leader elected: payload=Node-95ddba2d-619d-f9cd-e62a-1e28ffc5a348 -13:16.240 [INFO] TestAgent.server: Updating LAN server: server="Node-95ddba2d-619d-f9cd-e62a-1e28ffc5a348 (Addr: tcp/127.0.0.1:16349) (DC: dc1)" -13:16.240 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-95ddba2d-619d-f9cd-e62a-1e28ffc5a348.dc1 area=wan -13:16.241 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-95ddba2d-619d-f9cd-e62a-1e28ffc5a348 -13:16.241 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.502 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-95ddba2d-619d-f9cd-e62a-1e28ffc5a348 -13:16.502 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-95ddba2d-619d-f9cd-e62a-1e28ffc5a348 -13:16.502 [INFO] TestAgent: Requesting shutdown -13:16.503 [INFO] TestAgent.server: shutting down server -13:16.503 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:16.503 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:16.503 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:16.503 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:16.503 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:16.503 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:16.503 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:16.503 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:16.503 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:16.503 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:16.503 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:16.503 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:16.503 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:16.503 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:16.503 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:16.503 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:16.503Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:16.503Z [INFO] agent.router.manager: shutting down -13:16.503 [INFO] TestAgent: consul server down -13:16.503 [INFO] TestAgent: shutdown complete -13:16.503 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16302 network=tcp -13:16.503 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16302 network=udp -13:16.503 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16303 network=tcp -13:17.013 [INFO] TestAgent: Waiting for endpoints to shut down -13:17.013 [INFO] TestAgent: Endpoints down ---- PASS: TestCoordinate_Update_ACLDeny (0.96s) - --- PASS: TestCoordinate_Update_ACLDeny/no_token (0.00s) - --- PASS: TestCoordinate_Update_ACLDeny/valid_token (0.00s) -2021-10-31T23:13:15.993Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:16.000Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:16.031Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:16.031Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:16.032 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:27cf0d7b-85dd-faf5-33d0-e824609fa096 Address:127.0.0.1:16335}]" -13:16.033 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-27cf0d7b-85dd-faf5-33d0-e824609fa096.dc1 127.0.0.1 -13:16.034 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16335 [Follower]" leader= -13:16.034 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-27cf0d7b-85dd-faf5-33d0-e824609fa096 127.0.0.1 -2021-10-31T23:13:16.034Z [INFO] agent.router: Initializing LAN area manager -13:16.034 [INFO] TestAgent.server: Adding LAN server: server="Node-27cf0d7b-85dd-faf5-33d0-e824609fa096 (Addr: tcp/127.0.0.1:16335) (DC: dc1)" -13:16.034 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-27cf0d7b-85dd-faf5-33d0-e824609fa096.dc1 area=wan -13:16.034 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16309 network=udp -13:16.034 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16309 network=tcp -13:16.035 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16310 network=tcp -13:16.035 [INFO] TestAgent: started state syncer -13:16.035 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16336 network=tcp -13:16.084 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:16.103 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:16.103 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16335 [Candidate]" term=2 -13:16.103 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:16.103 [DEBUG] TestAgent.server.raft: vote granted: from=27cf0d7b-85dd-faf5-33d0-e824609fa096 term=2 tally=1 -13:16.103 [INFO] TestAgent.server.raft: election won: tally=1 -13:16.103 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16335 [Leader]" -13:16.103 [INFO] TestAgent.server: cluster leadership acquired -13:16.103 [INFO] TestAgent.server: initializing acls -13:16.103 [INFO] TestAgent.server: New leader elected: payload=Node-27cf0d7b-85dd-faf5-33d0-e824609fa096 -13:16.103 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:16.103 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:16.104 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:16.105 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:16.105 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:16.105 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:16.105 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-27cf0d7b-85dd-faf5-33d0-e824609fa096 -13:16.105 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-27cf0d7b-85dd-faf5-33d0-e824609fa096.dc1 -13:16.105 [INFO] TestAgent.server: Updating LAN server: server="Node-27cf0d7b-85dd-faf5-33d0-e824609fa096 (Addr: tcp/127.0.0.1:16335) (DC: dc1)" -13:16.106 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-27cf0d7b-85dd-faf5-33d0-e824609fa096.dc1 area=wan -13:16.107 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:16.130 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:16.130 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:16.130 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:16.130 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:16.130 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:16.130 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-27cf0d7b-85dd-faf5-33d0-e824609fa096 -13:16.130 [INFO] TestAgent.server: member joined, marking health alive: member=Node-27cf0d7b-85dd-faf5-33d0-e824609fa096 -13:16.131 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-27cf0d7b-85dd-faf5-33d0-e824609fa096 -13:16.131 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.275 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-27cf0d7b-85dd-faf5-33d0-e824609fa096 -13:16.275 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-27cf0d7b-85dd-faf5-33d0-e824609fa096 -13:16.440 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" -13:16.498 [INFO] TestAgent: Requesting shutdown -13:16.498 [INFO] TestAgent.server: shutting down server -13:16.498 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:16.498 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:16.498 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:16.498 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:16.498 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:16.498 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:16.498 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:16.498 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:16.498 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:16.498 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:16.498 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:16.498 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:16.498 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:16.498 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:16.498 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:16.498Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:16.498Z [INFO] agent.router.manager: shutting down -13:16.499 [INFO] TestAgent: consul server down -13:16.499 [INFO] TestAgent: shutdown complete -13:16.499 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16309 network=tcp -13:16.499 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16309 network=udp -13:16.499 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16310 network=tcp -13:17.014 [INFO] TestAgent: Waiting for endpoints to shut down -13:17.014 [INFO] TestAgent: Endpoints down -=== CONT TestPrettyPrint ---- PASS: TestPProfHandlers_ACLs (1.04s) - --- PASS: TestPProfHandlers_ACLs/case_0_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:200,_token:"master",_endpoint:"/debug/pprof/heap",_nilResponse:false}) (0.13s) - --- PASS: TestPProfHandlers_ACLs/case_1_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"agent",_endpoint:"/debug/pprof/heap",_nilResponse:true}) (0.00s) - --- PASS: TestPProfHandlers_ACLs/case_2_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"agent",_endpoint:"/debug/pprof/",_nilResponse:true}) (0.00s) - --- PASS: TestPProfHandlers_ACLs/case_3_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"",_endpoint:"/debug/pprof/",_nilResponse:true}) (0.01s) - --- PASS: TestPProfHandlers_ACLs/case_4_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:200,_token:"master",_endpoint:"/debug/pprof/heap",_nilResponse:false}) (0.09s) - --- PASS: TestPProfHandlers_ACLs/case_5_(struct_{_code_int;_token_string;_endpoint_string;_nilResponse_bool_}{code:403,_token:"towel",_endpoint:"/debug/pprof/heap",_nilResponse:true}) (0.00s) -=== CONT TestConfig_Apply_ProxyDefaultsExpose -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:16.195Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:16.261Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:16.263 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f689d7fe-745d-8754-3b1f-69b5ca0550dd Address:127.0.0.1:16356}]" -13:16.264 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f689d7fe-745d-8754-3b1f-69b5ca0550dd.dc1 127.0.0.1 -13:16.264 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f689d7fe-745d-8754-3b1f-69b5ca0550dd 127.0.0.1 -2021-10-31T23:13:16.264Z [INFO] agent.router: Initializing LAN area manager -13:16.264 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16351 network=udp -13:16.264 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16356 [Follower]" leader= -13:16.265 [INFO] TestAgent.server: Adding LAN server: server="Node-f689d7fe-745d-8754-3b1f-69b5ca0550dd (Addr: tcp/127.0.0.1:16356) (DC: dc1)" -13:16.265 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f689d7fe-745d-8754-3b1f-69b5ca0550dd.dc1 area=wan -13:16.265 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16351 network=tcp -13:16.274 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16352 network=tcp -13:16.274 [INFO] TestAgent: started state syncer -13:16.274 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16357 network=tcp -13:16.309 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:16.317 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16356 [Candidate]" term=2 -13:16.318 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:16.318 [DEBUG] TestAgent.server.raft: vote granted: from=f689d7fe-745d-8754-3b1f-69b5ca0550dd term=2 tally=1 -13:16.318 [INFO] TestAgent.server.raft: election won: tally=1 -13:16.318 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16356 [Leader]" -13:16.318 [INFO] TestAgent.server: cluster leadership acquired -13:16.318 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16356 -13:16.319 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:16.321 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:16.321 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:16.321 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:16.321 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:16.321 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:16.321 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f689d7fe-745d-8754-3b1f-69b5ca0550dd -13:16.321 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f689d7fe-745d-8754-3b1f-69b5ca0550dd -13:16.322 [INFO] TestAgent.server: New leader elected: payload=Node-f689d7fe-745d-8754-3b1f-69b5ca0550dd -13:16.322 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.674 [INFO] TestAgent: Requesting shutdown -13:16.674 [INFO] TestAgent.server: shutting down server -13:16.674 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:16.674 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:16.674 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:16.674 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:16.674 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:16.674 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:16.674 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:16.674 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:16.674 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:16.674 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:16.674 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:16.674 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:16.674Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:16.674Z [INFO] agent.router.manager: shutting down -13:16.674 [INFO] TestAgent: consul server down -13:16.674 [INFO] TestAgent: shutdown complete -13:16.674 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16351 network=tcp -13:16.674 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16351 network=udp -13:16.674 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16352 network=tcp -13:17.176 [INFO] TestAgent: Waiting for endpoints to shut down -13:17.176 [INFO] TestAgent: Endpoints down ---- PASS: TestCoordinate_Update (1.02s) === CONT TestConfig_Apply_Decoding -2021-10-31T23:13:16.452Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:16.507Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:16.520 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1be57274-772c-1c8c-cfce-cb8bd8e9f185 Address:127.0.0.1:16419}]" -13:16.521 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1be57274-772c-1c8c-cfce-cb8bd8e9f185.dc1 127.0.0.1 -13:16.521 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16419 [Follower]" leader= -13:16.526 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1be57274-772c-1c8c-cfce-cb8bd8e9f185 127.0.0.1 -2021-10-31T23:13:16.526Z [INFO] agent.router: Initializing LAN area manager -13:16.527 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16393 network=udp -13:16.527 [INFO] TestAgent.server: Adding LAN server: server="Node-1be57274-772c-1c8c-cfce-cb8bd8e9f185 (Addr: tcp/127.0.0.1:16419) (DC: dc1)" -13:16.527 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16393 network=tcp -13:16.527 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1be57274-772c-1c8c-cfce-cb8bd8e9f185.dc1 area=wan -13:16.527 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16394 network=tcp -13:16.527 [INFO] TestAgent: started state syncer -13:16.527 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16420 network=tcp -13:16.582 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:16.583 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16419 [Candidate]" term=2 -13:16.583 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:16.583 [DEBUG] TestAgent.server.raft: vote granted: from=1be57274-772c-1c8c-cfce-cb8bd8e9f185 term=2 tally=1 -13:16.583 [INFO] TestAgent.server.raft: election won: tally=1 -13:16.583 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16419 [Leader]" -13:16.585 [INFO] TestAgent.server: cluster leadership acquired -13:16.585 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16419 -13:16.585 [INFO] TestAgent.server: New leader elected: payload=Node-1be57274-772c-1c8c-cfce-cb8bd8e9f185 -13:16.594 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:16.596 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:16.596 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:16.596 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:16.596 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:16.596 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:16.596 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1be57274-772c-1c8c-cfce-cb8bd8e9f185 -13:16.596 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1be57274-772c-1c8c-cfce-cb8bd8e9f185 -13:16.597 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.696 [INFO] TestAgent: Requesting shutdown -13:16.697 [INFO] TestAgent.server: shutting down server -13:16.697 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:16.697 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:16.697 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:16.697 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:16.697 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:16.697 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:16.697 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:16.697 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:16.697 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:16.697 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:16.697 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:16.697 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:16.697Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:16.697Z [INFO] agent.router.manager: shutting down -13:16.697 [INFO] TestAgent: consul server down -13:16.697 [INFO] TestAgent: shutdown complete -13:16.697 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16393 network=tcp -13:16.697 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16393 network=udp -13:16.697 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16394 network=tcp -13:17.198 [INFO] TestAgent: Waiting for endpoints to shut down -13:17.198 [INFO] TestAgent: Endpoints down ---- PASS: TestCoordinate_Disabled_Response (0.77s) +2022-12-04T05:49:27.776Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.786Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.787 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a7b97801-c3e8-e305-ead9-f64375393684 Address:127.0.0.1:24150}]" +49:27.787 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24150 [Follower]" leader= +49:27.788 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a7b97801-c3e8-e305-ead9-f64375393684.dc1 127.0.0.1 +49:27.788 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a7b97801-c3e8-e305-ead9-f64375393684 127.0.0.1 +2022-12-04T05:49:27.788Z [INFO] agent.router: Initializing LAN area manager +49:27.789 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a7b97801-c3e8-e305-ead9-f64375393684.dc1 area=wan +49:27.789 [INFO] TestAgent.server: Adding LAN server: server="Node-a7b97801-c3e8-e305-ead9-f64375393684 (Addr: tcp/127.0.0.1:24150) (DC: dc1)" +49:27.789 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24138 network=tcp +49:27.789 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24138 network=udp +49:27.789 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24139 network=tcp +49:27.789 [INFO] TestAgent: started state syncer +49:27.789 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24151 network=tcp +49:27.836 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.836 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24150 [Candidate]" term=2 +49:27.836 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.836 [DEBUG] TestAgent.server.raft: vote granted: from=a7b97801-c3e8-e305-ead9-f64375393684 term=2 tally=1 +49:27.836 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.836 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24150 [Leader]" +49:27.836 [INFO] TestAgent.server: cluster leadership acquired +49:27.837 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24150 +49:27.838 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.840 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.840 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.840 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.840 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.840 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.840 [INFO] TestAgent.server: New leader elected: payload=Node-a7b97801-c3e8-e305-ead9-f64375393684 +49:27.840 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a7b97801-c3e8-e305-ead9-f64375393684 +49:27.840 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a7b97801-c3e8-e305-ead9-f64375393684 +49:27.840 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.851 [DEBUG] connect.ca.consul: consul CA provider configured: id=3f:15:30:96:d4:69:c0:6b:1f:45:32:9b:f4:63:1e:5f:b3:e5:d6:be:6f:9d:61:d2:fc:e4:dd:5c:b2:e7:61:c1 is_primary=true +49:27.854 [INFO] TestAgent.server.connect: CA rotated to new root under provider: provider=consul +49:27.855 [INFO] TestAgent: Requesting shutdown +49:27.855 [INFO] TestAgent.server: shutting down server +49:27.855 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.855 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.855 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.855 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.855 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.855 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.855 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.855 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:27.855 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.855 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.855 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:27.855 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:27.855Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.855Z [INFO] agent.router.manager: shutting down +49:27.855 [INFO] TestAgent: consul server down +49:27.855 [INFO] TestAgent: shutdown complete +49:27.855 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24138 network=tcp +49:27.855 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24138 network=udp +49:27.855 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24139 network=tcp +49:28.356 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.356 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectCARoots_list (0.61s) +=== CONT TestConfig_Apply_TerminatingGateway +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=255)_compressed=false +2022-12-04T05:49:27.667Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.676Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.678 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1bfd1aed-6460-4f52-accc-5bf24a23477f Address:127.0.0.1:24122}]" +49:27.678 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24122 [Follower]" leader= +49:27.678 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1bfd1aed-6460-4f52-accc-5bf24a23477f.dc1 127.0.0.1 +49:27.679 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1bfd1aed-6460-4f52-accc-5bf24a23477f 127.0.0.1 +2022-12-04T05:49:27.679Z [INFO] agent.router: Initializing LAN area manager +49:27.679 [INFO] TestAgent.server: Adding LAN server: server="Node-1bfd1aed-6460-4f52-accc-5bf24a23477f (Addr: tcp/127.0.0.1:24122) (DC: dc1)" +49:27.679 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1bfd1aed-6460-4f52-accc-5bf24a23477f.dc1 area=wan +49:27.679 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24117 network=udp +49:27.679 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24117 network=tcp +49:27.679 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24118 network=tcp +49:27.679 [INFO] TestAgent: started state syncer +49:27.679 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24123 network=tcp +49:27.741 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.741 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24122 [Candidate]" term=2 +49:27.741 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.741 [DEBUG] TestAgent.server.raft: vote granted: from=1bfd1aed-6460-4f52-accc-5bf24a23477f term=2 tally=1 +49:27.741 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.741 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24122 [Leader]" +49:27.741 [INFO] TestAgent.server: cluster leadership acquired +49:27.742 [INFO] TestAgent.server: New leader elected: payload=Node-1bfd1aed-6460-4f52-accc-5bf24a23477f +49:27.742 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24122 +49:27.742 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.744 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.744 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.744 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.744 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.744 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.744 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1bfd1aed-6460-4f52-accc-5bf24a23477f +49:27.744 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1bfd1aed-6460-4f52-accc-5bf24a23477f +49:27.744 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.985 [INFO] TestAgent.server.connect: CA provider config updated +49:27.985 [INFO] TestAgent: Requesting shutdown +49:27.985 [INFO] TestAgent.server: shutting down server +49:27.985 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.985 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.985 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.985 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.985 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.985 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.985 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.985 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:27.985 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:27.985 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.985 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.985 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:27.985Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.985Z [INFO] agent.router.manager: shutting down +49:27.985 [INFO] TestAgent: consul server down +49:27.985 [INFO] TestAgent: shutdown complete +49:27.985 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24117 network=tcp +49:27.986 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24117 network=udp +49:27.986 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24118 network=tcp +49:28.487 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.487 [INFO] TestAgent: Endpoints down +=== RUN TestConnectCAConfig/basic_with_IntermediateCertTTL +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:27.781Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.806Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.809 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:40e6b274-cdf8-9d61-7834-4f75593b0c9f Address:127.0.0.1:24164}]" +49:27.810 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24164 [Follower]" leader= +49:27.810 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-40e6b274-cdf8-9d61-7834-4f75593b0c9f.dc1 127.0.0.1 +49:27.810 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-40e6b274-cdf8-9d61-7834-4f75593b0c9f 127.0.0.1 +2022-12-04T05:49:27.810Z [INFO] agent.router: Initializing LAN area manager +49:27.810 [INFO] TestAgent.server: Adding LAN server: server="Node-40e6b274-cdf8-9d61-7834-4f75593b0c9f (Addr: tcp/127.0.0.1:24164) (DC: dc1)" +49:27.810 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-40e6b274-cdf8-9d61-7834-4f75593b0c9f.dc1 area=wan +49:27.811 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24152 network=udp +49:27.811 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24152 network=tcp +49:27.811 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24153 network=tcp +49:27.811 [INFO] TestAgent: started state syncer +49:27.811 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24165 network=tcp +49:27.850 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.850 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24164 [Candidate]" term=2 +49:27.850 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.850 [DEBUG] TestAgent.server.raft: vote granted: from=40e6b274-cdf8-9d61-7834-4f75593b0c9f term=2 tally=1 +49:27.850 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.850 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24164 [Leader]" +49:27.850 [INFO] TestAgent.server: cluster leadership acquired +49:27.850 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24164 +49:27.850 [INFO] TestAgent.server: New leader elected: payload=Node-40e6b274-cdf8-9d61-7834-4f75593b0c9f +49:27.853 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.856 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.856 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.856 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.856 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.856 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.856 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-40e6b274-cdf8-9d61-7834-4f75593b0c9f +49:27.856 [INFO] TestAgent.server: member joined, marking health alive: member=Node-40e6b274-cdf8-9d61-7834-4f75593b0c9f +49:27.856 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.909 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:27.910 [INFO] TestAgent: Synced node info +49:28.105 [INFO] TestAgent: Requesting shutdown +49:28.105 [INFO] TestAgent.server: shutting down server +49:28.105 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.105 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.105 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.105 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.105 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.105 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.105 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.105 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.105 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.105 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.105 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:28.106Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.106Z [INFO] agent.router.manager: shutting down +49:28.106 [INFO] TestAgent: consul server down +49:28.106 [INFO] TestAgent: shutdown complete +49:28.106 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24152 network=tcp +49:28.106 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24152 network=udp +49:28.106 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24153 network=tcp +49:28.608 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.608 [INFO] TestAgent: Endpoints down +--- PASS: TestCoordinate_Datacenters (0.85s) +=== CONT TestConfig_Apply_CAS +2022-12-04T05:49:27.550Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.565Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.566 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7c1ed7fa-507f-993e-cefe-293b9d475106 Address:127.0.0.1:24101}]" +49:27.566 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24101 [Follower]" leader= +49:27.567 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7c1ed7fa-507f-993e-cefe-293b9d475106.dc1 127.0.0.1 +49:27.567 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7c1ed7fa-507f-993e-cefe-293b9d475106 127.0.0.1 +2022-12-04T05:49:27.567Z [INFO] agent.router: Initializing LAN area manager +49:27.567 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24096 network=udp +49:27.567 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7c1ed7fa-507f-993e-cefe-293b9d475106.dc1 area=wan +49:27.567 [INFO] TestAgent.server: Adding LAN server: server="Node-7c1ed7fa-507f-993e-cefe-293b9d475106 (Addr: tcp/127.0.0.1:24101) (DC: dc1)" +49:27.567 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24096 network=tcp +49:27.568 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24097 network=tcp +49:27.568 [INFO] TestAgent: started state syncer +49:27.568 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24102 network=tcp +49:27.634 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.634 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24101 [Candidate]" term=2 +49:27.635 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.635 [DEBUG] TestAgent.server.raft: vote granted: from=7c1ed7fa-507f-993e-cefe-293b9d475106 term=2 tally=1 +49:27.635 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.635 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24101 [Leader]" +49:27.635 [INFO] TestAgent.server: cluster leadership acquired +49:27.635 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24101 +49:27.635 [INFO] TestAgent.server: New leader elected: payload=Node-7c1ed7fa-507f-993e-cefe-293b9d475106 +49:27.637 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.639 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.639 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.639 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.639 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.639 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.639 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7c1ed7fa-507f-993e-cefe-293b9d475106 +49:27.639 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7c1ed7fa-507f-993e-cefe-293b9d475106 +49:27.640 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.892 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:27.892 [INFO] TestAgent: Synced node info +49:28.112 [INFO] TestAgent: Requesting shutdown +49:28.112 [INFO] TestAgent.server: shutting down server +49:28.112 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.112 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.112 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.112 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.113 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.113 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.113 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.113 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.113 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.113 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.113 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:28.113Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.113Z [INFO] agent.router.manager: shutting down +49:28.113 [INFO] TestAgent: consul server down +49:28.113 [INFO] TestAgent: shutdown complete +49:28.113 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24096 network=tcp +49:28.113 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24096 network=udp +49:28.113 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24097 network=tcp +49:28.613 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.613 [INFO] TestAgent: Endpoints down +--- PASS: TestCoordinate_Nodes (1.08s) +=== CONT TestConfig_Delete +2022-12-04T05:49:27.952Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.962Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.963 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:110ff3bf-d06a-431d-fa6c-70ac1c0235da Address:127.0.0.1:24206}]" +49:27.963 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24206 [Follower]" leader= +49:27.964 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-110ff3bf-d06a-431d-fa6c-70ac1c0235da.dc1 127.0.0.1 +49:27.964 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-110ff3bf-d06a-431d-fa6c-70ac1c0235da 127.0.0.1 +2022-12-04T05:49:27.964Z [INFO] agent.router: Initializing LAN area manager +49:27.964 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24222 network=udp +49:27.964 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-110ff3bf-d06a-431d-fa6c-70ac1c0235da.dc1 area=wan +49:27.965 [INFO] TestAgent.server: Adding LAN server: server="Node-110ff3bf-d06a-431d-fa6c-70ac1c0235da (Addr: tcp/127.0.0.1:24206) (DC: dc1)" +49:27.965 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24222 network=tcp +49:27.965 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24223 network=tcp +49:27.965 [INFO] TestAgent: started state syncer +49:27.965 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24207 network=tcp +49:28.027 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.027 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24206 [Candidate]" term=2 +49:28.027 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.027 [DEBUG] TestAgent.server.raft: vote granted: from=110ff3bf-d06a-431d-fa6c-70ac1c0235da term=2 tally=1 +49:28.027 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.027 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24206 [Leader]" +49:28.028 [INFO] TestAgent.server: cluster leadership acquired +49:28.028 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24206 +49:28.028 [INFO] TestAgent.server: New leader elected: payload=Node-110ff3bf-d06a-431d-fa6c-70ac1c0235da +49:28.029 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.030 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.030 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.030 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.030 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.030 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.030 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-110ff3bf-d06a-431d-fa6c-70ac1c0235da +49:28.030 [INFO] TestAgent.server: member joined, marking health alive: member=Node-110ff3bf-d06a-431d-fa6c-70ac1c0235da +49:28.030 [INFO] TestAgent.server: federation state anti-entropy synced +49:28.042 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:28.042 [INFO] TestAgent: Synced node info +49:28.115 [INFO] TestAgent: Requesting shutdown +49:28.115 [INFO] TestAgent.server: shutting down server +49:28.115 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.115 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.115 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.115 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.115 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.115 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.115 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.115 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.115 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.115 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:28.115 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +2022-12-04T05:49:28.115Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.115Z [INFO] agent.router.manager: shutting down +49:28.115 [INFO] TestAgent: consul server down +49:28.115 [INFO] TestAgent: shutdown complete +49:28.115 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24222 network=tcp +49:28.115 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24222 network=udp +49:28.115 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24223 network=tcp +49:28.616 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.616 [INFO] TestAgent: Endpoints down +--- PASS: TestConfig_Apply (0.67s) +=== CONT TestCatalogNodeServices_WanTranslation +2022-12-04T05:49:27.900Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.909Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.911 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2cbb006c-a0d3-c27c-4afb-413742f2e27d Address:127.0.0.1:24220}]" +49:27.911 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24220 [Follower]" leader= +49:27.911 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2cbb006c-a0d3-c27c-4afb-413742f2e27d.dc1 127.0.0.1 +49:27.912 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2cbb006c-a0d3-c27c-4afb-413742f2e27d 127.0.0.1 +2022-12-04T05:49:27.912Z [INFO] agent.router: Initializing LAN area manager +49:27.912 [INFO] TestAgent.server: Adding LAN server: server="Node-2cbb006c-a0d3-c27c-4afb-413742f2e27d (Addr: tcp/127.0.0.1:24220) (DC: dc1)" +49:27.912 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2cbb006c-a0d3-c27c-4afb-413742f2e27d.dc1 area=wan +49:27.912 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24187 network=udp +49:27.912 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24187 network=tcp +49:27.913 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24188 network=tcp +49:27.913 [INFO] TestAgent: started state syncer +49:27.913 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24221 network=tcp +49:27.964 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.964 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24220 [Candidate]" term=2 +49:27.964 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.964 [DEBUG] TestAgent.server.raft: vote granted: from=2cbb006c-a0d3-c27c-4afb-413742f2e27d term=2 tally=1 +49:27.964 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.964 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24220 [Leader]" +49:27.964 [INFO] TestAgent.server: cluster leadership acquired +49:27.964 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24220 +49:27.965 [INFO] TestAgent.server: New leader elected: payload=Node-2cbb006c-a0d3-c27c-4afb-413742f2e27d +49:27.965 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.966 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.966 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.966 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.966 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.966 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.966 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2cbb006c-a0d3-c27c-4afb-413742f2e27d +49:27.966 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2cbb006c-a0d3-c27c-4afb-413742f2e27d +49:27.967 [INFO] TestAgent.server: federation state anti-entropy synced +49:28.107 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:28.108 [INFO] TestAgent: Synced node info +49:28.115 [INFO] TestAgent: Requesting shutdown +49:28.115 [INFO] TestAgent.server: shutting down server +49:28.115 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.115 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.115 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.115 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.115 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.115 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.115 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.115 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.115 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.115 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.115 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:28.115Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.115Z [INFO] agent.router.manager: shutting down +49:28.116 [INFO] TestAgent: consul server down +49:28.116 [INFO] TestAgent: shutdown complete +49:28.116 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24187 network=tcp +49:28.116 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24187 network=udp +49:28.116 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24188 network=tcp +49:28.616 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.616 [INFO] TestAgent: Endpoints down +--- PASS: TestConfig_Apply_ProxyDefaultsExpose (0.74s) +=== CONT TestCatalogNodeServices_ConnectProxy +2022-12-04T05:49:27.681Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.691Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.692 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4ff6a5f2-4306-a621-c02a-562b5e5da316 Address:127.0.0.1:24136}]" +49:27.692 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24136 [Follower]" leader= +49:27.692 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4ff6a5f2-4306-a621-c02a-562b5e5da316.dc1 127.0.0.1 +49:27.693 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4ff6a5f2-4306-a621-c02a-562b5e5da316 127.0.0.1 +2022-12-04T05:49:27.693Z [INFO] agent.router: Initializing LAN area manager +49:27.693 [INFO] TestAgent.server: Adding LAN server: server="Node-4ff6a5f2-4306-a621-c02a-562b5e5da316 (Addr: tcp/127.0.0.1:24136) (DC: dc1)" +49:27.693 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4ff6a5f2-4306-a621-c02a-562b5e5da316.dc1 area=wan +49:27.693 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24124 network=udp +49:27.693 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24124 network=tcp +49:27.694 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24125 network=tcp +49:27.694 [INFO] TestAgent: started state syncer +49:27.694 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24137 network=tcp +49:27.767 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.767 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24136 [Candidate]" term=2 +49:27.767 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.767 [DEBUG] TestAgent.server.raft: vote granted: from=4ff6a5f2-4306-a621-c02a-562b5e5da316 term=2 tally=1 +49:27.767 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.767 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24136 [Leader]" +49:27.767 [INFO] TestAgent.server: cluster leadership acquired +49:27.767 [INFO] TestAgent.server: New leader elected: payload=Node-4ff6a5f2-4306-a621-c02a-562b5e5da316 +49:27.767 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24136 +49:27.773 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.775 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.775 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.775 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.775 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.775 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.775 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4ff6a5f2-4306-a621-c02a-562b5e5da316 +49:27.775 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4ff6a5f2-4306-a621-c02a-562b5e5da316 +49:27.776 [INFO] TestAgent.server: federation state anti-entropy synced +49:28.054 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:28.054 [INFO] TestAgent: Synced node info +49:28.151 [INFO] TestAgent: Requesting shutdown +49:28.151 [INFO] TestAgent.server: shutting down server +49:28.151 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.151 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.151 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.151 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.151 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.151 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.151 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.151 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.151 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.151 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.151 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:28.151Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.151Z [INFO] agent.router.manager: shutting down +49:28.152 [INFO] TestAgent: consul server down +49:28.152 [INFO] TestAgent: shutdown complete +49:28.152 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24124 network=tcp +49:28.152 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24124 network=udp +49:28.152 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24125 network=tcp +49:28.652 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.652 [INFO] TestAgent: Endpoints down +--- PASS: TestCoordinate_Node (0.98s) +=== CONT TestConfig_Get +2022-12-04T05:49:27.864Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.876Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.883 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a7ca5f5c-e8ce-3297-65eb-ce8f934dc79f Address:127.0.0.1:24185}]" +49:27.884 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24185 [Follower]" leader= +49:27.884 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a7ca5f5c-e8ce-3297-65eb-ce8f934dc79f.dc1 127.0.0.1 +49:27.885 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a7ca5f5c-e8ce-3297-65eb-ce8f934dc79f 127.0.0.1 +2022-12-04T05:49:27.885Z [INFO] agent.router: Initializing LAN area manager +49:27.885 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24180 network=udp +49:27.886 [INFO] TestAgent.server: Adding LAN server: server="Node-a7ca5f5c-e8ce-3297-65eb-ce8f934dc79f (Addr: tcp/127.0.0.1:24185) (DC: dc1)" +49:27.886 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a7ca5f5c-e8ce-3297-65eb-ce8f934dc79f.dc1 area=wan +49:27.886 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24180 network=tcp +49:27.887 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24181 network=tcp +49:27.887 [INFO] TestAgent: started state syncer +49:27.887 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24186 network=tcp +49:27.924 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.924 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24185 [Candidate]" term=2 +49:27.924 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.924 [DEBUG] TestAgent.server.raft: vote granted: from=a7ca5f5c-e8ce-3297-65eb-ce8f934dc79f term=2 tally=1 +49:27.924 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.924 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24185 [Leader]" +49:27.924 [INFO] TestAgent.server: cluster leadership acquired +49:27.924 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24185 +49:27.925 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.926 [INFO] TestAgent.server: New leader elected: payload=Node-a7ca5f5c-e8ce-3297-65eb-ce8f934dc79f +49:27.926 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.926 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.926 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.926 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.926 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.926 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a7ca5f5c-e8ce-3297-65eb-ce8f934dc79f +49:27.926 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a7ca5f5c-e8ce-3297-65eb-ce8f934dc79f +49:27.927 [INFO] TestAgent.server: federation state anti-entropy synced +49:28.114 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:28.114 [INFO] TestAgent: Synced node info +49:28.201 [INFO] TestAgent: Requesting shutdown +49:28.201 [INFO] TestAgent.server: shutting down server +49:28.201 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.201 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.201 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.201 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.201 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.201 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.201 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.202 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.202 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.203 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.203 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:28.203Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.203Z [INFO] agent.router.manager: shutting down +49:28.204 [INFO] TestAgent: consul server down +49:28.204 [INFO] TestAgent: shutdown complete +49:28.204 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24180 network=tcp +49:28.204 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24180 network=udp +49:28.204 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24181 network=tcp +49:28.705 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.705 [INFO] TestAgent: Endpoints down +--- PASS: TestCoordinate_Disabled_Response (0.85s) --- PASS: TestCoordinate_Disabled_Response/0 (0.00s) --- PASS: TestCoordinate_Disabled_Response/1 (0.00s) --- PASS: TestCoordinate_Disabled_Response/2 (0.00s) --- PASS: TestCoordinate_Disabled_Response/3 (0.00s) -=== CONT TestConfig_Apply_CAS -2021-10-31T23:13:16.405Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:16.449Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:16.458 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f46b5fd1-221a-1ebc-3efa-7c589c726ecb Address:127.0.0.1:16377}]" -13:16.459 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f46b5fd1-221a-1ebc-3efa-7c589c726ecb.dc1 127.0.0.1 -13:16.460 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f46b5fd1-221a-1ebc-3efa-7c589c726ecb 127.0.0.1 -2021-10-31T23:13:16.460Z [INFO] agent.router: Initializing LAN area manager -13:16.460 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16428 network=udp -13:16.460 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16377 [Follower]" leader= -13:16.460 [INFO] TestAgent.server: Adding LAN server: server="Node-f46b5fd1-221a-1ebc-3efa-7c589c726ecb (Addr: tcp/127.0.0.1:16377) (DC: dc1)" -13:16.460 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f46b5fd1-221a-1ebc-3efa-7c589c726ecb.dc1 area=wan -13:16.461 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16428 network=tcp -13:16.461 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16429 network=tcp -13:16.461 [INFO] TestAgent: started state syncer -13:16.461 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16378 network=tcp -13:16.504 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:16.504 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16377 [Candidate]" term=2 -13:16.504 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:16.504 [DEBUG] TestAgent.server.raft: vote granted: from=f46b5fd1-221a-1ebc-3efa-7c589c726ecb term=2 tally=1 -13:16.504 [INFO] TestAgent.server.raft: election won: tally=1 -13:16.504 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16377 [Leader]" -13:16.504 [INFO] TestAgent.server: cluster leadership acquired -13:16.504 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16377 -13:16.505 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:16.507 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:16.507 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:16.507 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:16.507 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:16.507 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:16.507 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f46b5fd1-221a-1ebc-3efa-7c589c726ecb -13:16.507 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f46b5fd1-221a-1ebc-3efa-7c589c726ecb -13:16.507 [INFO] TestAgent.server: New leader elected: payload=Node-f46b5fd1-221a-1ebc-3efa-7c589c726ecb -13:16.508 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.809 [INFO] TestAgent: Requesting shutdown -13:16.810 [INFO] TestAgent.server: shutting down server -13:16.810 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:16.810 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:16.810 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:16.810 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:16.810 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:16.810 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:16.810 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:16.810 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:16.810 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:16.810 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:16.810 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:16.810 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:16.810Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:16.810Z [INFO] agent.router.manager: shutting down -13:16.810 [INFO] TestAgent: consul server down -13:16.810 [INFO] TestAgent: shutdown complete -13:16.810 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16428 network=tcp -13:16.810 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16428 network=udp -13:16.810 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16429 network=tcp -13:17.314 [INFO] TestAgent: Waiting for endpoints to shut down -13:17.314 [INFO] TestAgent: Endpoints down ---- PASS: TestCoordinate_Datacenters (0.98s) -=== CONT TestConfig_Apply_ProxyDefaultsMeshGateway -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:16.225Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:16.268Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:16.278 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ca61a6ba-0de0-5d6b-4011-9f1e0c75a8f6 Address:127.0.0.1:16405}]" -13:16.278 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16405 [Follower]" leader= -13:16.279 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ca61a6ba-0de0-5d6b-4011-9f1e0c75a8f6.dc1 127.0.0.1 -13:16.280 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ca61a6ba-0de0-5d6b-4011-9f1e0c75a8f6 127.0.0.1 -2021-10-31T23:13:16.280Z [INFO] agent.router: Initializing LAN area manager -13:16.280 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16400 network=udp -13:16.280 [INFO] TestAgent.server: Adding LAN server: server="Node-ca61a6ba-0de0-5d6b-4011-9f1e0c75a8f6 (Addr: tcp/127.0.0.1:16405) (DC: dc1)" -13:16.280 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ca61a6ba-0de0-5d6b-4011-9f1e0c75a8f6.dc1 area=wan -13:16.280 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16400 network=tcp -13:16.281 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16401 network=tcp -13:16.281 [INFO] TestAgent: started state syncer -13:16.281 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16406 network=tcp -13:16.345 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:16.345 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16405 [Candidate]" term=2 -13:16.346 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:16.346 [DEBUG] TestAgent.server.raft: vote granted: from=ca61a6ba-0de0-5d6b-4011-9f1e0c75a8f6 term=2 tally=1 -13:16.346 [INFO] TestAgent.server.raft: election won: tally=1 -13:16.346 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16405 [Leader]" -13:16.346 [INFO] TestAgent.server: cluster leadership acquired -13:16.346 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16405 -13:16.347 [INFO] TestAgent.server: New leader elected: payload=Node-ca61a6ba-0de0-5d6b-4011-9f1e0c75a8f6 -13:16.347 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:16.349 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:16.349 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:16.349 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:16.349 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:16.349 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:16.349 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ca61a6ba-0de0-5d6b-4011-9f1e0c75a8f6 -13:16.349 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ca61a6ba-0de0-5d6b-4011-9f1e0c75a8f6 -13:16.350 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.713 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:16.714 [INFO] TestAgent: Synced node info -13:16.714 [DEBUG] TestAgent: Node info in sync -13:16.874 [INFO] TestAgent: Requesting shutdown -13:16.874 [INFO] TestAgent.server: shutting down server -13:16.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:16.874 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:16.874 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:16.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:16.874 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:16.874 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:16.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:16.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:16.874 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:16.874 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:16.874 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:16.874Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:16.874Z [INFO] agent.router.manager: shutting down -13:16.875 [INFO] TestAgent: consul server down -13:16.875 [INFO] TestAgent: shutdown complete -13:16.875 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16400 network=tcp -13:16.875 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16400 network=udp -13:16.875 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16401 network=tcp -13:17.378 [INFO] TestAgent: Waiting for endpoints to shut down -13:17.378 [INFO] TestAgent: Endpoints down ---- PASS: TestCoordinate_Nodes (1.21s) -=== CONT TestAgent_RegisterCheck -2021-10-31T23:13:16.489Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:16.507Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:16.509 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a250c8ae-f7a7-705a-a63c-5c62960b0a1d Address:127.0.0.1:16440}]" -13:16.509 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16440 [Follower]" leader= -13:16.517 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a250c8ae-f7a7-705a-a63c-5c62960b0a1d.dc1 127.0.0.1 -13:16.518 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a250c8ae-f7a7-705a-a63c-5c62960b0a1d 127.0.0.1 -2021-10-31T23:13:16.518Z [INFO] agent.router: Initializing LAN area manager -13:16.519 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16421 network=udp -13:16.519 [INFO] TestAgent.server: Adding LAN server: server="Node-a250c8ae-f7a7-705a-a63c-5c62960b0a1d (Addr: tcp/127.0.0.1:16440) (DC: dc1)" -13:16.519 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a250c8ae-f7a7-705a-a63c-5c62960b0a1d.dc1 area=wan -13:16.519 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16421 network=tcp -13:16.520 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16422 network=tcp -13:16.520 [INFO] TestAgent: started state syncer -13:16.520 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16441 network=tcp -13:16.544 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:16.544 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16440 [Candidate]" term=2 -13:16.545 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:16.545 [DEBUG] TestAgent.server.raft: vote granted: from=a250c8ae-f7a7-705a-a63c-5c62960b0a1d term=2 tally=1 -13:16.545 [INFO] TestAgent.server.raft: election won: tally=1 -13:16.545 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16440 [Leader]" -13:16.546 [INFO] TestAgent.server: cluster leadership acquired -13:16.547 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16440 -13:16.548 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:16.549 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:16.549 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:16.549 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:16.549 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:16.549 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:16.549 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a250c8ae-f7a7-705a-a63c-5c62960b0a1d -13:16.549 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a250c8ae-f7a7-705a-a63c-5c62960b0a1d -13:16.550 [INFO] TestAgent.server: New leader elected: payload=Node-a250c8ae-f7a7-705a-a63c-5c62960b0a1d -13:16.552 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.597 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:16.598 [INFO] TestAgent: Synced node info -13:16.913 [INFO] TestAgent: Requesting shutdown -13:16.913 [INFO] TestAgent.server: shutting down server -13:16.913 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:16.913 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:16.913 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:16.913 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:16.913 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:16.913 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:16.913 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:16.913 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:16.913 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:16.913 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:16.913 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:16.914Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:16.914Z [INFO] agent.router.manager: shutting down -13:16.914 [INFO] TestAgent: consul server down -13:16.914 [INFO] TestAgent: shutdown complete -13:16.914 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16421 network=tcp -13:16.914 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16421 network=udp -13:16.914 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16422 network=tcp -13:17.414 [INFO] TestAgent: Waiting for endpoints to shut down -13:17.414 [INFO] TestAgent: Endpoints down ---- PASS: TestParseSource (0.95s) -=== CONT TestAgentConnectAuthorize_badBody -2021-10-31T23:13:16.775Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:16.800Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:16.810 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3e1e8396-72ec-c0cc-11d5-1447ef815943 Address:127.0.0.1:16454}]" -13:16.810 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16454 [Follower]" leader= -13:16.811 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3e1e8396-72ec-c0cc-11d5-1447ef815943.dc1 127.0.0.1 -13:16.811 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3e1e8396-72ec-c0cc-11d5-1447ef815943 127.0.0.1 -2021-10-31T23:13:16.811Z [INFO] agent.router: Initializing LAN area manager -13:16.812 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16435 network=udp -13:16.812 [INFO] TestAgent.server: Adding LAN server: server="Node-3e1e8396-72ec-c0cc-11d5-1447ef815943 (Addr: tcp/127.0.0.1:16454) (DC: dc1)" -13:16.812 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3e1e8396-72ec-c0cc-11d5-1447ef815943.dc1 area=wan -13:16.812 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16435 network=tcp -13:16.813 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16436 network=tcp -13:16.813 [INFO] TestAgent: started state syncer -13:16.813 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16455 network=tcp -13:16.873 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:16.873 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16454 [Candidate]" term=2 -13:16.874 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:16.874 [DEBUG] TestAgent.server.raft: vote granted: from=3e1e8396-72ec-c0cc-11d5-1447ef815943 term=2 tally=1 -13:16.874 [INFO] TestAgent.server.raft: election won: tally=1 -13:16.874 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16454 [Leader]" -13:16.874 [INFO] TestAgent.server: cluster leadership acquired -13:16.874 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16454 -13:16.874 [INFO] TestAgent.server: New leader elected: payload=Node-3e1e8396-72ec-c0cc-11d5-1447ef815943 -13:16.875 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:16.876 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:16.876 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:16.876 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:16.876 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:16.876 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:16.876 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3e1e8396-72ec-c0cc-11d5-1447ef815943 -13:16.876 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3e1e8396-72ec-c0cc-11d5-1447ef815943 -13:16.877 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.909 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:16.910 [INFO] TestAgent: Synced node info -13:16.910 [DEBUG] TestAgent: Node info in sync -13:16.926 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/key?pretty from= latency=62.874µs -13:16.926 [INFO] TestAgent: Requesting shutdown -13:16.926 [INFO] TestAgent.server: shutting down server -13:16.926 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:16.926 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:16.926 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:16.926 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:16.926 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:16.926 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:16.926 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:16.926 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:16.926 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:16.926 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:16.926 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2021-10-31T23:13:16.926Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:16.926Z [INFO] agent.router.manager: shutting down -13:16.926 [INFO] TestAgent: consul server down -13:16.926 [INFO] TestAgent: shutdown complete -13:16.926 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16435 network=tcp -13:16.926 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16435 network=udp -13:16.926 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16436 network=tcp -13:17.429 [INFO] TestAgent: Waiting for endpoints to shut down -13:17.429 [INFO] TestAgent: Endpoints down ---- PASS: TestPrettyPrintBare (0.70s) -=== CONT TestAgent_Join -2021-10-31T23:13:16.417Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:16.442Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:16.444 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9e8f681f-a5c3-03bf-6943-5cc99985bcf1 Address:127.0.0.1:16391}]" -13:16.445 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9e8f681f-a5c3-03bf-6943-5cc99985bcf1.dc1 127.0.0.1 -13:16.445 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16391 [Follower]" leader= -13:16.450 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9e8f681f-a5c3-03bf-6943-5cc99985bcf1 127.0.0.1 -2021-10-31T23:13:16.450Z [INFO] agent.router: Initializing LAN area manager -13:16.450 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16379 network=udp -13:16.451 [INFO] TestAgent.server: Adding LAN server: server="Node-9e8f681f-a5c3-03bf-6943-5cc99985bcf1 (Addr: tcp/127.0.0.1:16391) (DC: dc1)" -13:16.451 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9e8f681f-a5c3-03bf-6943-5cc99985bcf1.dc1 area=wan -13:16.451 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16379 network=tcp -13:16.451 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16380 network=tcp -13:16.451 [INFO] TestAgent: started state syncer -13:16.451 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16392 network=tcp -13:16.503 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:16.503 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16391 [Candidate]" term=2 -13:16.503 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:16.503 [DEBUG] TestAgent.server.raft: vote granted: from=9e8f681f-a5c3-03bf-6943-5cc99985bcf1 term=2 tally=1 -13:16.503 [INFO] TestAgent.server.raft: election won: tally=1 -13:16.503 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16391 [Leader]" -13:16.504 [INFO] TestAgent.server: cluster leadership acquired -13:16.504 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16391 -13:16.504 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:16.508 [INFO] TestAgent.server: New leader elected: payload=Node-9e8f681f-a5c3-03bf-6943-5cc99985bcf1 -13:16.510 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:16.510 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:16.510 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:16.510 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:16.510 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:16.510 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9e8f681f-a5c3-03bf-6943-5cc99985bcf1 -13:16.510 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9e8f681f-a5c3-03bf-6943-5cc99985bcf1 -13:16.511 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.626 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:16.626 [INFO] TestAgent: Synced node info -13:16.626 [DEBUG] TestAgent: Node info in sync -13:16.990 [INFO] TestAgent: Requesting shutdown -13:16.990 [INFO] TestAgent.server: shutting down server -13:16.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:16.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:16.990 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:16.990 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:16.990 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:16.990 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:16.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:16.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:16.990 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:16.990 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:16.990 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:16.990Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:16.990Z [INFO] agent.router.manager: shutting down -13:16.990 [INFO] TestAgent: consul server down -13:16.990 [INFO] TestAgent: shutdown complete -13:16.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16379 network=tcp -13:16.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16379 network=udp -13:16.991 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16380 network=tcp -13:17.492 [INFO] TestAgent: Waiting for endpoints to shut down -13:17.492 [INFO] TestAgent: Endpoints down ---- PASS: TestCoordinate_Node (1.12s) -=== CONT TestConfig_Apply_TerminatingGateway -2021-10-31T23:13:16.725Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:16.763Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:16.764 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fbcb37bb-0156-a6b2-6f64-27ec2f1a9200 Address:127.0.0.1:16447}]" -13:16.765 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fbcb37bb-0156-a6b2-6f64-27ec2f1a9200.dc1 127.0.0.1 -13:16.765 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16447 [Follower]" leader= -13:16.778 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fbcb37bb-0156-a6b2-6f64-27ec2f1a9200 127.0.0.1 -2021-10-31T23:13:16.778Z [INFO] agent.router: Initializing LAN area manager -13:16.778 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16442 network=udp -13:16.778 [INFO] TestAgent.server: Adding LAN server: server="Node-fbcb37bb-0156-a6b2-6f64-27ec2f1a9200 (Addr: tcp/127.0.0.1:16447) (DC: dc1)" -13:16.778 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fbcb37bb-0156-a6b2-6f64-27ec2f1a9200.dc1 area=wan -13:16.778 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16442 network=tcp -13:16.779 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16443 network=tcp -13:16.779 [INFO] TestAgent: started state syncer -13:16.779 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16448 network=tcp -13:16.823 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:16.824 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16447 [Candidate]" term=2 -13:16.824 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:16.824 [DEBUG] TestAgent.server.raft: vote granted: from=fbcb37bb-0156-a6b2-6f64-27ec2f1a9200 term=2 tally=1 -13:16.824 [INFO] TestAgent.server.raft: election won: tally=1 -13:16.824 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16447 [Leader]" -13:16.824 [INFO] TestAgent.server: cluster leadership acquired -13:16.824 [INFO] TestAgent.server: New leader elected: payload=Node-fbcb37bb-0156-a6b2-6f64-27ec2f1a9200 -13:16.824 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16447 -13:16.825 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:16.827 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:16.827 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:16.827 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:16.827 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:16.827 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:16.827 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fbcb37bb-0156-a6b2-6f64-27ec2f1a9200 -13:16.827 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fbcb37bb-0156-a6b2-6f64-27ec2f1a9200 -13:16.828 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.949 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:16.949 [INFO] TestAgent: Synced node info -13:17.023 [DEBUG] connect.ca.consul: consul CA provider configured: id=9b:5e:e1:8c:af:5b:e3:fe:5b:8f:ee:67:54:f4:59:32:86:fe:90:bd:f5:59:60:e3:7c:46:d2:ea:b6:16:ed:e2 is_primary=true -13:17.053 [INFO] TestAgent.server.connect: CA rotated to new root under provider: provider=consul -13:17.053 [INFO] TestAgent: Requesting shutdown -13:17.057 [INFO] TestAgent.server: shutting down server -13:17.057 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:17.057 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:17.057 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:17.057 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:17.057 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:17.057 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:17.057 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:17.057 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:17.057 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:17.057 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:17.057 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:17.058 [INFO] TestAgent: consul server down -13:17.058 [INFO] TestAgent: shutdown complete -13:17.058 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16442 network=tcp -2021-10-31T23:13:17.058Z [INFO] agent.router.manager: shutting down -13:17.061 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16442 network=udp -13:17.085 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16443 network=tcp -2021-10-31T23:13:17.058Z [INFO] agent.router.manager: shutting down -13:17.588 [INFO] TestAgent: Waiting for endpoints to shut down -13:17.588 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectCARoots_list (0.90s) -=== CONT TestConfig_Apply -[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:16.705Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:16.737Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:16.751 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1b935eb2-58b1-323a-554a-1cda0d204be2 Address:127.0.0.1:16433}]" -13:16.752 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1b935eb2-58b1-323a-554a-1cda0d204be2.dc1 127.0.0.1 -13:16.752 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1b935eb2-58b1-323a-554a-1cda0d204be2 127.0.0.1 -2021-10-31T23:13:16.752Z [INFO] agent.router: Initializing LAN area manager -13:16.752 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16449 network=udp -13:16.752 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16433 [Follower]" leader= -13:16.753 [INFO] TestAgent.server: Adding LAN server: server="Node-1b935eb2-58b1-323a-554a-1cda0d204be2 (Addr: tcp/127.0.0.1:16433) (DC: dc1)" -13:16.753 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1b935eb2-58b1-323a-554a-1cda0d204be2.dc1 area=wan -13:16.753 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16449 network=tcp -13:16.753 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16450 network=tcp -13:16.762 [INFO] TestAgent: started state syncer -13:16.763 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16434 network=tcp -13:16.813 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:16.813 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16433 [Candidate]" term=2 -13:16.824 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:16.824 [DEBUG] TestAgent.server.raft: vote granted: from=1b935eb2-58b1-323a-554a-1cda0d204be2 term=2 tally=1 -13:16.824 [INFO] TestAgent.server.raft: election won: tally=1 -13:16.824 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16433 [Leader]" -13:16.824 [INFO] TestAgent.server: cluster leadership acquired -13:16.824 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16433 -13:16.825 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:16.827 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:16.827 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:16.827 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:16.827 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:16.827 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:16.827 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1b935eb2-58b1-323a-554a-1cda0d204be2 -13:16.827 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1b935eb2-58b1-323a-554a-1cda0d204be2 -13:16.828 [INFO] TestAgent.server: New leader elected: payload=Node-1b935eb2-58b1-323a-554a-1cda0d204be2 -13:16.828 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.882 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:16.883 [INFO] TestAgent: Synced node info -13:17.186 [INFO] TestAgent.server.connect: CA provider config updated -13:17.187 [INFO] TestAgent: Requesting shutdown -13:17.187 [INFO] TestAgent.server: shutting down server -13:17.187 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:17.187 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:17.187 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:17.187 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:17.187 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:17.187 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:17.187 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:17.187 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:17.187 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:17.187Z [INFO] agent.router.manager: shutting down -13:17.187 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2021-10-31T23:13:17.187Z [INFO] agent.router.manager: shutting down -13:17.187 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:17.187 [INFO] TestAgent: consul server down -13:17.187 [INFO] TestAgent: shutdown complete -13:17.187 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16449 network=tcp -13:17.187 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16449 network=udp -13:17.187 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16450 network=tcp -13:17.695 [INFO] TestAgent: Waiting for endpoints to shut down -13:17.695 [INFO] TestAgent: Endpoints down -=== RUN TestConnectCAConfig/basic_with_IntermediateCertTTL +=== CONT TestCatalog_GatewayServices_Ingress +2022-12-04T05:49:27.819Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.836Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.839 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e32d06f5-d073-9ce0-d7f3-aa886a726762 Address:127.0.0.1:24178}]" +49:27.839 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e32d06f5-d073-9ce0-d7f3-aa886a726762.dc1 127.0.0.1 +49:27.840 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24178 [Follower]" leader= +49:27.840 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e32d06f5-d073-9ce0-d7f3-aa886a726762 127.0.0.1 +2022-12-04T05:49:27.840Z [INFO] agent.router: Initializing LAN area manager +49:27.840 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e32d06f5-d073-9ce0-d7f3-aa886a726762.dc1 area=wan +49:27.840 [INFO] TestAgent.server: Adding LAN server: server="Node-e32d06f5-d073-9ce0-d7f3-aa886a726762 (Addr: tcp/127.0.0.1:24178) (DC: dc1)" +49:27.841 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24166 network=udp +49:27.841 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24166 network=tcp +49:27.842 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24167 network=tcp +49:27.842 [INFO] TestAgent: started state syncer +49:27.842 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24179 network=tcp +49:27.906 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.906 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24178 [Candidate]" term=2 +49:27.906 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.906 [DEBUG] TestAgent.server.raft: vote granted: from=e32d06f5-d073-9ce0-d7f3-aa886a726762 term=2 tally=1 +49:27.906 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.906 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24178 [Leader]" +49:27.907 [INFO] TestAgent.server: cluster leadership acquired +49:27.907 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24178 +49:27.907 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.907 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.907 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e32d06f5-d073-9ce0-d7f3-aa886a726762 +49:27.907 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e32d06f5-d073-9ce0-d7f3-aa886a726762 +49:27.908 [INFO] TestAgent.server: New leader elected: payload=Node-e32d06f5-d073-9ce0-d7f3-aa886a726762 +49:27.908 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.936 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:27.937 [INFO] TestAgent: Synced node info +49:27.937 [DEBUG] TestAgent: Node info in sync +49:28.228 [INFO] TestAgent: Requesting shutdown +49:28.228 [INFO] TestAgent.server: shutting down server +49:28.228 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.228 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.228 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.228 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.228 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.228 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.228 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:28.228Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.228Z [INFO] agent.router.manager: shutting down +49:28.228 [INFO] TestAgent: consul server down +49:28.228 [INFO] TestAgent: shutdown complete +49:28.228 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24166 network=tcp +49:28.228 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24166 network=udp +49:28.228 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24167 network=tcp +49:28.729 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.729 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectCARoots_empty (0.92s) +=== CONT TestCatalogNodeServices_Filter === RUN TestConfig_Apply_Decoding/No_Kind === RUN TestConfig_Apply_Decoding/Kind_Not_String === RUN TestConfig_Apply_Decoding/Lowercase_kind -2021-10-31T23:13:17.106Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.129Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.131 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cbe8097f-0af1-a630-0879-a95894a60fac Address:127.0.0.1:16363}]" -13:17.132 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cbe8097f-0af1-a630-0879-a95894a60fac.dc1 127.0.0.1 -13:17.132 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cbe8097f-0af1-a630-0879-a95894a60fac 127.0.0.1 -2021-10-31T23:13:17.132Z [INFO] agent.router: Initializing LAN area manager -13:17.132 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16363 [Follower]" leader= -13:17.133 [INFO] TestAgent.server: Adding LAN server: server="Node-cbe8097f-0af1-a630-0879-a95894a60fac (Addr: tcp/127.0.0.1:16363) (DC: dc1)" -13:17.133 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cbe8097f-0af1-a630-0879-a95894a60fac.dc1 area=wan -13:17.133 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16414 network=tcp -13:17.133 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16414 network=udp -13:17.146 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16415 network=tcp -13:17.146 [INFO] TestAgent: started state syncer -13:17.146 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16364 network=tcp -13:17.197 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:17.197 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16363 [Candidate]" term=2 -13:17.198 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:17.198 [DEBUG] TestAgent.server.raft: vote granted: from=cbe8097f-0af1-a630-0879-a95894a60fac term=2 tally=1 -13:17.198 [INFO] TestAgent.server.raft: election won: tally=1 -13:17.198 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16363 [Leader]" -13:17.198 [INFO] TestAgent.server: cluster leadership acquired -13:17.198 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16363 -13:17.199 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:17.200 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:17.200 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:17.200 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:17.200 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:17.200 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:17.200 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cbe8097f-0af1-a630-0879-a95894a60fac -13:17.200 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cbe8097f-0af1-a630-0879-a95894a60fac -13:17.201 [INFO] TestAgent.server: New leader elected: payload=Node-cbe8097f-0af1-a630-0879-a95894a60fac -13:17.201 [INFO] TestAgent.server: federation state anti-entropy synced -13:17.237 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/key?pretty=1 from= latency=27.141µs -13:17.237 [INFO] TestAgent: Requesting shutdown -13:17.237 [INFO] TestAgent.server: shutting down server -13:17.237 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:17.237 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:17.238 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:17.238 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:17.238 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:17.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:17.238 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:17.238 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:17.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:17.238 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:17.238 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:17.238 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:17.238Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:17.238Z [INFO] agent.router.manager: shutting down -13:17.238 [INFO] TestAgent: consul server down -13:17.238 [INFO] TestAgent: shutdown complete -13:17.238 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16414 network=tcp -13:17.238 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16414 network=udp -13:17.238 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16415 network=tcp -13:17.738 [INFO] TestAgent: Waiting for endpoints to shut down -13:17.781 [INFO] TestAgent: Endpoints down ---- PASS: TestPrettyPrint (0.77s) -=== CONT TestAgentConnectCALeafCert_secondaryDC_good -2021-10-31T23:13:17.103Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.134Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.142 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:97164718-f525-523d-2de0-2870db6bcc60 Address:127.0.0.1:16412}]" -13:17.142 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16412 [Follower]" leader= -13:17.143 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-97164718-f525-523d-2de0-2870db6bcc60.dc1 127.0.0.1 -13:17.143 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-97164718-f525-523d-2de0-2870db6bcc60 127.0.0.1 -2021-10-31T23:13:17.143Z [INFO] agent.router: Initializing LAN area manager -13:17.144 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16463 network=udp -13:17.144 [INFO] TestAgent.server: Adding LAN server: server="Node-97164718-f525-523d-2de0-2870db6bcc60 (Addr: tcp/127.0.0.1:16412) (DC: dc1)" -13:17.144 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-97164718-f525-523d-2de0-2870db6bcc60.dc1 area=wan -13:17.144 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16463 network=tcp -13:17.145 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16464 network=tcp -13:17.145 [INFO] TestAgent: started state syncer -13:17.145 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16413 network=tcp -13:17.201 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:17.202 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16412 [Candidate]" term=2 -13:17.202 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:17.202 [DEBUG] TestAgent.server.raft: vote granted: from=97164718-f525-523d-2de0-2870db6bcc60 term=2 tally=1 -13:17.202 [INFO] TestAgent.server.raft: election won: tally=1 -13:17.202 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16412 [Leader]" -13:17.202 [INFO] TestAgent.server: cluster leadership acquired -13:17.202 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16412 -13:17.202 [INFO] TestAgent.server: New leader elected: payload=Node-97164718-f525-523d-2de0-2870db6bcc60 -13:17.203 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:17.205 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:17.205 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:17.205 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:17.205 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:17.205 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:17.205 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-97164718-f525-523d-2de0-2870db6bcc60 -13:17.205 [INFO] TestAgent.server: member joined, marking health alive: member=Node-97164718-f525-523d-2de0-2870db6bcc60 -13:17.214 [INFO] TestAgent.server: federation state anti-entropy synced -13:17.282 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=232.589µs client=127.0.0.1:52150 client_network=udp -13:17.286 [INFO] TestAgent: Requesting shutdown -13:17.286 [INFO] TestAgent.server: shutting down server -13:17.286 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:17.286 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:17.286 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:17.286 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:17.286 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:17.286 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:17.286 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:17.286 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:17.286 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:17.286 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:17.286 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:17.286 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:17.286Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:17.286Z [INFO] agent.router.manager: shutting down -13:17.286 [INFO] TestAgent: consul server down -13:17.286 [INFO] TestAgent: shutdown complete -13:17.286 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16463 network=tcp -13:17.286 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16463 network=udp -13:17.286 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16464 network=tcp -13:17.788 [INFO] TestAgent: Waiting for endpoints to shut down -13:17.788 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:16.253Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:16.358Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:16.361 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:70bfed57-3230-4454-51bf-67197f9e537d Address:127.0.0.1:16426}]" -13:16.361 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16426 [Follower]" leader= -13:16.370 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-70bfed57-3230-4454-51bf-67197f9e537d.dc1 127.0.0.1 -13:16.371 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-70bfed57-3230-4454-51bf-67197f9e537d 127.0.0.1 -2021-10-31T23:13:16.371Z [INFO] agent.router: Initializing LAN area manager -13:16.371 [INFO] TestAgent.server: Adding LAN server: server="Node-70bfed57-3230-4454-51bf-67197f9e537d (Addr: tcp/127.0.0.1:16426) (DC: dc1)" -13:16.371 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-70bfed57-3230-4454-51bf-67197f9e537d.dc1 area=wan -13:16.371 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16372 network=udp -13:16.390 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16372 network=tcp -13:16.391 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16373 network=tcp -13:16.391 [INFO] TestAgent: started state syncer -13:16.391 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16427 network=tcp -13:16.425 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:16.425 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16426 [Candidate]" term=2 -13:16.426 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:16.426 [DEBUG] TestAgent.server.raft: vote granted: from=70bfed57-3230-4454-51bf-67197f9e537d term=2 tally=1 -13:16.426 [INFO] TestAgent.server.raft: election won: tally=1 -13:16.426 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16426 [Leader]" -13:16.426 [INFO] TestAgent.server: cluster leadership acquired -13:16.426 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16426 -13:16.427 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:16.429 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:16.429 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:16.429 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:16.429 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:16.429 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:16.429 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-70bfed57-3230-4454-51bf-67197f9e537d -13:16.429 [INFO] TestAgent.server: member joined, marking health alive: member=Node-70bfed57-3230-4454-51bf-67197f9e537d -13:16.429 [INFO] TestAgent.server: New leader elected: payload=Node-70bfed57-3230-4454-51bf-67197f9e537d -13:16.442 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.470 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=101.323µs client=127.0.0.1:54550 client_network=udp -13:16.470 [INFO] TestAgent: Requesting shutdown -13:16.470 [INFO] TestAgent.server: shutting down server -13:16.470 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:16.470 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:16.470 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:16.470 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:16.470 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:16.470 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:16.470 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:16.470 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:16.470 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:16.470 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:16.470 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:16.470 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:16.471Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:16.471Z [INFO] agent.router.manager: shutting down -13:16.471 [INFO] TestAgent: consul server down -13:16.471 [INFO] TestAgent: shutdown complete -13:16.471 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16372 network=tcp -13:16.471 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16372 network=udp -13:16.471 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16373 network=tcp -13:16.972 [INFO] TestAgent: Waiting for endpoints to shut down -13:16.972 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:15.520Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:15.564Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:15.570 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fead647e-a4bb-8a87-0cb1-16d06230733a Address:127.0.0.1:16286}]" -13:15.571 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fead647e-a4bb-8a87-0cb1-16d06230733a.dc1 127.0.0.1 -13:15.571 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fead647e-a4bb-8a87-0cb1-16d06230733a 127.0.0.1 -2021-10-31T23:13:15.571Z [INFO] agent.router: Initializing LAN area manager -13:15.571 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16323 network=udp -13:15.571 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16286 [Follower]" leader= -13:15.572 [INFO] TestAgent.server: Adding LAN server: server="Node-fead647e-a4bb-8a87-0cb1-16d06230733a (Addr: tcp/127.0.0.1:16286) (DC: dc1)" -13:15.572 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fead647e-a4bb-8a87-0cb1-16d06230733a.dc1 area=wan -13:15.572 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16323 network=tcp -13:15.572 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16324 network=tcp -13:15.572 [INFO] TestAgent: started state syncer -13:15.573 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16287 network=tcp -13:15.650 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:15.650 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16286 [Candidate]" term=2 -13:15.651 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:15.651 [DEBUG] TestAgent.server.raft: vote granted: from=fead647e-a4bb-8a87-0cb1-16d06230733a term=2 tally=1 -13:15.651 [INFO] TestAgent.server.raft: election won: tally=1 -13:15.651 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16286 [Leader]" -13:15.651 [INFO] TestAgent.server: cluster leadership acquired -13:15.651 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16286 -13:15.652 [INFO] TestAgent.server: New leader elected: payload=Node-fead647e-a4bb-8a87-0cb1-16d06230733a -13:15.652 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:15.662 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:15.662 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:15.662 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:15.662 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:15.662 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:15.662 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fead647e-a4bb-8a87-0cb1-16d06230733a -13:15.662 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fead647e-a4bb-8a87-0cb1-16d06230733a -13:15.664 [INFO] TestAgent.server: federation state anti-entropy synced -13:15.710 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=239.096µs client=127.0.0.1:59576 client_network=udp -13:15.714 [INFO] TestAgent: Requesting shutdown -13:15.714 [INFO] TestAgent.server: shutting down server -13:15.714 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:15.714 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:15.714 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:15.714 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:15.714 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:15.714 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:15.714 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:15.714 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:15.714 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:15.714 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:15.714 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:15.714 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:15.714Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:15.714Z [INFO] agent.router.manager: shutting down -13:15.714 [INFO] TestAgent: consul server down -13:15.714 [INFO] TestAgent: shutdown complete -13:15.714 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16323 network=tcp -13:15.715 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16323 network=udp -13:15.715 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16324 network=tcp -13:16.217 [INFO] TestAgent: Waiting for endpoints to shut down -13:16.217 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:14.781Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:14.831Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:14.855 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2d371d3c-5d23-36b0-cd1a-a9b534361cac Address:127.0.0.1:16188}]" -13:14.856 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2d371d3c-5d23-36b0-cd1a-a9b534361cac.dc1 127.0.0.1 -13:14.857 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2d371d3c-5d23-36b0-cd1a-a9b534361cac 127.0.0.1 -2021-10-31T23:13:14.857Z [INFO] agent.router: Initializing LAN area manager -13:14.857 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16188 [Follower]" leader= -13:14.858 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16211 network=udp -13:14.858 [INFO] TestAgent.server: Adding LAN server: server="Node-2d371d3c-5d23-36b0-cd1a-a9b534361cac (Addr: tcp/127.0.0.1:16188) (DC: dc1)" -13:14.858 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16211 network=tcp -13:14.858 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2d371d3c-5d23-36b0-cd1a-a9b534361cac.dc1 area=wan -13:14.859 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16212 network=tcp -13:14.859 [INFO] TestAgent: started state syncer -13:14.859 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16189 network=tcp -13:14.898 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:14.898 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16188 [Candidate]" term=2 -13:14.899 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:14.899 [DEBUG] TestAgent.server.raft: vote granted: from=2d371d3c-5d23-36b0-cd1a-a9b534361cac term=2 tally=1 -13:14.899 [INFO] TestAgent.server.raft: election won: tally=1 -13:14.899 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16188 [Leader]" -13:14.899 [INFO] TestAgent.server: cluster leadership acquired -13:14.899 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16188 -13:14.900 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:14.902 [INFO] TestAgent.server: New leader elected: payload=Node-2d371d3c-5d23-36b0-cd1a-a9b534361cac -13:14.911 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:14.911 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:14.911 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:14.911 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:14.911 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:14.911 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2d371d3c-5d23-36b0-cd1a-a9b534361cac -13:14.911 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2d371d3c-5d23-36b0-cd1a-a9b534361cac -13:14.912 [INFO] TestAgent.server: federation state anti-entropy synced -13:14.990 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=234.18µs client=127.0.0.1:34988 client_network=udp -13:14.990 [INFO] TestAgent: Requesting shutdown -13:14.990 [INFO] TestAgent.server: shutting down server -13:14.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.990 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.990 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.990 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.990 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.990 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:14.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.990 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.990 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:14.990 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:14.990Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.990Z [INFO] agent.router.manager: shutting down -13:14.990 [INFO] TestAgent: consul server down -13:14.990 [INFO] TestAgent: shutdown complete -13:14.991 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16211 network=tcp -13:14.991 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16211 network=udp -13:14.991 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16212 network=tcp -13:15.492 [INFO] TestAgent: Waiting for endpoints to shut down -13:15.492 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_SOA_Settings (3.05s) -=== CONT TestConfig_Delete -[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:17.129Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.152Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.162 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8ad1d8a3-65d5-e8ed-df8a-6a845bd1229e Address:127.0.0.1:16482}]" -13:17.163 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8ad1d8a3-65d5-e8ed-df8a-6a845bd1229e.dc1 127.0.0.1 -13:17.163 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8ad1d8a3-65d5-e8ed-df8a-6a845bd1229e 127.0.0.1 -2021-10-31T23:13:17.163Z [INFO] agent.router: Initializing LAN area manager -13:17.164 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16365 network=udp -13:17.164 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16482 [Follower]" leader= -13:17.164 [INFO] TestAgent.server: Adding LAN server: server="Node-8ad1d8a3-65d5-e8ed-df8a-6a845bd1229e (Addr: tcp/127.0.0.1:16482) (DC: dc1)" -13:17.164 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8ad1d8a3-65d5-e8ed-df8a-6a845bd1229e.dc1 area=wan -13:17.164 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16365 network=tcp -13:17.165 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16366 network=tcp -13:17.165 [INFO] TestAgent: started state syncer -13:17.165 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16483 network=tcp -13:17.217 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:17.217 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16482 [Candidate]" term=2 -13:17.218 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:17.218 [DEBUG] TestAgent.server.raft: vote granted: from=8ad1d8a3-65d5-e8ed-df8a-6a845bd1229e term=2 tally=1 -13:17.218 [INFO] TestAgent.server.raft: election won: tally=1 -13:17.218 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16482 [Leader]" -13:17.218 [INFO] TestAgent.server: cluster leadership acquired -13:17.218 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16482 -13:17.218 [INFO] TestAgent.server: New leader elected: payload=Node-8ad1d8a3-65d5-e8ed-df8a-6a845bd1229e -13:17.219 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:17.221 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:17.221 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:17.221 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:17.221 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:17.221 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:17.221 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8ad1d8a3-65d5-e8ed-df8a-6a845bd1229e -13:17.221 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8ad1d8a3-65d5-e8ed-df8a-6a845bd1229e -13:17.222 [INFO] TestAgent.server: federation state anti-entropy synced -13:17.371 [INFO] TestAgent: Requesting shutdown -13:17.371 [INFO] TestAgent.server: shutting down server -13:17.371 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:17.371 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:17.371 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:17.371 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:17.371 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:17.371 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:17.371 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:17.371 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:17.371 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:17.371 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:17.371 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:17.371 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:17.371Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:17.371Z [INFO] agent.router.manager: shutting down -13:17.371 [INFO] TestAgent: consul server down -13:17.371 [INFO] TestAgent: shutdown complete -13:17.371 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16365 network=tcp -13:17.371 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16365 network=udp -13:17.371 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16366 network=tcp -13:17.872 [INFO] TestAgent: Waiting for endpoints to shut down -13:17.872 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Apply_ProxyDefaultsExpose (0.86s) -=== CONT TestConfig_Get -2021-10-31T23:13:17.051Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.115Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.117 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ced282e0-2184-e4a0-8034-5541c165dcba Address:127.0.0.1:16461}]" -13:17.117 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16461 [Follower]" leader= -13:17.122 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ced282e0-2184-e4a0-8034-5541c165dcba.dc1 127.0.0.1 -13:17.122 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ced282e0-2184-e4a0-8034-5541c165dcba 127.0.0.1 -2021-10-31T23:13:17.122Z [INFO] agent.router: Initializing LAN area manager -13:17.123 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16456 network=udp -13:17.123 [INFO] TestAgent.server: Adding LAN server: server="Node-ced282e0-2184-e4a0-8034-5541c165dcba (Addr: tcp/127.0.0.1:16461) (DC: dc1)" -13:17.123 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ced282e0-2184-e4a0-8034-5541c165dcba.dc1 area=wan -13:17.123 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16456 network=tcp -13:17.124 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16457 network=tcp -13:17.124 [INFO] TestAgent: started state syncer -13:17.124 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16462 network=tcp -13:17.170 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:17.170 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16461 [Candidate]" term=2 -13:17.170 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:17.170 [DEBUG] TestAgent.server.raft: vote granted: from=ced282e0-2184-e4a0-8034-5541c165dcba term=2 tally=1 -13:17.170 [INFO] TestAgent.server.raft: election won: tally=1 -13:17.170 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16461 [Leader]" -13:17.170 [INFO] TestAgent.server: cluster leadership acquired -13:17.171 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16461 -13:17.171 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:17.171 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:17.171 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ced282e0-2184-e4a0-8034-5541c165dcba -13:17.171 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ced282e0-2184-e4a0-8034-5541c165dcba -13:17.172 [INFO] TestAgent.server: New leader elected: payload=Node-ced282e0-2184-e4a0-8034-5541c165dcba -13:17.172 [INFO] TestAgent.server: federation state anti-entropy synced -13:17.462 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:17.462 [INFO] TestAgent: Synced node info -13:17.462 [DEBUG] TestAgent: Node info in sync -13:17.550 [INFO] TestAgent: Requesting shutdown -13:17.550 [INFO] TestAgent.server: shutting down server -13:17.550 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:17.550 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:17.550 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:17.550 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:17.550 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:17.550 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:17.550 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:17.550Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:17.550Z [INFO] agent.router.manager: shutting down -13:17.551 [INFO] TestAgent: consul server down -13:17.551 [INFO] TestAgent: shutdown complete -13:17.551 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16456 network=tcp -13:17.551 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16456 network=udp -13:17.551 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16457 network=tcp -13:18.054 [INFO] TestAgent: Waiting for endpoints to shut down -13:18.054 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectCARoots_empty (1.09s) -=== CONT TestAgentConnectCALeafCert_aclServiceReadDeny -2021-10-31T23:13:17.424Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.479Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.481 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d4c09d80-7cd1-8f32-942c-bfb5e052adcc Address:127.0.0.1:16468}]" -13:17.481 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16468 [Follower]" leader= -13:17.498 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d4c09d80-7cd1-8f32-942c-bfb5e052adcc.dc1 127.0.0.1 -13:17.499 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d4c09d80-7cd1-8f32-942c-bfb5e052adcc 127.0.0.1 -2021-10-31T23:13:17.499Z [INFO] agent.router: Initializing LAN area manager -13:17.499 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16484 network=udp -13:17.499 [INFO] TestAgent.server: Adding LAN server: server="Node-d4c09d80-7cd1-8f32-942c-bfb5e052adcc (Addr: tcp/127.0.0.1:16468) (DC: dc1)" -13:17.499 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d4c09d80-7cd1-8f32-942c-bfb5e052adcc.dc1 area=wan -13:17.499 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16484 network=tcp -13:17.500 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16485 network=tcp -13:17.500 [INFO] TestAgent: started state syncer -13:17.500 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16469 network=tcp -13:17.542 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:17.542 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16468 [Candidate]" term=2 -13:17.542 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:17.542 [DEBUG] TestAgent.server.raft: vote granted: from=d4c09d80-7cd1-8f32-942c-bfb5e052adcc term=2 tally=1 -13:17.542 [INFO] TestAgent.server.raft: election won: tally=1 -13:17.542 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16468 [Leader]" -13:17.542 [INFO] TestAgent.server: cluster leadership acquired -13:17.543 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16468 -13:17.543 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:17.545 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:17.545 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:17.545 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:17.545 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:17.545 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:17.545 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d4c09d80-7cd1-8f32-942c-bfb5e052adcc -13:17.545 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d4c09d80-7cd1-8f32-942c-bfb5e052adcc -13:17.545 [INFO] TestAgent.server: New leader elected: payload=Node-d4c09d80-7cd1-8f32-942c-bfb5e052adcc -13:17.550 [INFO] TestAgent.server: federation state anti-entropy synced -13:17.598 [INFO] TestAgent: Synced node info -13:17.599 [INFO] TestAgent: Synced check: check=test -13:17.599 [INFO] TestAgent: Requesting shutdown -13:17.599 [INFO] TestAgent.server: shutting down server -13:17.599 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:17.599 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:17.599 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:17.599 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:17.599 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:17.599 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:17.599 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:17.599 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:17.599 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:17.599 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:17.599 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:17.599 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:17.599Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:17.599Z [INFO] agent.router.manager: shutting down -13:17.599 [INFO] TestAgent: consul server down -13:17.599 [INFO] TestAgent: shutdown complete -13:17.599 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16484 network=tcp -13:17.599 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16484 network=udp -13:17.599 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16485 network=tcp -13:18.099 [INFO] TestAgent: Waiting for endpoints to shut down -13:18.100 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RegisterCheck (0.72s) -=== CONT TestCatalog_GatewayServices_Ingress -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:17.224Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.291Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.293 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:545d345f-ccc4-a313-2cf4-35ebd0e4b32d Address:127.0.0.1:16384}]" -13:17.293 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16384 [Follower]" leader= -13:17.294 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-545d345f-ccc4-a313-2cf4-35ebd0e4b32d.dc1 127.0.0.1 -13:17.295 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-545d345f-ccc4-a313-2cf4-35ebd0e4b32d 127.0.0.1 -2021-10-31T23:13:17.295Z [INFO] agent.router: Initializing LAN area manager -13:17.295 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16477 network=udp -13:17.295 [INFO] TestAgent.server: Adding LAN server: server="Node-545d345f-ccc4-a313-2cf4-35ebd0e4b32d (Addr: tcp/127.0.0.1:16384) (DC: dc1)" -13:17.295 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-545d345f-ccc4-a313-2cf4-35ebd0e4b32d.dc1 area=wan -13:17.295 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16477 network=tcp -13:17.296 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16478 network=tcp -13:17.296 [INFO] TestAgent: started state syncer -13:17.296 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16385 network=tcp -13:17.332 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:17.332 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16384 [Candidate]" term=2 -13:17.332 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:17.332 [DEBUG] TestAgent.server.raft: vote granted: from=545d345f-ccc4-a313-2cf4-35ebd0e4b32d term=2 tally=1 -13:17.332 [INFO] TestAgent.server.raft: election won: tally=1 -13:17.332 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16384 [Leader]" -13:17.332 [INFO] TestAgent.server: cluster leadership acquired -13:17.332 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16384 -13:17.333 [INFO] TestAgent.server: New leader elected: payload=Node-545d345f-ccc4-a313-2cf4-35ebd0e4b32d -13:17.334 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:17.336 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:17.336 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:17.336 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:17.336 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:17.336 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:17.337 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-545d345f-ccc4-a313-2cf4-35ebd0e4b32d -13:17.337 [INFO] TestAgent.server: member joined, marking health alive: member=Node-545d345f-ccc4-a313-2cf4-35ebd0e4b32d -13:17.347 [INFO] TestAgent: Synced node info -13:17.347 [INFO] TestAgent.server: federation state anti-entropy synced -13:17.609 [INFO] TestAgent: Requesting shutdown -13:17.609 [INFO] TestAgent.server: shutting down server -13:17.609 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:17.609 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:17.609 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:17.609 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:17.609 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:17.609 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:17.609 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:17.609 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:17.609 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:17.609 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:17.610 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:17.610Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:17.610Z [INFO] agent.router.manager: shutting down -13:17.610 [INFO] TestAgent: consul server down -13:17.610 [INFO] TestAgent: shutdown complete -13:17.610 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16477 network=tcp -13:17.610 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16477 network=udp -13:17.610 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16478 network=tcp -13:18.113 [INFO] TestAgent: Waiting for endpoints to shut down -13:18.118 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Apply_CAS (0.96s) -=== CONT TestAgentConnectCALeafCert_aclServiceWrite -2021-10-31T23:13:17.212Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.241Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.254 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:13471fc5-6ede-3014-7f0a-5d46b171c614 Address:127.0.0.1:16475}]" -13:17.255 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-13471fc5-6ede-3014-7f0a-5d46b171c614.dc1 127.0.0.1 -13:17.256 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-13471fc5-6ede-3014-7f0a-5d46b171c614 127.0.0.1 -2021-10-31T23:13:17.256Z [INFO] agent.router: Initializing LAN area manager -13:17.256 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16491 network=udp -13:17.256 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16475 [Follower]" leader= -13:17.256 [INFO] TestAgent.server: Adding LAN server: server="Node-13471fc5-6ede-3014-7f0a-5d46b171c614 (Addr: tcp/127.0.0.1:16475) (DC: dc1)" -13:17.256 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-13471fc5-6ede-3014-7f0a-5d46b171c614.dc1 area=wan -13:17.256 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16491 network=tcp -13:17.257 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16492 network=tcp -13:17.257 [INFO] TestAgent: started state syncer -13:17.257 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16476 network=tcp -13:17.312 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:17.312 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16475 [Candidate]" term=2 -13:17.312 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:17.312 [DEBUG] TestAgent.server.raft: vote granted: from=13471fc5-6ede-3014-7f0a-5d46b171c614 term=2 tally=1 -13:17.312 [INFO] TestAgent.server.raft: election won: tally=1 -13:17.312 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16475 [Leader]" -13:17.314 [INFO] TestAgent.server: cluster leadership acquired -13:17.314 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16475 -13:17.316 [INFO] TestAgent.server: New leader elected: payload=Node-13471fc5-6ede-3014-7f0a-5d46b171c614 -13:17.316 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:17.331 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:17.331 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:17.331 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:17.331 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:17.331 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:17.331 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-13471fc5-6ede-3014-7f0a-5d46b171c614 -13:17.331 [INFO] TestAgent.server: member joined, marking health alive: member=Node-13471fc5-6ede-3014-7f0a-5d46b171c614 -13:17.332 [INFO] TestAgent.server: federation state anti-entropy synced -13:17.690 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:17.690 [INFO] TestAgent: Synced node info -13:17.690 [DEBUG] TestAgent: Node info in sync -13:17.703 [INFO] TestAgent: Requesting shutdown -13:17.704 [INFO] TestAgent.server: shutting down server -13:17.704 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:17.704 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:17.704 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:17.704 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:17.704 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:17.704 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:17.704 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:17.704 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:17.704 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:17.704 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:17.704 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:17.704Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:17.704Z [INFO] agent.router.manager: shutting down -13:17.705 [INFO] TestAgent: consul server down -13:17.705 [INFO] TestAgent: shutdown complete -13:17.705 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16491 network=tcp -13:17.705 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16491 network=udp -13:17.705 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16492 network=tcp -13:18.205 [INFO] TestAgent: Waiting for endpoints to shut down -13:18.205 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Apply_Decoding (1.03s) - --- PASS: TestConfig_Apply_Decoding/No_Kind (0.00s) - --- PASS: TestConfig_Apply_Decoding/Kind_Not_String (0.00s) - --- PASS: TestConfig_Apply_Decoding/Lowercase_kind (0.00s) +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=255)_compressed=true +2022-12-04T05:49:28.191Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.201Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.204 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:71abe443-45ff-14ec-d3ae-bef7f605f9ab Address:127.0.0.1:24213}]" +49:28.204 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24213 [Follower]" leader= +49:28.205 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-71abe443-45ff-14ec-d3ae-bef7f605f9ab.dc1 127.0.0.1 +49:28.206 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-71abe443-45ff-14ec-d3ae-bef7f605f9ab 127.0.0.1 +2022-12-04T05:49:28.206Z [INFO] agent.router: Initializing LAN area manager +49:28.206 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24043 network=udp +49:28.206 [INFO] TestAgent.server: Adding LAN server: server="Node-71abe443-45ff-14ec-d3ae-bef7f605f9ab (Addr: tcp/127.0.0.1:24213) (DC: dc1)" +49:28.206 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-71abe443-45ff-14ec-d3ae-bef7f605f9ab.dc1 area=wan +49:28.207 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24043 network=tcp +49:28.208 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24058 network=tcp +49:28.208 [INFO] TestAgent: started state syncer +49:28.208 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24214 network=tcp +49:28.254 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.254 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24213 [Candidate]" term=2 +49:28.254 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.254 [DEBUG] TestAgent.server.raft: vote granted: from=71abe443-45ff-14ec-d3ae-bef7f605f9ab term=2 tally=1 +49:28.254 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.254 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24213 [Leader]" +49:28.254 [INFO] TestAgent.server: cluster leadership acquired +49:28.254 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24213 +49:28.255 [INFO] TestAgent.server: New leader elected: payload=Node-71abe443-45ff-14ec-d3ae-bef7f605f9ab +49:28.256 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.258 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.258 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.258 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.258 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.258 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.258 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-71abe443-45ff-14ec-d3ae-bef7f605f9ab +49:28.258 [INFO] TestAgent.server: member joined, marking health alive: member=Node-71abe443-45ff-14ec-d3ae-bef7f605f9ab +49:28.259 [INFO] TestAgent.server: federation state anti-entropy synced +49:28.295 [INFO] TestAgent: Requesting shutdown +49:28.295 [INFO] TestAgent.server: shutting down server +49:28.295 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.295 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.295 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.295 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.295 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.295 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.295 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.295 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:28.295 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.295 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.295 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.295 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:28.295Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.296Z [INFO] agent.router.manager: shutting down +49:28.296 [INFO] TestAgent: consul server down +49:28.296 [INFO] TestAgent: shutdown complete +49:28.296 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24043 network=tcp +49:28.296 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24043 network=udp +49:28.296 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24058 network=tcp +49:28.797 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.797 [INFO] TestAgent: Endpoints down +--- PASS: TestConfig_Apply_IngressGateway (0.62s) === CONT TestCatalog_GatewayServices_Terminating -2021-10-31T23:13:16.230Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:16.255Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:16.257 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:698357cc-e9fa-71d8-c6b2-af3128d15fa4 Address:127.0.0.1:16370}]" -13:16.257 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-698357cc-e9fa-71d8-c6b2-af3128d15fa4.dc1 127.0.0.1 -13:16.258 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16370 [Follower]" leader= -13:16.266 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-698357cc-e9fa-71d8-c6b2-af3128d15fa4 127.0.0.1 -2021-10-31T23:13:16.266Z [INFO] agent.router: Initializing LAN area manager -13:16.266 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16407 network=udp -13:16.267 [INFO] TestAgent.server: Adding LAN server: server="Node-698357cc-e9fa-71d8-c6b2-af3128d15fa4 (Addr: tcp/127.0.0.1:16370) (DC: dc1)" -13:16.267 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-698357cc-e9fa-71d8-c6b2-af3128d15fa4.dc1 area=wan -13:16.267 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16407 network=tcp -13:16.267 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16408 network=tcp -13:16.267 [INFO] TestAgent: started state syncer -13:16.268 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16371 network=tcp -13:16.322 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:16.322 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16370 [Candidate]" term=2 -13:16.323 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:16.323 [DEBUG] TestAgent.server.raft: vote granted: from=698357cc-e9fa-71d8-c6b2-af3128d15fa4 term=2 tally=1 -13:16.323 [INFO] TestAgent.server.raft: election won: tally=1 -13:16.323 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16370 [Leader]" -13:16.323 [INFO] TestAgent.server: cluster leadership acquired -13:16.323 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16370 -13:16.323 [INFO] TestAgent.server: New leader elected: payload=Node-698357cc-e9fa-71d8-c6b2-af3128d15fa4 -13:16.342 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:16.344 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:16.344 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:16.344 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:16.344 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:16.344 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:16.344 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-698357cc-e9fa-71d8-c6b2-af3128d15fa4 -13:16.344 [INFO] TestAgent.server: member joined, marking health alive: member=Node-698357cc-e9fa-71d8-c6b2-af3128d15fa4 -13:16.348 [INFO] TestAgent.server: federation state anti-entropy synced -13:16.626 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:16.626 [INFO] TestAgent: Synced node info -13:16.626 [DEBUG] TestAgent: Node info in sync -13:17.410 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:17.410 [DEBUG] TestAgent: Node info in sync -13:17.831 [INFO] TestAgent: Requesting shutdown -13:17.831 [INFO] TestAgent.server: shutting down server -13:17.831 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:17.831 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:17.831 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:17.831 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:17.831 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:17.831 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:17.831 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:17.831 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:17.831 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:17.831 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:17.831 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:17.831Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:17.831Z [INFO] agent.router.manager: shutting down -13:17.832 [INFO] TestAgent: consul server down -13:17.832 [INFO] TestAgent: shutdown complete -13:17.832 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16407 network=tcp -13:17.832 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16407 network=udp -13:17.832 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16408 network=tcp -13:18.333 [INFO] TestAgent: Waiting for endpoints to shut down -13:18.333 [INFO] TestAgent: Endpoints down ---- PASS: TestPProfHandlers_EnableDebug (2.14s) -=== CONT TestCatalogNodeServices_WanTranslation -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:17.402Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.441Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.451 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bec14bd0-6a14-2d50-25b7-0bcc58357f80 Address:127.0.0.1:16496}]" -13:17.452 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bec14bd0-6a14-2d50-25b7-0bcc58357f80.dc1 127.0.0.1 -13:17.452 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bec14bd0-6a14-2d50-25b7-0bcc58357f80 127.0.0.1 -2021-10-31T23:13:17.452Z [INFO] agent.router: Initializing LAN area manager -13:17.452 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16496 [Follower]" leader= -13:17.453 [INFO] TestAgent.server: Adding LAN server: server="Node-bec14bd0-6a14-2d50-25b7-0bcc58357f80 (Addr: tcp/127.0.0.1:16496) (DC: dc1)" -13:17.453 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bec14bd0-6a14-2d50-25b7-0bcc58357f80.dc1 area=wan -13:17.453 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16386 network=tcp -13:17.453 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16386 network=udp -13:17.466 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16387 network=tcp -13:17.466 [INFO] TestAgent: started state syncer -13:17.466 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16497 network=tcp -13:17.505 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:17.505 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16496 [Candidate]" term=2 -13:17.506 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:17.506 [DEBUG] TestAgent.server.raft: vote granted: from=bec14bd0-6a14-2d50-25b7-0bcc58357f80 term=2 tally=1 -13:17.506 [INFO] TestAgent.server.raft: election won: tally=1 -13:17.506 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16496 [Leader]" -13:17.506 [INFO] TestAgent.server: cluster leadership acquired -13:17.506 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16496 -13:17.507 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:17.508 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:17.508 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:17.508 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:17.508 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:17.508 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:17.509 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bec14bd0-6a14-2d50-25b7-0bcc58357f80 -13:17.509 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bec14bd0-6a14-2d50-25b7-0bcc58357f80 -13:17.509 [INFO] TestAgent.server: New leader elected: payload=Node-bec14bd0-6a14-2d50-25b7-0bcc58357f80 -13:17.518 [INFO] TestAgent.server: federation state anti-entropy synced -13:17.578 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:17.578 [INFO] TestAgent: Synced node info -13:17.859 [INFO] TestAgent: Requesting shutdown -13:17.859 [INFO] TestAgent.server: shutting down server -13:17.859 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:17.859 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:17.859 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:17.859 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:17.859 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:17.860 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:17.860 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:17.860 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:17.860 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:17.860 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:17.860 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:17.860Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:17.860Z [INFO] agent.router.manager: shutting down -13:17.860 [INFO] TestAgent: consul server down -13:17.860 [INFO] TestAgent: shutdown complete -13:17.860 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16386 network=tcp -13:17.860 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16386 network=udp -13:17.860 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16387 network=tcp -13:18.361 [INFO] TestAgent: Waiting for endpoints to shut down -13:18.361 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Apply_ProxyDefaultsMeshGateway (1.05s) -=== CONT TestAgentConnectCALeafCert_aclDefaultDeny +[WARN] freeport: 10 out of 66 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:27.610Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.626Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.629 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1505912d-2dbb-6587-d7ea-94fbde2f9bc4 Address:127.0.0.1:24129}]" +49:27.630 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24129 [Follower]" leader= +49:27.631 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1505912d-2dbb-6587-d7ea-94fbde2f9bc4.dc1 127.0.0.1 +49:27.632 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1505912d-2dbb-6587-d7ea-94fbde2f9bc4 127.0.0.1 +2022-12-04T05:49:27.632Z [INFO] agent.router: Initializing LAN area manager +49:27.632 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1505912d-2dbb-6587-d7ea-94fbde2f9bc4.dc1 area=wan +49:27.632 [INFO] TestAgent.server: Adding LAN server: server="Node-1505912d-2dbb-6587-d7ea-94fbde2f9bc4 (Addr: tcp/127.0.0.1:24129) (DC: dc1)" +49:27.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24103 network=tcp +49:27.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24103 network=udp +49:27.635 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24104 network=tcp +49:27.635 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24130 network=tcp +49:27.635 [INFO] TestAgent: started state syncer +49:27.678 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.678 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24129 [Candidate]" term=2 +49:27.679 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.679 [DEBUG] TestAgent.server.raft: vote granted: from=1505912d-2dbb-6587-d7ea-94fbde2f9bc4 term=2 tally=1 +49:27.679 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.679 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24129 [Leader]" +49:27.679 [INFO] TestAgent.server: cluster leadership acquired +49:27.679 [INFO] TestAgent.server: New leader elected: payload=Node-1505912d-2dbb-6587-d7ea-94fbde2f9bc4 +49:27.679 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24129 +49:27.680 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.681 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.681 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.681 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.681 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.681 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.681 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1505912d-2dbb-6587-d7ea-94fbde2f9bc4 +49:27.681 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1505912d-2dbb-6587-d7ea-94fbde2f9bc4 +49:27.682 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.890 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:27.891 [INFO] TestAgent: Synced node info +49:27.891 [DEBUG] TestAgent: Node info in sync +49:28.306 [INFO] TestAgent: Requesting shutdown +49:28.306 [INFO] TestAgent.server: shutting down server +49:28.306 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.306 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.306 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.306 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.306 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.306 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.306 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.306 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.306 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.306 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:28.306 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:49:28.306Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.306Z [INFO] agent.router.manager: shutting down +49:28.306 [INFO] TestAgent: consul server down +49:28.306 [INFO] TestAgent: shutdown complete +49:28.306 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24103 network=tcp +49:28.306 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24103 network=udp +49:28.306 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24104 network=tcp +49:28.807 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.807 [INFO] TestAgent: Endpoints down +--- PASS: TestCoordinate_Update (1.21s) +=== CONT TestCatalogNodeServiceList +2022-12-04T05:49:25.218Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.228Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.230 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f604e8a9-7c05-8bb4-322b-b69b414a36bb Address:127.0.0.1:23838}]" +49:25.230 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23838 [Follower]" leader= +49:25.230 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f604e8a9-7c05-8bb4-322b-b69b414a36bb.dc1 127.0.0.1 +49:25.231 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f604e8a9-7c05-8bb4-322b-b69b414a36bb 127.0.0.1 +2022-12-04T05:49:25.231Z [INFO] agent.router: Initializing LAN area manager +49:25.231 [DEBUG] TestAgent.dns: recursor enabled +49:25.231 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f604e8a9-7c05-8bb4-322b-b69b414a36bb.dc1 area=wan +49:25.231 [DEBUG] TestAgent.dns: recursor enabled +49:25.231 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23833 network=udp +49:25.231 [INFO] TestAgent.server: Adding LAN server: server="Node-f604e8a9-7c05-8bb4-322b-b69b414a36bb (Addr: tcp/127.0.0.1:23838) (DC: dc1)" +49:25.231 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23833 network=tcp +49:25.231 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23834 network=tcp +49:25.231 [INFO] TestAgent: started state syncer +49:25.232 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23839 network=tcp +49:25.279 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.279 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23838 [Candidate]" term=2 +49:25.279 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.279 [DEBUG] TestAgent.server.raft: vote granted: from=f604e8a9-7c05-8bb4-322b-b69b414a36bb term=2 tally=1 +49:25.279 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.279 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23838 [Leader]" +49:25.279 [INFO] TestAgent.server: cluster leadership acquired +49:25.279 [INFO] TestAgent.server: New leader elected: payload=Node-f604e8a9-7c05-8bb4-322b-b69b414a36bb +49:25.279 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23838 +49:25.280 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.281 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.281 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.281 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.281 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.281 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.281 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f604e8a9-7c05-8bb4-322b-b69b414a36bb +49:25.281 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f604e8a9-7c05-8bb4-322b-b69b414a36bb +49:25.281 [INFO] TestAgent.server: federation state anti-entropy synced +49:25.584 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:25.584 [INFO] TestAgent: Synced node info +49:26.897 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:26.897 [DEBUG] TestAgent: Node info in sync +49:26.897 [DEBUG] TestAgent: Node info in sync +49:28.324 [ERROR] TestAgent.dns: recurse failed: error="read udp 127.0.0.1:58177->127.0.0.1:36917: i/o timeout" +49:28.324 [ERROR] TestAgent.dns: all resolvers failed for question from client: question="{apple.com. 255 1}" client=127.0.0.1:42656 client_network=udp +49:28.324 [DEBUG] TestAgent.dns: request served from client: question="{apple.com. 255 1}" network=udp latency=3.000925994s client=127.0.0.1:42656 client_network=udp +49:28.325 [INFO] TestAgent: Requesting shutdown +49:28.325 [INFO] TestAgent.server: shutting down server +49:28.325 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.325 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.325 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.325 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.325 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.325 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.325 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.325 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.325 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.325 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.325 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:28.325Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.325Z [INFO] agent.router.manager: shutting down +49:28.325 [INFO] TestAgent: consul server down +49:28.325 [INFO] TestAgent: shutdown complete +49:28.325 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23833 network=tcp +49:28.325 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23833 network=udp +49:28.325 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23834 network=tcp +49:28.826 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.826 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_RecursorTimeout (3.62s) +=== CONT TestCatalogNodeServices +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=255)_compressed=false === RUN TestConfig_Get/get_a_single_service_entry === RUN TestConfig_Get/list_both_service_entries === RUN TestConfig_Get/get_global_proxy_config === RUN TestConfig_Get/error_on_no_arguments -2021-10-31T23:13:17.437Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.487Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.489 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:700d704f-6791-5d70-7403-212e9752a624 Address:127.0.0.1:16524}]" -13:17.489 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16524 [Follower]" leader= -13:17.490 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-700d704f-6791-5d70-7403-212e9752a624.dc1 127.0.0.1 -13:17.490 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-700d704f-6791-5d70-7403-212e9752a624 127.0.0.1 -2021-10-31T23:13:17.490Z [INFO] agent.router: Initializing LAN area manager -13:17.491 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16470 network=udp -13:17.491 [INFO] TestAgent.server: Adding LAN server: server="Node-700d704f-6791-5d70-7403-212e9752a624 (Addr: tcp/127.0.0.1:16524) (DC: dc1)" -13:17.491 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-700d704f-6791-5d70-7403-212e9752a624.dc1 area=wan -13:17.491 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16470 network=tcp -13:17.492 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16471 network=tcp -13:17.492 [INFO] TestAgent: started state syncer -13:17.492 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16525 network=tcp -13:17.550 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:17.550 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16524 [Candidate]" term=2 -13:17.550 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:17.550 [DEBUG] TestAgent.server.raft: vote granted: from=700d704f-6791-5d70-7403-212e9752a624 term=2 tally=1 -13:17.550 [INFO] TestAgent.server.raft: election won: tally=1 -13:17.550 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16524 [Leader]" -13:17.550 [INFO] TestAgent.server: cluster leadership acquired -13:17.550 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16524 -13:17.551 [INFO] TestAgent.server: New leader elected: payload=Node-700d704f-6791-5d70-7403-212e9752a624 -13:17.551 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:17.553 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:17.553 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:17.553 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:17.553 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:17.553 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:17.553 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-700d704f-6791-5d70-7403-212e9752a624 -13:17.553 [INFO] TestAgent.server: member joined, marking health alive: member=Node-700d704f-6791-5d70-7403-212e9752a624 -13:17.558 [INFO] TestAgent.server: federation state anti-entropy synced -13:17.862 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:17.862 [INFO] TestAgent: Synced node info -13:17.862 [DEBUG] TestAgent: Node info in sync -13:17.910 [INFO] TestAgent: Requesting shutdown -13:17.910 [INFO] TestAgent.server: shutting down server -13:17.910 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:17.910 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:17.910 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:17.910 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:17.910 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:17.910 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:17.910 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:17.910 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:17.910 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:17.910 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:17.910 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:17.910Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:17.910Z [INFO] agent.router.manager: shutting down -13:17.910 [INFO] TestAgent: consul server down -13:17.910 [INFO] TestAgent: shutdown complete -13:17.910 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16470 network=tcp -13:17.910 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16470 network=udp -13:17.911 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16471 network=tcp -13:18.411 [INFO] TestAgent: Waiting for endpoints to shut down -13:18.411 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_badBody (1.00s) -=== CONT TestCatalogNodeServices_ConnectProxy -2021-10-31T23:13:17.826Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.854Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.856 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:de416480-387a-c7d1-e715-22cbd5f63490 Address:127.0.0.1:16517}]" -13:17.857 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-de416480-387a-c7d1-e715-22cbd5f63490.dc1 127.0.0.1 -13:17.857 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16517 [Follower]" leader= -13:17.878 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-de416480-387a-c7d1-e715-22cbd5f63490 127.0.0.1 -2021-10-31T23:13:17.878Z [INFO] agent.router: Initializing LAN area manager -13:17.879 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16547 network=udp -13:17.879 [INFO] TestAgent.server: Adding LAN server: server="Node-de416480-387a-c7d1-e715-22cbd5f63490 (Addr: tcp/127.0.0.1:16517) (DC: dc1)" -13:17.879 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-de416480-387a-c7d1-e715-22cbd5f63490.dc1 area=wan -13:17.879 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16547 network=tcp -13:17.880 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16548 network=tcp -13:17.881 [INFO] TestAgent: started state syncer -13:17.881 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16518 network=tcp -13:17.908 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:17.908 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16517 [Candidate]" term=2 -13:17.908 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:17.908 [DEBUG] TestAgent.server.raft: vote granted: from=de416480-387a-c7d1-e715-22cbd5f63490 term=2 tally=1 -13:17.908 [INFO] TestAgent.server.raft: election won: tally=1 -13:17.908 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16517 [Leader]" -13:17.908 [INFO] TestAgent.server: cluster leadership acquired -13:17.908 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16517 -13:17.909 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:17.909 [INFO] TestAgent.server: New leader elected: payload=Node-de416480-387a-c7d1-e715-22cbd5f63490 -13:17.911 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:17.911 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:17.911 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:17.911 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:17.911 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:17.911 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-de416480-387a-c7d1-e715-22cbd5f63490 -13:17.911 [INFO] TestAgent.server: member joined, marking health alive: member=Node-de416480-387a-c7d1-e715-22cbd5f63490 -13:17.912 [INFO] TestAgent.server: federation state anti-entropy synced -13:17.918 [INFO] TestAgent: Requesting shutdown -13:17.918 [INFO] TestAgent.server: shutting down server -13:17.918 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:17.918 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:17.918 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:17.918 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:17.918 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:17.918 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:17.918 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:17.918 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:17.918 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:17.918 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:17.918 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:17.918 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:17.918Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:17.918Z [INFO] agent.router.manager: shutting down -13:17.918 [INFO] TestAgent: consul server down -13:17.918 [INFO] TestAgent: shutdown complete -13:17.918 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16547 network=tcp -13:17.918 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16547 network=udp -13:17.918 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16548 network=tcp -13:18.450 [INFO] TestAgent: Waiting for endpoints to shut down -13:18.450 [INFO] TestAgent: Endpoints down +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=true +2022-12-04T05:49:28.090Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.106Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.107 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:da99cbd0-8b8f-7433-b86c-27e2635c7e37 Address:127.0.0.1:24041}]" +49:28.108 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24041 [Follower]" leader= +49:28.108 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-da99cbd0-8b8f-7433-b86c-27e2635c7e37.dc1 127.0.0.1 +49:28.108 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-da99cbd0-8b8f-7433-b86c-27e2635c7e37 127.0.0.1 +2022-12-04T05:49:28.109Z [INFO] agent.router: Initializing LAN area manager +49:28.109 [INFO] TestAgent.server: Adding LAN server: server="Node-da99cbd0-8b8f-7433-b86c-27e2635c7e37 (Addr: tcp/127.0.0.1:24041) (DC: dc1)" +49:28.109 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-da99cbd0-8b8f-7433-b86c-27e2635c7e37.dc1 area=wan +49:28.109 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24208 network=udp +49:28.109 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24208 network=tcp +49:28.109 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24209 network=tcp +49:28.109 [INFO] TestAgent: started state syncer +49:28.109 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24042 network=tcp +49:28.159 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.159 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24041 [Candidate]" term=2 +49:28.160 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.160 [DEBUG] TestAgent.server.raft: vote granted: from=da99cbd0-8b8f-7433-b86c-27e2635c7e37 term=2 tally=1 +49:28.160 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.160 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24041 [Leader]" +49:28.160 [INFO] TestAgent.server: cluster leadership acquired +49:28.160 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24041 +49:28.161 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.161 [INFO] TestAgent.server: New leader elected: payload=Node-da99cbd0-8b8f-7433-b86c-27e2635c7e37 +49:28.162 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.162 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.162 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.162 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.162 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.162 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-da99cbd0-8b8f-7433-b86c-27e2635c7e37 +49:28.162 [INFO] TestAgent.server: member joined, marking health alive: member=Node-da99cbd0-8b8f-7433-b86c-27e2635c7e37 +49:28.163 [INFO] TestAgent.server: federation state anti-entropy synced +49:28.463 [INFO] TestAgent: Requesting shutdown +49:28.463 [INFO] TestAgent.server: shutting down server +49:28.463 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.463 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.463 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.463 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.463 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.463 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.463 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:28.463 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.463 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.463 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.463 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.463 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:28.463Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.463Z [INFO] agent.router.manager: shutting down +49:28.464 [INFO] TestAgent: consul server down +49:28.464 [INFO] TestAgent: shutdown complete +49:28.464 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24208 network=tcp +49:28.464 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24208 network=udp +49:28.464 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24209 network=tcp +49:28.964 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.964 [INFO] TestAgent: Endpoints down +--- PASS: TestConfig_Apply_ProxyDefaultsMeshGateway (0.89s) +=== CONT TestCatalogServiceNodes_ConnectProxy +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:28.368Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.383Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.386 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0a5a765c-2213-b8eb-7678-d099d792d342 Address:127.0.0.1:24199}]" +49:28.386 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24199 [Follower]" leader= +49:28.387 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0a5a765c-2213-b8eb-7678-d099d792d342.dc1 127.0.0.1 +49:28.387 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0a5a765c-2213-b8eb-7678-d099d792d342 127.0.0.1 +2022-12-04T05:49:28.387Z [INFO] agent.router: Initializing LAN area manager +49:28.388 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24159 network=udp +49:28.388 [INFO] TestAgent.server: Adding LAN server: server="Node-0a5a765c-2213-b8eb-7678-d099d792d342 (Addr: tcp/127.0.0.1:24199) (DC: dc1)" +49:28.388 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24159 network=tcp +49:28.388 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0a5a765c-2213-b8eb-7678-d099d792d342.dc1 area=wan +49:28.389 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24160 network=tcp +49:28.390 [INFO] TestAgent: started state syncer +49:28.390 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24200 network=tcp +49:28.449 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.449 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24199 [Candidate]" term=2 +49:28.449 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.449 [DEBUG] TestAgent.server.raft: vote granted: from=0a5a765c-2213-b8eb-7678-d099d792d342 term=2 tally=1 +49:28.449 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.449 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24199 [Leader]" +49:28.449 [INFO] TestAgent.server: cluster leadership acquired +49:28.449 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24199 +49:28.450 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.450 [INFO] TestAgent.server: New leader elected: payload=Node-0a5a765c-2213-b8eb-7678-d099d792d342 +49:28.451 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.451 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.451 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.451 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.451 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.451 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a5a765c-2213-b8eb-7678-d099d792d342 +49:28.451 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0a5a765c-2213-b8eb-7678-d099d792d342 +49:28.452 [INFO] TestAgent.server: federation state anti-entropy synced +49:28.573 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:28.573 [INFO] TestAgent: Synced node info +49:28.573 [DEBUG] TestAgent: Node info in sync +49:28.717 [INFO] TestAgent: Requesting shutdown +49:28.717 [INFO] TestAgent.server: shutting down server +49:28.717 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.717 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.717 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.717 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.717 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.717 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.717 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.717 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.717 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.717 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:28.717 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:28.717Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.717Z [INFO] agent.router.manager: shutting down +49:28.717 [INFO] TestAgent: consul server down +49:28.717 [INFO] TestAgent: shutdown complete +49:28.717 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24159 network=tcp +49:28.717 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24159 network=udp +49:28.718 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24160 network=tcp +49:29.218 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.219 [INFO] TestAgent: Endpoints down +--- PASS: TestConfig_Apply_TerminatingGateway (0.86s) +=== CONT TestCatalogConnectServiceNodes_good +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=false +2022-12-04T05:49:28.329Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.348Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.350 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:412fe5f8-6869-8d54-8ddc-1ba70c2d8e41 Address:127.0.0.1:24157}]" +49:28.351 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-412fe5f8-6869-8d54-8ddc-1ba70c2d8e41.dc1 127.0.0.1 +49:28.351 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24157 [Follower]" leader= +49:28.352 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-412fe5f8-6869-8d54-8ddc-1ba70c2d8e41 127.0.0.1 +2022-12-04T05:49:28.352Z [INFO] agent.router: Initializing LAN area manager +49:28.357 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24215 network=tcp +49:28.357 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24215 network=udp +49:28.357 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-412fe5f8-6869-8d54-8ddc-1ba70c2d8e41.dc1 area=wan +49:28.358 [INFO] TestAgent.server: Adding LAN server: server="Node-412fe5f8-6869-8d54-8ddc-1ba70c2d8e41 (Addr: tcp/127.0.0.1:24157) (DC: dc1)" +49:28.358 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24216 network=tcp +49:28.358 [INFO] TestAgent: started state syncer +49:28.358 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24158 network=tcp +49:28.418 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.418 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24157 [Candidate]" term=2 +49:28.418 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.418 [DEBUG] TestAgent.server.raft: vote granted: from=412fe5f8-6869-8d54-8ddc-1ba70c2d8e41 term=2 tally=1 +49:28.418 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.418 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24157 [Leader]" +49:28.418 [INFO] TestAgent.server: cluster leadership acquired +49:28.418 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24157 +49:28.419 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.420 [INFO] TestAgent.server: New leader elected: payload=Node-412fe5f8-6869-8d54-8ddc-1ba70c2d8e41 +49:28.420 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.420 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.420 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.420 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.420 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.421 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-412fe5f8-6869-8d54-8ddc-1ba70c2d8e41 +49:28.421 [INFO] TestAgent.server: member joined, marking health alive: member=Node-412fe5f8-6869-8d54-8ddc-1ba70c2d8e41 +49:28.421 [INFO] TestAgent.server: federation state anti-entropy synced +49:28.526 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:28.526 [INFO] TestAgent: Synced node info +49:28.526 [DEBUG] TestAgent: Node info in sync +49:28.644 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:28.644 [DEBUG] TestAgent: Node info in sync +49:28.755 [INFO] TestAgent: Requesting shutdown +49:28.756 [INFO] TestAgent.server: shutting down server +49:28.756 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.756 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.756 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.756 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.756 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.756 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.756 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.756 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.756 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.756 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.756 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:28.756Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.756Z [INFO] agent.router.manager: shutting down +49:28.756 [INFO] TestAgent: consul server down +49:28.756 [INFO] TestAgent: shutdown complete +49:28.756 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24215 network=tcp +49:28.756 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24215 network=udp +49:28.756 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24216 network=tcp +49:29.257 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.257 [INFO] TestAgent: Endpoints down +--- PASS: TestConfig_Apply_Decoding (0.94s) + --- PASS: TestConfig_Apply_Decoding/No_Kind (0.00s) + --- PASS: TestConfig_Apply_Decoding/Kind_Not_String (0.00s) + --- PASS: TestConfig_Apply_Decoding/Lowercase_kind (0.00s) +=== CONT TestCatalogServiceNodes_DistanceSort +2022-12-04T05:49:28.625Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.643Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.646 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:253e2bf2-5193-d0da-21dc-6d1231beae75 Address:127.0.0.1:24269}]" +49:28.648 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-253e2bf2-5193-d0da-21dc-6d1231beae75.dc1 127.0.0.1 +49:28.648 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24269 [Follower]" leader= +49:28.649 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-253e2bf2-5193-d0da-21dc-6d1231beae75 127.0.0.1 +2022-12-04T05:49:28.649Z [INFO] agent.router: Initializing LAN area manager +49:28.649 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24257 network=udp +49:28.650 [INFO] TestAgent.server: Adding LAN server: server="Node-253e2bf2-5193-d0da-21dc-6d1231beae75 (Addr: tcp/127.0.0.1:24269) (DC: dc1)" +49:28.650 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-253e2bf2-5193-d0da-21dc-6d1231beae75.dc1 area=wan +49:28.650 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24257 network=tcp +49:28.651 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24258 network=tcp +49:28.651 [INFO] TestAgent: started state syncer +49:28.651 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24270 network=tcp +49:28.691 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.691 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24269 [Candidate]" term=2 +49:28.691 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.691 [DEBUG] TestAgent.server.raft: vote granted: from=253e2bf2-5193-d0da-21dc-6d1231beae75 term=2 tally=1 +49:28.691 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.691 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24269 [Leader]" +49:28.691 [INFO] TestAgent.server: cluster leadership acquired +49:28.692 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24269 +49:28.692 [INFO] TestAgent.server: New leader elected: payload=Node-253e2bf2-5193-d0da-21dc-6d1231beae75 +49:28.693 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.695 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.695 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.695 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.695 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.695 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.695 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-253e2bf2-5193-d0da-21dc-6d1231beae75 +49:28.695 [INFO] TestAgent.server: member joined, marking health alive: member=Node-253e2bf2-5193-d0da-21dc-6d1231beae75 +49:28.697 [INFO] TestAgent.server: federation state anti-entropy synced +49:28.772 [INFO] TestAgent: Requesting shutdown +49:28.772 [INFO] TestAgent.server: shutting down server +49:28.772 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.772 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.772 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.772 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.772 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.772 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:28.772 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.772 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.772 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.772 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.772 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:28.772 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:49:28.772Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.772Z [INFO] agent.router.manager: shutting down +49:28.772 [INFO] TestAgent: consul server down +49:28.772 [INFO] TestAgent: shutdown complete +49:28.772 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24257 network=tcp +49:28.772 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24257 network=udp +49:28.772 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24258 network=tcp +49:29.273 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.273 [INFO] TestAgent: Endpoints down --- PASS: TestConfig_Delete (0.66s) -=== CONT TestAgentConnectCARoots_list -2021-10-31T23:13:17.541Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.621Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.634 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b984498c-40e6-a0b3-aabd-51e33fa5ce4f Address:127.0.0.1:16552}]" -13:17.635 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b984498c-40e6-a0b3-aabd-51e33fa5ce4f.dc1 127.0.0.1 -13:17.635 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b984498c-40e6-a0b3-aabd-51e33fa5ce4f 127.0.0.1 -2021-10-31T23:13:17.635Z [INFO] agent.router: Initializing LAN area manager -13:17.636 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16498 network=udp -13:17.636 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16552 [Follower]" leader= -13:17.636 [INFO] TestAgent.server: Adding LAN server: server="Node-b984498c-40e6-a0b3-aabd-51e33fa5ce4f (Addr: tcp/127.0.0.1:16552) (DC: dc1)" -13:17.636 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b984498c-40e6-a0b3-aabd-51e33fa5ce4f.dc1 area=wan -13:17.636 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16498 network=tcp -13:17.637 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16499 network=tcp -13:17.637 [INFO] TestAgent: started state syncer -13:17.637 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16553 network=tcp -13:17.678 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:17.678 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16552 [Candidate]" term=2 -13:17.679 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:17.679 [DEBUG] TestAgent.server.raft: vote granted: from=b984498c-40e6-a0b3-aabd-51e33fa5ce4f term=2 tally=1 -13:17.679 [INFO] TestAgent.server.raft: election won: tally=1 -13:17.679 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16552 [Leader]" -13:17.682 [INFO] TestAgent.server: cluster leadership acquired -13:17.683 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16552 -13:17.685 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:17.685 [INFO] TestAgent.server: New leader elected: payload=Node-b984498c-40e6-a0b3-aabd-51e33fa5ce4f -13:17.700 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:17.700 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:17.700 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:17.700 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:17.700 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:17.700 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b984498c-40e6-a0b3-aabd-51e33fa5ce4f -13:17.700 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b984498c-40e6-a0b3-aabd-51e33fa5ce4f -13:17.701 [INFO] TestAgent.server: federation state anti-entropy synced -13:17.901 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:17.902 [INFO] TestAgent: Synced node info -13:18.019 [INFO] TestAgent: Requesting shutdown -13:18.019 [INFO] TestAgent.server: shutting down server -13:18.019 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:18.019 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:18.019 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:18.019 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:18.019 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:18.019 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:18.019 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:18.019 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:18.019 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:18.019 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:18.019 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:18.019Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:18.019Z [INFO] agent.router.manager: shutting down -13:18.019 [INFO] TestAgent: consul server down -13:18.019 [INFO] TestAgent: shutdown complete -13:18.019 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16498 network=tcp -13:18.019 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16498 network=udp -13:18.019 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16499 network=tcp -13:18.527 [INFO] TestAgent: Waiting for endpoints to shut down -13:18.527 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Apply_TerminatingGateway (1.04s) -=== CONT TestCatalogNodeServices_Filter -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=true -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:17.626Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.684Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.691 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e5948a91-8572-3e68-4343-0e8fe8c923ce Address:127.0.0.1:16489}]" -13:17.693 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e5948a91-8572-3e68-4343-0e8fe8c923ce.dc1 127.0.0.1 -13:17.694 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16489 [Follower]" leader= -13:17.704 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e5948a91-8572-3e68-4343-0e8fe8c923ce 127.0.0.1 -2021-10-31T23:13:17.704Z [INFO] agent.router: Initializing LAN area manager -13:17.706 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e5948a91-8572-3e68-4343-0e8fe8c923ce.dc1 area=wan -13:17.706 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16554 network=tcp -13:17.710 [INFO] TestAgent.server: Adding LAN server: server="Node-e5948a91-8572-3e68-4343-0e8fe8c923ce (Addr: tcp/127.0.0.1:16489) (DC: dc1)" -13:17.718 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16554 network=udp -13:17.744 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:17.744 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16489 [Candidate]" term=2 -13:17.774 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16555 network=tcp -13:17.774 [INFO] TestAgent: started state syncer -13:17.774 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16490 network=tcp -13:17.782 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:17.782 [DEBUG] TestAgent.server.raft: vote granted: from=e5948a91-8572-3e68-4343-0e8fe8c923ce term=2 tally=1 -13:17.782 [INFO] TestAgent.server.raft: election won: tally=1 -13:17.782 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16489 [Leader]" -13:17.782 [INFO] TestAgent.server: cluster leadership acquired -13:17.782 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16489 -13:17.782 [INFO] TestAgent.server: New leader elected: payload=Node-e5948a91-8572-3e68-4343-0e8fe8c923ce -13:17.783 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:17.784 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:17.784 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:17.784 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:17.784 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:17.784 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:17.784 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e5948a91-8572-3e68-4343-0e8fe8c923ce -13:17.784 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e5948a91-8572-3e68-4343-0e8fe8c923ce -13:17.785 [INFO] TestAgent.server: federation state anti-entropy synced -13:17.813 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:17.814 [INFO] TestAgent: Synced node info -13:18.102 [INFO] TestAgent: Requesting shutdown -13:18.102 [INFO] TestAgent.server: shutting down server -13:18.102 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:18.102 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:18.102 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:18.102 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:18.102 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:18.102 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:18.102 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:18.102 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:18.102 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:18.102 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:18.103 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:18.103Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:18.103Z [INFO] agent.router.manager: shutting down -13:18.103 [INFO] TestAgent: consul server down -13:18.103 [INFO] TestAgent: shutdown complete -13:18.103 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16554 network=tcp -13:18.103 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16554 network=udp -13:18.103 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16555 network=tcp -13:18.605 [INFO] TestAgent: Waiting for endpoints to shut down -13:18.605 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Apply (1.02s) -=== CONT TestCatalogNodeServiceList -2021-10-31T23:13:17.796Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.847Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.848 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:04eaa438-2533-2dad-bf63-a62d1be7fbe2 Address:127.0.0.1:16545}]" -13:17.848 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16545 [Follower]" leader= -13:17.849 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-04eaa438-2533-2dad-bf63-a62d1be7fbe2.dc1 127.0.0.1 -13:17.849 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-04eaa438-2533-2dad-bf63-a62d1be7fbe2 127.0.0.1 -2021-10-31T23:13:17.849Z [INFO] agent.router: Initializing LAN area manager -13:17.850 [INFO] TestAgent.server: Adding LAN server: server="Node-04eaa438-2533-2dad-bf63-a62d1be7fbe2 (Addr: tcp/127.0.0.1:16545) (DC: dc1)" -13:17.862 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-04eaa438-2533-2dad-bf63-a62d1be7fbe2.dc1 area=wan -13:17.862 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16512 network=udp -13:17.862 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16512 network=tcp -13:17.862 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16513 network=tcp -13:17.863 [INFO] TestAgent: started state syncer -13:17.863 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16546 network=tcp -13:17.917 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:17.917 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16545 [Candidate]" term=2 -13:17.918 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:17.918 [DEBUG] TestAgent.server.raft: vote granted: from=04eaa438-2533-2dad-bf63-a62d1be7fbe2 term=2 tally=1 -13:17.918 [INFO] TestAgent.server.raft: election won: tally=1 -13:17.918 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16545 [Leader]" -13:17.918 [INFO] TestAgent.server: cluster leadership acquired -13:17.918 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16545 -13:17.918 [INFO] TestAgent.server: New leader elected: payload=Node-04eaa438-2533-2dad-bf63-a62d1be7fbe2 -13:17.919 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:17.920 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:17.920 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:17.920 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:17.920 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:17.920 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:17.920 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-04eaa438-2533-2dad-bf63-a62d1be7fbe2 -13:17.920 [INFO] TestAgent.server: member joined, marking health alive: member=Node-04eaa438-2533-2dad-bf63-a62d1be7fbe2 -13:17.922 [INFO] TestAgent.server: federation state anti-entropy synced -13:18.158 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:18.158 [INFO] TestAgent: Synced node info -13:18.158 [DEBUG] TestAgent: Node info in sync -13:18.296 [INFO] TestAgent.server.connect: CA provider config updated -13:18.296 [INFO] TestAgent: Requesting shutdown -13:18.296 [INFO] TestAgent.server: shutting down server -13:18.296 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:18.296 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:18.296 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:18.296 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:18.296 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:18.296 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:18.296 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:18.296 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:18.296 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:18.296 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:18.296 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:18.296Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:18.296Z [INFO] agent.router.manager: shutting down -13:18.297 [INFO] TestAgent: consul server down -13:18.297 [INFO] TestAgent: shutdown complete -13:18.297 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16512 network=tcp -13:18.297 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16512 network=udp -13:18.297 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16513 network=tcp -13:18.801 [INFO] TestAgent: Waiting for endpoints to shut down -13:18.801 [INFO] TestAgent: Endpoints down -=== RUN TestConnectCAConfig/force_without_cross_sign_CamelCase -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:17.930Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.978Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.979 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5b4840f6-80c6-f87e-d014-68bdb675a130 Address:127.0.0.1:16531}]" -13:17.980 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5b4840f6-80c6-f87e-d014-68bdb675a130.dc1 127.0.0.1 -13:17.980 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16531 [Follower]" leader= -13:17.981 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5b4840f6-80c6-f87e-d014-68bdb675a130 127.0.0.1 -2021-10-31T23:13:17.981Z [INFO] agent.router: Initializing LAN area manager -13:17.981 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16533 network=udp -13:17.981 [INFO] TestAgent.server: Adding LAN server: server="Node-5b4840f6-80c6-f87e-d014-68bdb675a130 (Addr: tcp/127.0.0.1:16531) (DC: dc1)" -13:17.981 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16533 network=tcp -13:17.981 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5b4840f6-80c6-f87e-d014-68bdb675a130.dc1 area=wan -13:17.994 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16534 network=tcp -13:17.994 [INFO] TestAgent: started state syncer -13:17.994 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16532 network=tcp -13:18.050 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:18.050 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16531 [Candidate]" term=2 -13:18.050 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:18.050 [DEBUG] TestAgent.server.raft: vote granted: from=5b4840f6-80c6-f87e-d014-68bdb675a130 term=2 tally=1 -13:18.050 [INFO] TestAgent.server.raft: election won: tally=1 -13:18.050 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16531 [Leader]" -13:18.050 [INFO] TestAgent.server: cluster leadership acquired -13:18.050 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16531 -13:18.051 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:18.054 [INFO] TestAgent.server: New leader elected: payload=Node-5b4840f6-80c6-f87e-d014-68bdb675a130 -13:18.074 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:18.074 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:18.074 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:18.074 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:18.074 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:18.074 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5b4840f6-80c6-f87e-d014-68bdb675a130 -13:18.074 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5b4840f6-80c6-f87e-d014-68bdb675a130 -13:18.075 [INFO] TestAgent.server: federation state anti-entropy synced -13:18.290 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:18.290 [INFO] TestAgent: Synced node info -13:18.290 [DEBUG] TestAgent: Node info in sync -13:18.393 [INFO] TestAgent: Requesting shutdown -13:18.393 [INFO] TestAgent.server: shutting down server -13:18.393 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:18.393 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:18.393 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:18.393 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:18.393 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:18.393 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:18.393 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:18.393 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:18.393 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:18.393 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:18.394 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:18.394Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:18.394Z [INFO] agent.router.manager: shutting down -13:18.394 [INFO] TestAgent: consul server down -13:18.394 [INFO] TestAgent: shutdown complete -13:18.394 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16533 network=tcp -13:18.394 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16533 network=udp -13:18.394 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16534 network=tcp -13:18.894 [INFO] TestAgent: Waiting for endpoints to shut down -13:18.894 [INFO] TestAgent: Endpoints down ---- PASS: TestConfig_Get (1.02s) +=== CONT TestCatalogServiceNodes_WanTranslation +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:28.623Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.641Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.644 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8d9ddfcc-dae3-2a60-dcaa-88a49d51455a Address:127.0.0.1:24255}]" +49:28.644 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24255 [Follower]" leader= +49:28.645 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8d9ddfcc-dae3-2a60-dcaa-88a49d51455a.dc1 127.0.0.1 +49:28.646 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8d9ddfcc-dae3-2a60-dcaa-88a49d51455a 127.0.0.1 +2022-12-04T05:49:28.646Z [INFO] agent.router: Initializing LAN area manager +49:28.646 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24236 network=udp +49:28.647 [INFO] TestAgent.server: Adding LAN server: server="Node-8d9ddfcc-dae3-2a60-dcaa-88a49d51455a (Addr: tcp/127.0.0.1:24255) (DC: dc1)" +49:28.647 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8d9ddfcc-dae3-2a60-dcaa-88a49d51455a.dc1 area=wan +49:28.648 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24236 network=tcp +49:28.649 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24237 network=tcp +49:28.650 [INFO] TestAgent: started state syncer +49:28.650 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24256 network=tcp +49:28.685 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.685 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24255 [Candidate]" term=2 +49:28.685 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.685 [DEBUG] TestAgent.server.raft: vote granted: from=8d9ddfcc-dae3-2a60-dcaa-88a49d51455a term=2 tally=1 +49:28.685 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.685 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24255 [Leader]" +49:28.686 [INFO] TestAgent.server: cluster leadership acquired +49:28.686 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24255 +49:28.687 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.688 [INFO] TestAgent.server: New leader elected: payload=Node-8d9ddfcc-dae3-2a60-dcaa-88a49d51455a +49:28.689 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.689 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.689 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.689 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.689 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.689 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8d9ddfcc-dae3-2a60-dcaa-88a49d51455a +49:28.689 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8d9ddfcc-dae3-2a60-dcaa-88a49d51455a +49:28.691 [INFO] TestAgent.server: federation state anti-entropy synced +49:28.750 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:28.750 [INFO] TestAgent: Synced node info +49:28.750 [DEBUG] TestAgent: Node info in sync +49:28.867 [INFO] TestAgent: Requesting shutdown +49:28.867 [INFO] TestAgent.server: shutting down server +49:28.867 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.867 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.867 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.867 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.867 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.867 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.867 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.867 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.867 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.867 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.867 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:28.867Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.867Z [INFO] agent.router.manager: shutting down +49:28.867 [INFO] TestAgent: consul server down +49:28.867 [INFO] TestAgent: shutdown complete +49:28.867 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24236 network=tcp +49:28.867 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24236 network=udp +49:28.867 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24237 network=tcp +49:29.368 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.368 [INFO] TestAgent: Endpoints down +--- PASS: TestConfig_Apply_CAS (0.76s) +=== CONT TestCatalogServiceNodes +2022-12-04T05:49:28.674Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.695Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.698 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0f3790d0-45be-4323-3fcf-b64341966351 Address:127.0.0.1:24262}]" +49:28.699 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24262 [Follower]" leader= +49:28.699 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0f3790d0-45be-4323-3fcf-b64341966351.dc1 127.0.0.1 +49:28.700 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0f3790d0-45be-4323-3fcf-b64341966351 127.0.0.1 +2022-12-04T05:49:28.700Z [INFO] agent.router: Initializing LAN area manager +49:28.701 [INFO] TestAgent.server: Adding LAN server: server="Node-0f3790d0-45be-4323-3fcf-b64341966351 (Addr: tcp/127.0.0.1:24262) (DC: dc1)" +49:28.701 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0f3790d0-45be-4323-3fcf-b64341966351.dc1 area=wan +49:28.701 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24292 network=udp +49:28.701 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24292 network=tcp +49:28.702 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24293 network=tcp +49:28.703 [INFO] TestAgent: started state syncer +49:28.703 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24263 network=tcp +49:28.739 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.739 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24262 [Candidate]" term=2 +49:28.740 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.740 [DEBUG] TestAgent.server.raft: vote granted: from=0f3790d0-45be-4323-3fcf-b64341966351 term=2 tally=1 +49:28.740 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.740 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24262 [Leader]" +49:28.740 [INFO] TestAgent.server: cluster leadership acquired +49:28.740 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24262 +49:28.740 [INFO] TestAgent.server: New leader elected: payload=Node-0f3790d0-45be-4323-3fcf-b64341966351 +49:28.741 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.744 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.744 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.744 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.744 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.744 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.744 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0f3790d0-45be-4323-3fcf-b64341966351 +49:28.744 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0f3790d0-45be-4323-3fcf-b64341966351 +49:28.745 [INFO] TestAgent.server: federation state anti-entropy synced +49:28.872 [INFO] TestAgent: Requesting shutdown +49:28.872 [INFO] TestAgent.server: shutting down server +49:28.872 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.872 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.872 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.872 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.872 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.872 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.872 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:28.872 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.872 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.872 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.872 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:28.872Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.873Z [INFO] agent.router.manager: shutting down +49:28.873 [INFO] TestAgent: consul server down +49:28.873 [INFO] TestAgent: shutdown complete +49:28.873 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24292 network=tcp +49:28.873 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24292 network=udp +49:28.873 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24293 network=tcp +49:29.376 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.376 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:28.809Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.820Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.821 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0d1b4990-f4f0-0128-2260-eb7039ed99e0 Address:127.0.0.1:23291}]" +49:28.821 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23291 [Follower]" leader= +49:28.822 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0d1b4990-f4f0-0128-2260-eb7039ed99e0.dc1 127.0.0.1 +49:28.822 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0d1b4990-f4f0-0128-2260-eb7039ed99e0 127.0.0.1 +2022-12-04T05:49:28.822Z [INFO] agent.router: Initializing LAN area manager +49:28.822 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0d1b4990-f4f0-0128-2260-eb7039ed99e0.dc1 area=wan +49:28.822 [INFO] TestAgent.server: Adding LAN server: server="Node-0d1b4990-f4f0-0128-2260-eb7039ed99e0 (Addr: tcp/127.0.0.1:23291) (DC: dc1)" +49:28.823 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24173 network=tcp +49:28.823 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24173 network=udp +49:28.823 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24174 network=tcp +49:28.823 [INFO] TestAgent: started state syncer +49:28.823 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23292 network=tcp +49:28.865 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.865 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23291 [Candidate]" term=2 +49:28.865 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.865 [DEBUG] TestAgent.server.raft: vote granted: from=0d1b4990-f4f0-0128-2260-eb7039ed99e0 term=2 tally=1 +49:28.865 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.865 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23291 [Leader]" +49:28.865 [INFO] TestAgent.server: cluster leadership acquired +49:28.866 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23291 +49:28.866 [INFO] TestAgent.server: New leader elected: payload=Node-0d1b4990-f4f0-0128-2260-eb7039ed99e0 +49:28.867 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.868 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.868 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.868 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.868 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.868 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.868 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d1b4990-f4f0-0128-2260-eb7039ed99e0 +49:28.868 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0d1b4990-f4f0-0128-2260-eb7039ed99e0 +49:28.869 [INFO] TestAgent.server: federation state anti-entropy synced +49:28.873 [INFO] TestAgent: Requesting shutdown +49:28.873 [INFO] TestAgent.server: shutting down server +49:28.873 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.873 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.873 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.873 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.873 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.873 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.873 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.873 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:28.873 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.873 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.873 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.873 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:28.873Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.873Z [INFO] agent.router.manager: shutting down +49:28.873 [INFO] TestAgent: consul server down +49:28.873 [INFO] TestAgent: shutdown complete +49:28.873 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24173 network=tcp +49:28.873 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24173 network=udp +49:28.873 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24174 network=tcp +49:29.376 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.376 [INFO] TestAgent: Endpoints down +--- PASS: TestConfig_Get (0.72s) --- PASS: TestConfig_Get/get_a_single_service_entry (0.00s) --- PASS: TestConfig_Get/list_both_service_entries (0.00s) --- PASS: TestConfig_Get/get_global_proxy_config (0.00s) --- PASS: TestConfig_Get/error_on_no_arguments (0.00s) +=== CONT TestCatalogServices_NodeMetaFilter +--- PASS: TestCatalog_GatewayServices_Terminating (0.58s) +=== CONT TestCatalogRegister_checkRegistration +2022-12-04T05:49:28.498Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.518Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.520 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c371766e-d5ed-a8cf-f939-207c40b184c5 Address:127.0.0.1:24192}]" +49:28.520 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24192 [Follower]" leader= +49:28.520 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c371766e-d5ed-a8cf-f939-207c40b184c5.dc1 127.0.0.1 +49:28.521 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c371766e-d5ed-a8cf-f939-207c40b184c5 127.0.0.1 +2022-12-04T05:49:28.521Z [INFO] agent.router: Initializing LAN area manager +49:28.521 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c371766e-d5ed-a8cf-f939-207c40b184c5.dc1 area=wan +49:28.521 [INFO] TestAgent.server: Adding LAN server: server="Node-c371766e-d5ed-a8cf-f939-207c40b184c5 (Addr: tcp/127.0.0.1:24192) (DC: dc1)" +49:28.521 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24201 network=udp +49:28.521 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24201 network=tcp +49:28.522 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24202 network=tcp +49:28.522 [INFO] TestAgent: started state syncer +49:28.522 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24193 network=tcp +49:28.562 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.562 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24192 [Candidate]" term=2 +49:28.563 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.563 [DEBUG] TestAgent.server.raft: vote granted: from=c371766e-d5ed-a8cf-f939-207c40b184c5 term=2 tally=1 +49:28.563 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.563 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24192 [Leader]" +49:28.563 [INFO] TestAgent.server: cluster leadership acquired +49:28.563 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24192 +49:28.563 [INFO] TestAgent.server: New leader elected: payload=Node-c371766e-d5ed-a8cf-f939-207c40b184c5 +49:28.563 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.565 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.565 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.565 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.565 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.565 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.565 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c371766e-d5ed-a8cf-f939-207c40b184c5 +49:28.565 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c371766e-d5ed-a8cf-f939-207c40b184c5 +49:28.566 [INFO] TestAgent.server: federation state anti-entropy synced +49:28.684 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:28.685 [INFO] TestAgent: Synced node info +49:28.909 [INFO] TestAgent.server.connect: CA provider config updated +49:28.909 [INFO] TestAgent: Requesting shutdown +49:28.909 [INFO] TestAgent.server: shutting down server +49:28.909 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.909 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.909 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.909 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.909 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.909 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.909 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.909 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.909 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:28.910 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.910 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:49:28.910Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.910Z [INFO] agent.router.manager: shutting down +49:28.910 [INFO] TestAgent: consul server down +49:28.910 [INFO] TestAgent: shutdown complete +49:28.910 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24201 network=tcp +49:28.910 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24201 network=udp +49:28.910 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24202 network=tcp +49:29.410 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.417 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:28.819Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.829Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.831 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:53a72316-2804-9528-4ee6-cb99630fe7d1 Address:127.0.0.1:24276}]" +49:28.831 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24276 [Follower]" leader= +49:28.831 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-53a72316-2804-9528-4ee6-cb99630fe7d1.dc1 127.0.0.1 +49:28.832 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-53a72316-2804-9528-4ee6-cb99630fe7d1 127.0.0.1 +2022-12-04T05:49:28.832Z [INFO] agent.router: Initializing LAN area manager +49:28.832 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-53a72316-2804-9528-4ee6-cb99630fe7d1.dc1 area=wan +49:28.832 [INFO] TestAgent.server: Adding LAN server: server="Node-53a72316-2804-9528-4ee6-cb99630fe7d1 (Addr: tcp/127.0.0.1:24276) (DC: dc1)" +49:28.832 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23293 network=udp +49:28.832 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23293 network=tcp +49:28.833 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23294 network=tcp +49:28.833 [INFO] TestAgent: started state syncer +49:28.833 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24277 network=tcp +49:28.889 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.889 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24276 [Candidate]" term=2 +49:28.889 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.889 [DEBUG] TestAgent.server.raft: vote granted: from=53a72316-2804-9528-4ee6-cb99630fe7d1 term=2 tally=1 +49:28.889 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.889 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24276 [Leader]" +49:28.889 [INFO] TestAgent.server: cluster leadership acquired +49:28.889 [INFO] TestAgent.server: New leader elected: payload=Node-53a72316-2804-9528-4ee6-cb99630fe7d1 +49:28.889 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24276 +49:28.890 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.891 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.891 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.891 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.891 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.891 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.891 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-53a72316-2804-9528-4ee6-cb99630fe7d1 +49:28.891 [INFO] TestAgent.server: member joined, marking health alive: member=Node-53a72316-2804-9528-4ee6-cb99630fe7d1 +49:28.891 [INFO] TestAgent.server: federation state anti-entropy synced +49:28.911 [INFO] TestAgent: Requesting shutdown +49:28.911 [INFO] TestAgent.server: shutting down server +49:28.911 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.911 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.911 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.911 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.911 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.911 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.911 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.911 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:28.911 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.911 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.911 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.912 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:28.912Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.912Z [INFO] agent.router.manager: shutting down +49:28.912 [INFO] TestAgent: consul server down +49:28.912 [INFO] TestAgent: shutdown complete +49:28.912 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23293 network=tcp +49:28.912 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23293 network=udp +49:28.912 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23294 network=tcp +49:29.412 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.417 [INFO] TestAgent: Endpoints down +=== RUN TestConnectCAConfig/force_without_cross_sign_CamelCase +--- PASS: TestCatalogNodeServiceList (0.61s) +=== CONT TestCatalogServices +2022-12-04T05:49:28.752Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.766Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.768 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6e851e30-70f8-4748-e2a8-4a74ff3c5fb4 Address:127.0.0.1:24171}]" +49:28.768 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24171 [Follower]" leader= +49:28.768 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6e851e30-70f8-4748-e2a8-4a74ff3c5fb4.dc1 127.0.0.1 +49:28.769 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6e851e30-70f8-4748-e2a8-4a74ff3c5fb4 127.0.0.1 +2022-12-04T05:49:28.769Z [INFO] agent.router: Initializing LAN area manager +49:28.769 [INFO] TestAgent.server: Adding LAN server: server="Node-6e851e30-70f8-4748-e2a8-4a74ff3c5fb4 (Addr: tcp/127.0.0.1:24171) (DC: dc1)" +49:28.769 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6e851e30-70f8-4748-e2a8-4a74ff3c5fb4.dc1 area=wan +49:28.769 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24229 network=udp +49:28.769 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24229 network=tcp +49:28.770 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24230 network=tcp +49:28.770 [INFO] TestAgent: started state syncer +49:28.770 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24172 network=tcp +49:28.814 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.814 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24171 [Candidate]" term=2 +49:28.814 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.814 [DEBUG] TestAgent.server.raft: vote granted: from=6e851e30-70f8-4748-e2a8-4a74ff3c5fb4 term=2 tally=1 +49:28.815 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.815 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24171 [Leader]" +49:28.815 [INFO] TestAgent.server: cluster leadership acquired +49:28.815 [INFO] TestAgent.server: New leader elected: payload=Node-6e851e30-70f8-4748-e2a8-4a74ff3c5fb4 +49:28.815 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24171 +49:28.816 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.817 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.817 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.817 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.817 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.817 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.817 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6e851e30-70f8-4748-e2a8-4a74ff3c5fb4 +49:28.817 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6e851e30-70f8-4748-e2a8-4a74ff3c5fb4 +49:28.818 [INFO] TestAgent.server: federation state anti-entropy synced +49:28.885 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:28.886 [INFO] TestAgent: Synced node info +49:28.925 [INFO] TestAgent: Requesting shutdown +49:28.925 [INFO] TestAgent.server: shutting down server +49:28.925 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.925 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.925 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.925 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.925 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.925 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.925 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.925 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.925 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:28.925 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.925 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:49:28.925Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.925Z [INFO] agent.router.manager: shutting down +49:28.925 [INFO] TestAgent: consul server down +49:28.925 [INFO] TestAgent: shutdown complete +49:28.925 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24229 network=tcp +49:28.925 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24229 network=udp +49:28.925 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24230 network=tcp +49:29.425 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.425 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodeServices_Filter (0.70s) +=== CONT TestCatalogNodes_DistanceSort +2022-12-04T05:49:28.633Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.661Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.663 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a09f5fee-3a46-7df4-a5d2-767878ed2af4 Address:127.0.0.1:24290}]" +49:28.663 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24290 [Follower]" leader= +49:28.663 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a09f5fee-3a46-7df4-a5d2-767878ed2af4.dc1 127.0.0.1 +49:28.664 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a09f5fee-3a46-7df4-a5d2-767878ed2af4 127.0.0.1 +2022-12-04T05:49:28.664Z [INFO] agent.router: Initializing LAN area manager +49:28.664 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24250 network=udp +49:28.664 [INFO] TestAgent.server: Adding LAN server: server="Node-a09f5fee-3a46-7df4-a5d2-767878ed2af4 (Addr: tcp/127.0.0.1:24290) (DC: dc1)" +49:28.664 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a09f5fee-3a46-7df4-a5d2-767878ed2af4.dc1 area=wan +49:28.664 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24250 network=tcp +49:28.665 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24251 network=tcp +49:28.665 [INFO] TestAgent: started state syncer +49:28.665 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24291 network=tcp +49:28.728 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.729 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24290 [Candidate]" term=2 +49:28.729 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.729 [DEBUG] TestAgent.server.raft: vote granted: from=a09f5fee-3a46-7df4-a5d2-767878ed2af4 term=2 tally=1 +49:28.729 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.729 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24290 [Leader]" +49:28.729 [INFO] TestAgent.server: cluster leadership acquired +49:28.729 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24290 +49:28.731 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.732 [INFO] TestAgent.server: New leader elected: payload=Node-a09f5fee-3a46-7df4-a5d2-767878ed2af4 +49:28.738 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.738 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.738 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.738 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.738 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.738 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a09f5fee-3a46-7df4-a5d2-767878ed2af4 +49:28.738 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a09f5fee-3a46-7df4-a5d2-767878ed2af4 +49:28.739 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.006 [INFO] TestAgent: Requesting shutdown +49:29.006 [INFO] TestAgent.server: shutting down server +49:29.006 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.006 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.006 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.006 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.006 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.006 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.006 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.006 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:29.006 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.006 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:29.006 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:29.006 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:49:29.007Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.007Z [INFO] agent.router.manager: shutting down +49:29.007 [INFO] TestAgent: consul server down +49:29.007 [INFO] TestAgent: shutdown complete +49:29.007 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24250 network=tcp +49:29.007 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24250 network=udp +49:29.007 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24251 network=tcp +49:29.508 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.508 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodeServices_ConnectProxy (0.89s) +=== CONT TestCatalogServiceNodes_Filter +2022-12-04T05:49:28.836Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.846Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.847 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:609a84d1-d259-0c8c-6fb3-4efda1282f6e Address:127.0.0.1:23312}]" +49:28.847 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23312 [Follower]" leader= +49:28.848 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-609a84d1-d259-0c8c-6fb3-4efda1282f6e.dc1 127.0.0.1 +49:28.848 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-609a84d1-d259-0c8c-6fb3-4efda1282f6e 127.0.0.1 +2022-12-04T05:49:28.848Z [INFO] agent.router: Initializing LAN area manager +49:28.848 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-609a84d1-d259-0c8c-6fb3-4efda1282f6e.dc1 area=wan +49:28.848 [INFO] TestAgent.server: Adding LAN server: server="Node-609a84d1-d259-0c8c-6fb3-4efda1282f6e (Addr: tcp/127.0.0.1:23312) (DC: dc1)" +49:28.849 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24278 network=udp +49:28.849 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24278 network=tcp +49:28.849 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24279 network=tcp +49:28.849 [INFO] TestAgent: started state syncer +49:28.849 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23313 network=tcp +49:28.892 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.892 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23312 [Candidate]" term=2 +49:28.892 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.892 [DEBUG] TestAgent.server.raft: vote granted: from=609a84d1-d259-0c8c-6fb3-4efda1282f6e term=2 tally=1 +49:28.892 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.892 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23312 [Leader]" +49:28.892 [INFO] TestAgent.server: cluster leadership acquired +49:28.893 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23312 +49:28.893 [INFO] TestAgent.server: New leader elected: payload=Node-609a84d1-d259-0c8c-6fb3-4efda1282f6e +49:28.898 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.900 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.900 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.900 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.900 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.900 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.900 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-609a84d1-d259-0c8c-6fb3-4efda1282f6e +49:28.900 [INFO] TestAgent.server: member joined, marking health alive: member=Node-609a84d1-d259-0c8c-6fb3-4efda1282f6e +49:28.901 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.032 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:29.033 [INFO] TestAgent: Synced node info +49:29.034 [INFO] TestAgent: Requesting shutdown +49:29.034 [INFO] TestAgent.server: shutting down server +49:29.034 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.034 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.034 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.034 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.034 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.034 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.034 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.034 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.034 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.034 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:29.034 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:29.035Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.035Z [INFO] agent.router.manager: shutting down +49:29.035 [INFO] TestAgent: consul server down +49:29.035 [INFO] TestAgent: shutdown complete +49:29.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24278 network=tcp +49:29.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24278 network=udp +49:29.035 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24279 network=tcp +49:29.536 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.536 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodeServices (0.71s) +=== CONT TestCatalogNodes_Blocking +[WARN] freeport: 10 out of 66 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=true +2022-12-04T05:49:28.716Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.728Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.731 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:950a7153-2d0f-e801-9548-d17935837584 Address:127.0.0.1:24227}]" +49:28.732 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-950a7153-2d0f-e801-9548-d17935837584.dc1 127.0.0.1 +49:28.732 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24227 [Follower]" leader= +49:28.732 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-950a7153-2d0f-e801-9548-d17935837584 127.0.0.1 +2022-12-04T05:49:28.732Z [INFO] agent.router: Initializing LAN area manager +49:28.733 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-950a7153-2d0f-e801-9548-d17935837584.dc1 area=wan +49:28.733 [INFO] TestAgent.server: Adding LAN server: server="Node-950a7153-2d0f-e801-9548-d17935837584 (Addr: tcp/127.0.0.1:24227) (DC: dc1)" +49:28.733 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24264 network=udp +49:28.733 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24264 network=tcp +49:28.734 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24265 network=tcp +49:28.735 [INFO] TestAgent: started state syncer +49:28.735 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24228 network=tcp +49:28.800 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.800 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24227 [Candidate]" term=2 +49:28.800 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.800 [DEBUG] TestAgent.server.raft: vote granted: from=950a7153-2d0f-e801-9548-d17935837584 term=2 tally=1 +49:28.800 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.800 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24227 [Leader]" +49:28.800 [INFO] TestAgent.server: cluster leadership acquired +49:28.801 [INFO] TestAgent.server: New leader elected: payload=Node-950a7153-2d0f-e801-9548-d17935837584 +49:28.801 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24227 +49:28.801 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.803 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.803 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.803 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.803 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.803 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.803 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-950a7153-2d0f-e801-9548-d17935837584 +49:28.803 [INFO] TestAgent.server: member joined, marking health alive: member=Node-950a7153-2d0f-e801-9548-d17935837584 +49:28.804 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.071 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:29.071 [INFO] TestAgent: Synced node info +49:29.106 [INFO] TestAgent: Requesting shutdown +49:29.106 [INFO] TestAgent.server: shutting down server +49:29.106 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.106 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.106 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.106 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.106 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.106 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.106 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:29.106 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.106 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.106 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.106 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:29.106Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.106Z [INFO] agent.router.manager: shutting down +49:29.107 [INFO] TestAgent: consul server down +49:29.107 [INFO] TestAgent: shutdown complete +49:29.107 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24264 network=tcp +49:29.107 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24264 network=udp +49:29.107 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24265 network=tcp +49:29.607 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.607 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalog_GatewayServices_Ingress (0.90s) +=== CONT TestCatalogNodes_WanTranslation +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=1)_compressed=true +2022-12-04T05:49:28.981Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.991Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.992 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:de3db1da-c3cb-67e6-4f89-406347ffd5f4 Address:127.0.0.1:23361}]" +49:28.992 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23361 [Follower]" leader= +49:28.993 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-de3db1da-c3cb-67e6-4f89-406347ffd5f4.dc1 127.0.0.1 +49:28.993 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-de3db1da-c3cb-67e6-4f89-406347ffd5f4 127.0.0.1 +2022-12-04T05:49:28.993Z [INFO] agent.router: Initializing LAN area manager +49:28.994 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23314 network=udp +49:28.994 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-de3db1da-c3cb-67e6-4f89-406347ffd5f4.dc1 area=wan +49:28.994 [INFO] TestAgent.server: Adding LAN server: server="Node-de3db1da-c3cb-67e6-4f89-406347ffd5f4 (Addr: tcp/127.0.0.1:23361) (DC: dc1)" +49:28.994 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23314 network=tcp +49:28.994 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23315 network=tcp +49:28.994 [INFO] TestAgent: started state syncer +49:28.994 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23362 network=tcp +49:29.038 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.038 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23361 [Candidate]" term=2 +49:29.046 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.046 [DEBUG] TestAgent.server.raft: vote granted: from=de3db1da-c3cb-67e6-4f89-406347ffd5f4 term=2 tally=1 +49:29.046 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.046 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23361 [Leader]" +49:29.046 [INFO] TestAgent.server: cluster leadership acquired +49:29.046 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23361 +49:29.046 [INFO] TestAgent.server: New leader elected: payload=Node-de3db1da-c3cb-67e6-4f89-406347ffd5f4 +49:29.047 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.048 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.048 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.048 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.048 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.048 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.048 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-de3db1da-c3cb-67e6-4f89-406347ffd5f4 +49:29.048 [INFO] TestAgent.server: member joined, marking health alive: member=Node-de3db1da-c3cb-67e6-4f89-406347ffd5f4 +49:29.049 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.099 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:29.100 [INFO] TestAgent: Synced node info +49:29.290 [INFO] TestAgent: Requesting shutdown +49:29.290 [INFO] TestAgent.server: shutting down server +49:29.290 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.290 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.290 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.290 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.290 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.291 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.291 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.291 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:29.291 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:29.291Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.291Z [INFO] agent.router.manager: shutting down +49:29.291 [INFO] TestAgent: consul server down +49:29.291 [INFO] TestAgent: shutdown complete +49:29.291 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23314 network=tcp +49:29.291 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23314 network=udp +49:29.291 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23315 network=tcp +49:29.794 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.794 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogServiceNodes_ConnectProxy (0.83s) +=== CONT TestCatalogNodes_Filter +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:29.233Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:29.248Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:29.250 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:30e9eba2-b766-d480-8c4c-bf301e909d91 Address:127.0.0.1:23333}]" +49:29.250 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23333 [Follower]" leader= +49:29.250 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-30e9eba2-b766-d480-8c4c-bf301e909d91.dc1 127.0.0.1 +49:29.251 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-30e9eba2-b766-d480-8c4c-bf301e909d91 127.0.0.1 +2022-12-04T05:49:29.251Z [INFO] agent.router: Initializing LAN area manager +49:29.251 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-30e9eba2-b766-d480-8c4c-bf301e909d91.dc1 area=wan +49:29.251 [INFO] TestAgent.server: Adding LAN server: server="Node-30e9eba2-b766-d480-8c4c-bf301e909d91 (Addr: tcp/127.0.0.1:23333) (DC: dc1)" +49:29.251 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24285 network=udp +49:29.251 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24285 network=tcp +49:29.252 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24286 network=tcp +49:29.252 [INFO] TestAgent: started state syncer +49:29.252 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23334 network=tcp +49:29.321 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.321 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23333 [Candidate]" term=2 +49:29.321 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.321 [DEBUG] TestAgent.server.raft: vote granted: from=30e9eba2-b766-d480-8c4c-bf301e909d91 term=2 tally=1 +49:29.321 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.321 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23333 [Leader]" +49:29.321 [INFO] TestAgent.server: cluster leadership acquired +49:29.321 [INFO] TestAgent.server: New leader elected: payload=Node-30e9eba2-b766-d480-8c4c-bf301e909d91 +49:29.321 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23333 +49:29.322 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.323 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.323 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.323 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.323 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.323 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.323 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-30e9eba2-b766-d480-8c4c-bf301e909d91 +49:29.323 [INFO] TestAgent.server: member joined, marking health alive: member=Node-30e9eba2-b766-d480-8c4c-bf301e909d91 +49:29.324 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.458 [INFO] TestAgent: Requesting shutdown +49:29.458 [INFO] TestAgent.server: shutting down server +49:29.458 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.458 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.458 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.458 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.458 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.458 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.458 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.458 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:29.458 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.458 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.458 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:29.458 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:29.459Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.459Z [INFO] agent.router.manager: shutting down +49:29.460 [INFO] TestAgent: consul server down +49:29.460 [INFO] TestAgent: shutdown complete +49:29.460 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24285 network=tcp +49:29.460 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24285 network=udp +49:29.460 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24286 network=tcp +49:29.960 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.960 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogConnectServiceNodes_good (0.74s) +=== CONT TestCatalogNodes_MetaFilter +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:29.431Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:29.453Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:29.457 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b9ec20ec-9091-be05-567f-f0413327ff1d Address:127.0.0.1:23305}]" +49:29.458 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23305 [Follower]" leader= +49:29.459 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b9ec20ec-9091-be05-567f-f0413327ff1d.dc1 127.0.0.1 +49:29.459 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b9ec20ec-9091-be05-567f-f0413327ff1d 127.0.0.1 +2022-12-04T05:49:29.459Z [INFO] agent.router: Initializing LAN area manager +49:29.460 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23349 network=udp +49:29.461 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23349 network=tcp +49:29.461 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b9ec20ec-9091-be05-567f-f0413327ff1d.dc1 area=wan +49:29.461 [INFO] TestAgent.server: Adding LAN server: server="Node-b9ec20ec-9091-be05-567f-f0413327ff1d (Addr: tcp/127.0.0.1:23305) (DC: dc1)" +49:29.464 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23350 network=tcp +49:29.465 [INFO] TestAgent: started state syncer +49:29.465 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23306 network=tcp +49:29.508 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.508 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23305 [Candidate]" term=2 +49:29.508 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.508 [DEBUG] TestAgent.server.raft: vote granted: from=b9ec20ec-9091-be05-567f-f0413327ff1d term=2 tally=1 +49:29.508 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.508 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23305 [Leader]" +49:29.509 [INFO] TestAgent.server: cluster leadership acquired +49:29.509 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23305 +49:29.510 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.510 [INFO] TestAgent.server: New leader elected: payload=Node-b9ec20ec-9091-be05-567f-f0413327ff1d +49:29.512 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.512 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.512 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.512 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.512 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.512 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b9ec20ec-9091-be05-567f-f0413327ff1d +49:29.512 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b9ec20ec-9091-be05-567f-f0413327ff1d +49:29.513 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.538 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:29.538 [INFO] TestAgent: Synced node info +49:29.597 [INFO] TestAgent.server.connect: CA provider config updated +49:29.597 [INFO] TestAgent: Requesting shutdown +49:29.597 [INFO] TestAgent.server: shutting down server +49:29.597 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.597 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.597 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.597 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.597 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.597 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.597 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.597 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.597 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.597 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:29.597 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:29.597Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.597Z [INFO] agent.router.manager: shutting down +49:29.597 [INFO] TestAgent: consul server down +49:29.597 [INFO] TestAgent: shutdown complete +49:29.597 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23349 network=tcp +49:29.597 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23349 network=udp +49:29.597 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23350 network=tcp +49:30.097 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.097 [INFO] TestAgent: Endpoints down +=== RUN TestConnectCAConfig/force_without_cross_sign_snake_case +2022-12-04T05:49:29.398Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:29.424Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:29.426 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:92e0d1e8-8ee5-7df6-e7ec-243db979beb6 Address:127.0.0.1:23347}]" +49:29.426 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23347 [Follower]" leader= +49:29.427 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-92e0d1e8-8ee5-7df6-e7ec-243db979beb6.dc1 127.0.0.1 +49:29.428 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-92e0d1e8-8ee5-7df6-e7ec-243db979beb6 127.0.0.1 +2022-12-04T05:49:29.428Z [INFO] agent.router: Initializing LAN area manager +49:29.428 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23384 network=udp +49:29.428 [INFO] TestAgent.server: Adding LAN server: server="Node-92e0d1e8-8ee5-7df6-e7ec-243db979beb6 (Addr: tcp/127.0.0.1:23347) (DC: dc1)" +49:29.428 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-92e0d1e8-8ee5-7df6-e7ec-243db979beb6.dc1 area=wan +49:29.429 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23384 network=tcp +49:29.429 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23385 network=tcp +49:29.429 [INFO] TestAgent: started state syncer +49:29.430 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23348 network=tcp +49:29.473 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.473 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23347 [Candidate]" term=2 +49:29.474 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.474 [DEBUG] TestAgent.server.raft: vote granted: from=92e0d1e8-8ee5-7df6-e7ec-243db979beb6 term=2 tally=1 +49:29.474 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.474 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23347 [Leader]" +49:29.474 [INFO] TestAgent.server: cluster leadership acquired +49:29.474 [INFO] TestAgent.server: New leader elected: payload=Node-92e0d1e8-8ee5-7df6-e7ec-243db979beb6 +49:29.474 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23347 +49:29.475 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.477 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.477 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.477 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.477 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.477 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.478 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-92e0d1e8-8ee5-7df6-e7ec-243db979beb6 +49:29.478 [INFO] TestAgent.server: member joined, marking health alive: member=Node-92e0d1e8-8ee5-7df6-e7ec-243db979beb6 +49:29.479 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.588 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:29.588 [INFO] TestAgent: Synced node info +49:29.588 [DEBUG] TestAgent: Node info in sync +49:29.623 [INFO] TestAgent: Requesting shutdown +49:29.623 [INFO] TestAgent.server: shutting down server +49:29.623 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.623 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.623 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.623 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.623 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.623 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.623 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.623 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.623 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.623 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:29.624 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:29.624Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.624Z [INFO] agent.router.manager: shutting down +49:29.624 [INFO] TestAgent: consul server down +49:29.624 [INFO] TestAgent: shutdown complete +49:29.624 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23384 network=tcp +49:29.624 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23384 network=udp +49:29.624 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23385 network=tcp +49:30.125 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.125 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogRegister_checkRegistration (0.75s) +=== CONT TestCatalogNodes +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=1)_compressed=false +2022-12-04T05:49:29.538Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:29.561Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:29.564 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f7a46eba-4463-6806-bdca-9a19b38e3b35 Address:127.0.0.1:24241}]" +49:29.565 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24241 [Follower]" leader= +49:29.567 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f7a46eba-4463-6806-bdca-9a19b38e3b35.dc1 127.0.0.1 +49:29.567 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f7a46eba-4463-6806-bdca-9a19b38e3b35 127.0.0.1 +2022-12-04T05:49:29.567Z [INFO] agent.router: Initializing LAN area manager +49:29.568 [INFO] TestAgent.server: Adding LAN server: server="Node-f7a46eba-4463-6806-bdca-9a19b38e3b35 (Addr: tcp/127.0.0.1:24241) (DC: dc1)" +49:29.568 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f7a46eba-4463-6806-bdca-9a19b38e3b35.dc1 area=wan +49:29.568 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23279 network=udp +49:29.568 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23279 network=tcp +49:29.569 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23280 network=tcp +49:29.569 [INFO] TestAgent: started state syncer +49:29.569 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24242 network=tcp +49:29.631 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.632 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24241 [Candidate]" term=2 +49:29.632 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.632 [DEBUG] TestAgent.server.raft: vote granted: from=f7a46eba-4463-6806-bdca-9a19b38e3b35 term=2 tally=1 +49:29.632 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.632 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24241 [Leader]" +49:29.632 [INFO] TestAgent.server: cluster leadership acquired +49:29.632 [INFO] TestAgent.server: New leader elected: payload=Node-f7a46eba-4463-6806-bdca-9a19b38e3b35 +49:29.632 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24241 +49:29.633 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.634 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.634 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.634 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.634 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.634 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.634 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f7a46eba-4463-6806-bdca-9a19b38e3b35 +49:29.634 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f7a46eba-4463-6806-bdca-9a19b38e3b35 +49:29.635 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.661 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=107.892µs client=127.0.0.1:54536 client_network=udp +49:29.661 [INFO] TestAgent: Requesting shutdown +49:29.661 [INFO] TestAgent.server: shutting down server +49:29.661 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.661 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.661 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.661 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.661 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.661 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.661 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.661 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:29.661 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.661 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.661 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:29.661 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:29.661Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.661Z [INFO] agent.router.manager: shutting down +49:29.662 [INFO] TestAgent: consul server down +49:29.662 [INFO] TestAgent: shutdown complete +49:29.662 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23279 network=tcp +49:29.662 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23279 network=udp +49:29.662 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23280 network=tcp +49:30.162 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.162 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:28.579Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.603Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.606 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e0e90032-8aca-5899-c8cf-7d74e5d20093 Address:127.0.0.1:24234}]" +49:28.607 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e0e90032-8aca-5899-c8cf-7d74e5d20093.dc1 127.0.0.1 +49:28.608 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24234 [Follower]" leader= +49:28.609 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e0e90032-8aca-5899-c8cf-7d74e5d20093 127.0.0.1 +2022-12-04T05:49:28.609Z [INFO] agent.router: Initializing LAN area manager +49:28.609 [INFO] TestAgent.server: Adding LAN server: server="Node-e0e90032-8aca-5899-c8cf-7d74e5d20093 (Addr: tcp/127.0.0.1:24234) (DC: dc1)" +49:28.610 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e0e90032-8aca-5899-c8cf-7d74e5d20093.dc1 area=wan +49:28.610 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24194 network=udp +49:28.610 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24194 network=tcp +49:28.610 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24195 network=tcp +49:28.611 [INFO] TestAgent: started state syncer +49:28.611 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24235 network=tcp +49:28.659 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.659 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24234 [Candidate]" term=2 +49:28.660 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.660 [DEBUG] TestAgent.server.raft: vote granted: from=e0e90032-8aca-5899-c8cf-7d74e5d20093 term=2 tally=1 +49:28.660 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.660 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24234 [Leader]" +49:28.660 [INFO] TestAgent.server: cluster leadership acquired +49:28.660 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24234 +49:28.660 [INFO] TestAgent.server: New leader elected: payload=Node-e0e90032-8aca-5899-c8cf-7d74e5d20093 +49:28.661 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.662 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.662 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.662 [INFO] TestAgent: Synced node info +49:28.662 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.662 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.662 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.662 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e0e90032-8aca-5899-c8cf-7d74e5d20093 +49:28.662 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e0e90032-8aca-5899-c8cf-7d74e5d20093 +49:28.755 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.025 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=119.821µs client=127.0.0.1:38315 client_network=udp +49:29.026 [INFO] TestAgent: Requesting shutdown +49:29.026 [INFO] TestAgent.server: shutting down server +49:29.026 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.026 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.026 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.026 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.026 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.026 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.026 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.026 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.026 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.026 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:29.026 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:29.026Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.026Z [INFO] agent.router.manager: shutting down +49:29.027 [INFO] TestAgent: consul server down +49:29.027 [INFO] TestAgent: shutdown complete +49:29.027 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24194 network=tcp +49:29.027 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24194 network=udp +49:29.027 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24195 network=tcp +49:29.527 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.527 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:27.639Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.649Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.651 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b7dbd517-ab82-bd98-3e12-307abf43061a Address:127.0.0.1:24115}]" +49:27.651 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24115 [Follower]" leader= +49:27.651 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b7dbd517-ab82-bd98-3e12-307abf43061a.dc1 127.0.0.1 +49:27.652 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b7dbd517-ab82-bd98-3e12-307abf43061a 127.0.0.1 +2022-12-04T05:49:27.652Z [INFO] agent.router: Initializing LAN area manager +49:27.652 [INFO] TestAgent.server: Adding LAN server: server="Node-b7dbd517-ab82-bd98-3e12-307abf43061a (Addr: tcp/127.0.0.1:24115) (DC: dc1)" +49:27.652 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b7dbd517-ab82-bd98-3e12-307abf43061a.dc1 area=wan +49:27.652 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24131 network=udp +49:27.652 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24131 network=tcp +49:27.652 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24132 network=tcp +49:27.652 [INFO] TestAgent: started state syncer +49:27.652 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24116 network=tcp +49:27.687 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.687 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24115 [Candidate]" term=2 +49:27.688 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.688 [DEBUG] TestAgent.server.raft: vote granted: from=b7dbd517-ab82-bd98-3e12-307abf43061a term=2 tally=1 +49:27.688 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.688 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24115 [Leader]" +49:27.688 [INFO] TestAgent.server: cluster leadership acquired +49:27.688 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24115 +49:27.688 [INFO] TestAgent.server: New leader elected: payload=Node-b7dbd517-ab82-bd98-3e12-307abf43061a +49:27.688 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.690 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.690 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.690 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.690 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.690 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.690 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b7dbd517-ab82-bd98-3e12-307abf43061a +49:27.690 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b7dbd517-ab82-bd98-3e12-307abf43061a +49:27.690 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.936 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:27.936 [INFO] TestAgent: Synced node info +49:27.936 [DEBUG] TestAgent: Node info in sync +49:28.065 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=160.733µs client=127.0.0.1:50078 client_network=udp +49:28.066 [INFO] TestAgent: Requesting shutdown +49:28.066 [INFO] TestAgent.server: shutting down server +49:28.066 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.066 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.066 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.066 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.066 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.066 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.066 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.066 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.066 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.066 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:28.066 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:49:28.066Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.066Z [INFO] agent.router.manager: shutting down +49:28.066 [INFO] TestAgent: consul server down +49:28.066 [INFO] TestAgent: shutdown complete +49:28.066 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24131 network=tcp +49:28.066 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24131 network=udp +49:28.066 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24132 network=tcp +49:28.567 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.567 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:26.702Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.715Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.717 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1e8aa916-174c-f8a7-f89e-20699936334b Address:127.0.0.1:23978}]" +49:26.717 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23978 [Follower]" leader= +49:26.717 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1e8aa916-174c-f8a7-f89e-20699936334b.dc1 127.0.0.1 +49:26.718 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1e8aa916-174c-f8a7-f89e-20699936334b 127.0.0.1 +2022-12-04T05:49:26.718Z [INFO] agent.router: Initializing LAN area manager +49:26.718 [INFO] TestAgent.server: Adding LAN server: server="Node-1e8aa916-174c-f8a7-f89e-20699936334b (Addr: tcp/127.0.0.1:23978) (DC: dc1)" +49:26.718 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1e8aa916-174c-f8a7-f89e-20699936334b.dc1 area=wan +49:26.718 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24008 network=udp +49:26.718 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24008 network=tcp +49:26.719 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23995 network=tcp +49:26.719 [INFO] TestAgent: started state syncer +49:26.719 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23979 network=tcp +49:26.773 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.773 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23978 [Candidate]" term=2 +49:26.773 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.773 [DEBUG] TestAgent.server.raft: vote granted: from=1e8aa916-174c-f8a7-f89e-20699936334b term=2 tally=1 +49:26.773 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.773 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23978 [Leader]" +49:26.773 [INFO] TestAgent.server: cluster leadership acquired +49:26.779 [INFO] TestAgent.server: New leader elected: payload=Node-1e8aa916-174c-f8a7-f89e-20699936334b +49:26.780 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23978 +49:26.781 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.783 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.783 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.783 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.783 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.783 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.783 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1e8aa916-174c-f8a7-f89e-20699936334b +49:26.783 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1e8aa916-174c-f8a7-f89e-20699936334b +49:26.784 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.002 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:27.002 [INFO] TestAgent: Synced node info +49:27.112 [DEBUG] TestAgent.dns: request served from client: name=nofoo.node.dc1.consul. type=ANY class=IN latency=121.55µs client=127.0.0.1:56516 client_network=udp +49:27.115 [INFO] TestAgent: Requesting shutdown +49:27.115 [INFO] TestAgent.server: shutting down server +49:27.115 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.115 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.115 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.115 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.115 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.115 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.115 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.115 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.115 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.115 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:27.115 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:27.115Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.115Z [INFO] agent.router.manager: shutting down +49:27.115 [INFO] TestAgent: consul server down +49:27.115 [INFO] TestAgent: shutdown complete +49:27.115 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24008 network=tcp +49:27.115 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24008 network=udp +49:27.115 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23995 network=tcp +49:27.620 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.620 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_SOA_Settings (3.47s) +=== CONT TestCatalogServiceNodes_NodeMetaFilter +2022-12-04T05:49:29.393Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:29.421Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:29.423 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4873d90e-f660-705a-0ff2-e3fa267e73ea Address:127.0.0.1:23382}]" +49:29.423 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23382 [Follower]" leader= +49:29.424 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4873d90e-f660-705a-0ff2-e3fa267e73ea.dc1 127.0.0.1 +49:29.424 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4873d90e-f660-705a-0ff2-e3fa267e73ea 127.0.0.1 +2022-12-04T05:49:29.424Z [INFO] agent.router: Initializing LAN area manager +49:29.425 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23300 network=udp +49:29.425 [INFO] TestAgent.server: Adding LAN server: server="Node-4873d90e-f660-705a-0ff2-e3fa267e73ea (Addr: tcp/127.0.0.1:23382) (DC: dc1)" +49:29.425 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4873d90e-f660-705a-0ff2-e3fa267e73ea.dc1 area=wan +49:29.425 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23300 network=tcp +49:29.425 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23301 network=tcp +49:29.425 [INFO] TestAgent: started state syncer +49:29.426 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23383 network=tcp +49:29.487 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.487 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23382 [Candidate]" term=2 +49:29.487 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.487 [DEBUG] TestAgent.server.raft: vote granted: from=4873d90e-f660-705a-0ff2-e3fa267e73ea term=2 tally=1 +49:29.487 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.487 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23382 [Leader]" +49:29.487 [INFO] TestAgent.server: cluster leadership acquired +49:29.487 [INFO] TestAgent.server: New leader elected: payload=Node-4873d90e-f660-705a-0ff2-e3fa267e73ea +49:29.487 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23382 +49:29.488 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.490 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.490 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.490 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.490 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.490 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.490 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4873d90e-f660-705a-0ff2-e3fa267e73ea +49:29.490 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4873d90e-f660-705a-0ff2-e3fa267e73ea +49:29.491 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.512 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:29.513 [INFO] TestAgent: Synced node info +49:29.662 [INFO] TestAgent: Requesting shutdown +49:29.662 [INFO] TestAgent.server: shutting down server +49:29.662 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.662 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.662 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.662 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:29.662 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.662 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.662 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.663 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.663 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.663 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.663 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:29.663Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.663Z [INFO] agent.router.manager: shutting down +49:29.663 [INFO] TestAgent: consul server down +49:29.663 [INFO] TestAgent: shutdown complete +49:29.663 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23300 network=tcp +49:29.663 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23300 network=udp +49:29.663 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23301 network=tcp +49:30.164 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.164 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogServices_NodeMetaFilter (0.79s) +=== CONT TestCatalogRegister_Service_InvalidAddress +2022-12-04T05:49:29.527Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:29.543Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:29.546 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bdf23a59-138e-9701-7346-356084ce9284 Address:127.0.0.1:23277}]" +49:29.547 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bdf23a59-138e-9701-7346-356084ce9284.dc1 127.0.0.1 +49:29.548 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bdf23a59-138e-9701-7346-356084ce9284 127.0.0.1 +2022-12-04T05:49:29.548Z [INFO] agent.router: Initializing LAN area manager +49:29.548 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23277 [Follower]" leader= +49:29.550 [INFO] TestAgent.server: Adding LAN server: server="Node-bdf23a59-138e-9701-7346-356084ce9284 (Addr: tcp/127.0.0.1:23277) (DC: dc1)" +49:29.550 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bdf23a59-138e-9701-7346-356084ce9284.dc1 area=wan +49:29.548 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23356 network=udp +49:29.550 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23356 network=tcp +49:29.551 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23357 network=tcp +49:29.551 [INFO] TestAgent: started state syncer +49:29.551 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23278 network=tcp +49:29.604 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.604 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23277 [Candidate]" term=2 +49:29.604 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.604 [DEBUG] TestAgent.server.raft: vote granted: from=bdf23a59-138e-9701-7346-356084ce9284 term=2 tally=1 +49:29.604 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.604 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23277 [Leader]" +49:29.604 [INFO] TestAgent.server: cluster leadership acquired +49:29.604 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23277 +49:29.604 [INFO] TestAgent.server: New leader elected: payload=Node-bdf23a59-138e-9701-7346-356084ce9284 +49:29.605 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.606 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.606 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.606 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.606 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.606 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.607 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bdf23a59-138e-9701-7346-356084ce9284 +49:29.607 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bdf23a59-138e-9701-7346-356084ce9284 +49:29.607 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.635 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:29.636 [INFO] TestAgent: Synced node info +49:29.706 [INFO] TestAgent: Requesting shutdown +49:29.706 [INFO] TestAgent.server: shutting down server +49:29.707 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.707 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.707 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.707 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.707 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.707 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.707 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.707 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:29.707 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.707 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.707 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:29.707Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.707Z [INFO] agent.router.manager: shutting down +49:29.707 [INFO] TestAgent: consul server down +49:29.707 [INFO] TestAgent: shutdown complete +49:29.707 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23356 network=tcp +49:29.707 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23356 network=udp +49:29.707 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23357 network=tcp +49:30.207 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.207 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:29.269Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:29.281Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:29.283 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2032c89b-e07b-988b-aae6-49821f5d2439 Address:127.0.0.1:23375}]" +49:29.283 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23375 [Follower]" leader= +49:29.284 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2032c89b-e07b-988b-aae6-49821f5d2439.dc1 127.0.0.1 +49:29.285 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2032c89b-e07b-988b-aae6-49821f5d2439 127.0.0.1 +2022-12-04T05:49:29.285Z [INFO] agent.router: Initializing LAN area manager +49:29.285 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23335 network=udp +49:29.285 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23335 network=tcp +49:29.285 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2032c89b-e07b-988b-aae6-49821f5d2439.dc1 area=wan +49:29.285 [INFO] TestAgent.server: Adding LAN server: server="Node-2032c89b-e07b-988b-aae6-49821f5d2439 (Addr: tcp/127.0.0.1:23375) (DC: dc1)" +49:29.286 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23336 network=tcp +49:29.286 [INFO] TestAgent: started state syncer +49:29.286 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23376 network=tcp +49:29.352 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.352 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23375 [Candidate]" term=2 +49:29.352 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.352 [DEBUG] TestAgent.server.raft: vote granted: from=2032c89b-e07b-988b-aae6-49821f5d2439 term=2 tally=1 +49:29.352 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.352 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23375 [Leader]" +49:29.352 [INFO] TestAgent.server: cluster leadership acquired +49:29.353 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23375 +49:29.354 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.355 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.355 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.355 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.355 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.355 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.355 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2032c89b-e07b-988b-aae6-49821f5d2439 +49:29.355 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2032c89b-e07b-988b-aae6-49821f5d2439 +49:29.356 [INFO] TestAgent.server: New leader elected: payload=Node-2032c89b-e07b-988b-aae6-49821f5d2439 +49:29.360 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.634 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:29.634 [INFO] TestAgent: Synced node info +49:29.707 [INFO] TestAgent: Requesting shutdown +49:29.707 [INFO] TestAgent.server: shutting down server +49:29.707 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.707 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.707 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.707 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.707 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.707 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.707 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.707 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.707 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.707 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:29.707 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:29.707Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.707Z [INFO] agent.router.manager: shutting down +49:29.708 [INFO] TestAgent: consul server down +49:29.708 [INFO] TestAgent: shutdown complete +49:29.708 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23335 network=tcp +49:29.708 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23335 network=udp +49:29.708 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23336 network=tcp +49:30.208 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.208 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogServiceNodes_Filter (0.70s) +=== CONT TestCatalogDatacenters +--- PASS: TestCatalogServiceNodes_DistanceSort (0.95s) +=== CONT TestAgent_JoinWAN_viaMeshGateway +2022-12-04T05:49:29.378Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:29.396Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:29.403 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:43523365-2264-d973-83e5-acae6defdbab Address:127.0.0.1:23298}]" +49:29.404 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-43523365-2264-d973-83e5-acae6defdbab.dc1 127.0.0.1 +49:29.404 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23298 [Follower]" leader= +49:29.406 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-43523365-2264-d973-83e5-acae6defdbab 127.0.0.1 +2022-12-04T05:49:29.406Z [INFO] agent.router: Initializing LAN area manager +49:29.407 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23286 network=udp +49:29.407 [INFO] TestAgent.server: Adding LAN server: server="Node-43523365-2264-d973-83e5-acae6defdbab (Addr: tcp/127.0.0.1:23298) (DC: dc1)" +49:29.407 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-43523365-2264-d973-83e5-acae6defdbab.dc1 area=wan +49:29.408 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23286 network=tcp +49:29.409 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23287 network=tcp +49:29.409 [INFO] TestAgent: started state syncer +49:29.409 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23299 network=tcp +49:29.442 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.442 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23298 [Candidate]" term=2 +49:29.442 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.442 [DEBUG] TestAgent.server.raft: vote granted: from=43523365-2264-d973-83e5-acae6defdbab term=2 tally=1 +49:29.442 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.442 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23298 [Leader]" +49:29.443 [INFO] TestAgent.server: cluster leadership acquired +49:29.443 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23298 +49:29.444 [INFO] TestAgent.server: New leader elected: payload=Node-43523365-2264-d973-83e5-acae6defdbab +49:29.444 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.446 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.446 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.446 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.446 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.446 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.446 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-43523365-2264-d973-83e5-acae6defdbab +49:29.447 [INFO] TestAgent.server: member joined, marking health alive: member=Node-43523365-2264-d973-83e5-acae6defdbab +49:29.448 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.674 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:29.674 [INFO] TestAgent: Synced node info +49:29.712 [INFO] TestAgent: Requesting shutdown +49:29.712 [INFO] TestAgent.server: shutting down server +49:29.712 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.712 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.712 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.712 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.712 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.712 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.712 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.712 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.712 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:29.712 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.712 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:29.712Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.712Z [INFO] agent.router.manager: shutting down +49:29.712 [INFO] TestAgent: consul server down +49:29.712 [INFO] TestAgent: shutdown complete +49:29.712 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23286 network=tcp +49:29.712 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23286 network=udp +49:29.712 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23287 network=tcp +49:30.213 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.213 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogServiceNodes (0.85s) +=== CONT TestAgent_RerouteNewHTTPChecks +2022-12-04T05:49:29.429Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:29.445Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:29.450 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2b5eacde-20ea-6d8b-e135-16239d4a1b9c Address:127.0.0.1:23417}]" +49:29.452 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2b5eacde-20ea-6d8b-e135-16239d4a1b9c.dc1 127.0.0.1 +49:29.453 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2b5eacde-20ea-6d8b-e135-16239d4a1b9c 127.0.0.1 +2022-12-04T05:49:29.453Z [INFO] agent.router: Initializing LAN area manager +49:29.454 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23307 network=udp +49:29.454 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23417 [Follower]" leader= +49:29.455 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2b5eacde-20ea-6d8b-e135-16239d4a1b9c.dc1 area=wan +49:29.456 [INFO] TestAgent.server: Adding LAN server: server="Node-2b5eacde-20ea-6d8b-e135-16239d4a1b9c (Addr: tcp/127.0.0.1:23417) (DC: dc1)" +49:29.456 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23307 network=tcp +49:29.457 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23308 network=tcp +49:29.457 [INFO] TestAgent: started state syncer +49:29.458 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23418 network=tcp +49:29.516 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.516 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23417 [Candidate]" term=2 +49:29.517 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.517 [DEBUG] TestAgent.server.raft: vote granted: from=2b5eacde-20ea-6d8b-e135-16239d4a1b9c term=2 tally=1 +49:29.517 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.517 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23417 [Leader]" +49:29.517 [INFO] TestAgent.server: cluster leadership acquired +49:29.517 [INFO] TestAgent.server: New leader elected: payload=Node-2b5eacde-20ea-6d8b-e135-16239d4a1b9c +49:29.517 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23417 +49:29.518 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.523 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.523 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.523 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.523 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.523 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.523 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2b5eacde-20ea-6d8b-e135-16239d4a1b9c +49:29.523 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2b5eacde-20ea-6d8b-e135-16239d4a1b9c +49:29.523 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.528 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:29.528 [INFO] TestAgent: Synced node info +49:29.785 [INFO] TestAgent: Requesting shutdown +49:29.785 [INFO] TestAgent.server: shutting down server +49:29.785 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.785 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.785 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.785 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.785 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.785 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.785 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.785 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.785 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.785 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:29.785 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:29.786Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.786Z [INFO] agent.router.manager: shutting down +49:29.786 [INFO] TestAgent: consul server down +49:29.786 [INFO] TestAgent: shutdown complete +49:29.786 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23307 network=tcp +49:29.786 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23307 network=udp +49:29.786 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23308 network=tcp +49:30.286 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.286 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogServices (0.87s) +=== CONT TestACL_filterChecks +[WARN] freeport: 10 out of 87 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:30.315Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:30.315Z [WARN] agent: bootstrap = true: do not enable unless necessary +49:30.315 [DEBUG] TestACL_filterChecks: dropping check from result due to ACLs: check=my-node +49:30.315 [DEBUG] TestACL_filterChecks: dropping check from result due to ACLs: check=my-other +49:30.316 [DEBUG] TestACL_filterChecks: dropping check from result due to ACLs: check=my-service +49:30.316 [DEBUG] TestACL_filterChecks: dropping check from result due to ACLs: check=my-other +--- PASS: TestACL_filterChecks (0.03s) +=== CONT TestCatalogDeregister +2022-12-04T05:49:29.057Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:29.057Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:29.067Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:29.067Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:29.069 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a7d14d3e-3e03-4d95-9850-55af8dd59de9 Address:127.0.0.1:24283}]" +49:29.069 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24283 [Follower]" leader= +49:29.069 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a7d14d3e-3e03-4d95-9850-55af8dd59de9.dc2 127.0.0.1 +49:29.070 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a7d14d3e-3e03-4d95-9850-55af8dd59de9 127.0.0.1 +2022-12-04T05:49:29.070Z [INFO] agent.router: Initializing LAN area manager +49:29.070 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a7d14d3e-3e03-4d95-9850-55af8dd59de9.dc2 area=wan +49:29.070 [INFO] TestAgent.server: Adding LAN server: server="Node-a7d14d3e-3e03-4d95-9850-55af8dd59de9 (Addr: tcp/127.0.0.1:24283) (DC: dc2)" +49:29.070 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23363 network=udp +49:29.070 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23363 network=tcp +49:29.071 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23364 network=tcp +49:29.071 [INFO] TestAgent: started state syncer +49:29.071 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24284 network=tcp +49:29.105 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.105 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24283 [Candidate]" term=2 +49:29.105 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.105 [DEBUG] TestAgent.server.raft: vote granted: from=a7d14d3e-3e03-4d95-9850-55af8dd59de9 term=2 tally=1 +49:29.105 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.105 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24283 [Leader]" +49:29.105 [INFO] TestAgent.server: cluster leadership acquired +49:29.105 [INFO] TestAgent.server: New leader elected: payload=Node-a7d14d3e-3e03-4d95-9850-55af8dd59de9 +49:29.105 [INFO] TestAgent.server: initializing acls +49:29.106 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:29.106 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:29.106 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:29.106 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:29.106 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a7d14d3e-3e03-4d95-9850-55af8dd59de9 +49:29.106 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a7d14d3e-3e03-4d95-9850-55af8dd59de9.dc2 +49:29.106 [INFO] TestAgent.server: Updating LAN server: server="Node-a7d14d3e-3e03-4d95-9850-55af8dd59de9 (Addr: tcp/127.0.0.1:24283) (DC: dc2)" +49:29.106 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a7d14d3e-3e03-4d95-9850-55af8dd59de9.dc2 area=wan +49:29.107 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.108 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.108 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.108 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.108 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.108 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.108 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a7d14d3e-3e03-4d95-9850-55af8dd59de9 +49:29.108 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a7d14d3e-3e03-4d95-9850-55af8dd59de9 +49:29.109 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a7d14d3e-3e03-4d95-9850-55af8dd59de9 +49:29.109 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.267 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:29.267 [INFO] TestAgent: Synced node info +49:29.267 [DEBUG] TestAgent: Node info in sync +49:29.423 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:24247] +49:29.424 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:24247 +49:29.425 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7a3722b4-228e-6e70-3d35-d2f45f688698.dc1 127.0.0.1 +49:29.426 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +49:29.426 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7a3722b4-228e-6e70-3d35-d2f45f688698.dc1 area=wan +49:29.433 [INFO] TestAgent: Requesting shutdown +49:29.433 [INFO] TestAgent.server: shutting down server +49:29.433 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:29.433 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:29.433 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.433 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.433 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.433 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.433 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.433 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.433 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.433 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:29.433 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:29.433 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.433 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.433 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:29.433 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:29.433Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.433Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.433Z [INFO] agent.router.manager: shutting down +49:29.433 [INFO] TestAgent: consul server down +49:29.433 [INFO] TestAgent: shutdown complete +49:29.433 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23363 network=tcp +49:29.433 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23363 network=udp +49:29.433 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23364 network=tcp +49:29.936 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.936 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:28.635Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:28.635Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.664Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:28.664Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.666 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7a3722b4-228e-6e70-3d35-d2f45f688698 Address:127.0.0.1:24248}]" +49:28.666 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7a3722b4-228e-6e70-3d35-d2f45f688698.dc1 127.0.0.1 +49:28.667 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7a3722b4-228e-6e70-3d35-d2f45f688698 127.0.0.1 +2022-12-04T05:49:28.667Z [INFO] agent.router: Initializing LAN area manager +49:28.667 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24271 network=udp +49:28.667 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24248 [Follower]" leader= +49:28.667 [INFO] TestAgent.server: Adding LAN server: server="Node-7a3722b4-228e-6e70-3d35-d2f45f688698 (Addr: tcp/127.0.0.1:24248) (DC: dc1)" +49:28.667 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7a3722b4-228e-6e70-3d35-d2f45f688698.dc1 area=wan +49:28.668 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24271 network=tcp +49:28.668 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24272 network=tcp +49:28.668 [INFO] TestAgent: started state syncer +49:28.668 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24249 network=tcp +49:28.718 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:28.726 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.726 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24248 [Candidate]" term=2 +49:28.726 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.726 [DEBUG] TestAgent.server.raft: vote granted: from=7a3722b4-228e-6e70-3d35-d2f45f688698 term=2 tally=1 +49:28.726 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.726 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24248 [Leader]" +49:28.728 [INFO] TestAgent.server: cluster leadership acquired +49:28.733 [INFO] TestAgent.server: New leader elected: payload=Node-7a3722b4-228e-6e70-3d35-d2f45f688698 +49:28.733 [INFO] TestAgent.server: initializing acls +49:28.734 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:28.734 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:28.734 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:28.734 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:28.735 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7a3722b4-228e-6e70-3d35-d2f45f688698 +49:28.735 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7a3722b4-228e-6e70-3d35-d2f45f688698.dc1 +49:28.735 [INFO] TestAgent.server: Updating LAN server: server="Node-7a3722b4-228e-6e70-3d35-d2f45f688698 (Addr: tcp/127.0.0.1:24248) (DC: dc1)" +49:28.735 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7a3722b4-228e-6e70-3d35-d2f45f688698.dc1 area=wan +49:28.736 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.738 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.738 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.738 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.738 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.738 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.738 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7a3722b4-228e-6e70-3d35-d2f45f688698 +49:28.738 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7a3722b4-228e-6e70-3d35-d2f45f688698 +49:28.739 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7a3722b4-228e-6e70-3d35-d2f45f688698 +49:28.739 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.067 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:29.067 [INFO] TestAgent: Synced node info +49:29.067 [DEBUG] TestAgent: Node info in sync +49:29.424 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:42376 +49:29.425 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a7d14d3e-3e03-4d95-9850-55af8dd59de9.dc2 127.0.0.1 +49:29.425 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a7d14d3e-3e03-4d95-9850-55af8dd59de9.dc2 area=wan +49:29.936 [INFO] TestAgent: Requesting shutdown +49:29.936 [INFO] TestAgent.server: shutting down server +49:29.936 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:29.936 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:29.936 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.936 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.936 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.936 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.936 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.936 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.936 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.936 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:29.936 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:29.936 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.936 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.936 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:29.937 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:29.937Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.937Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.937Z [INFO] agent.router.manager: shutting down +49:29.937 [INFO] TestAgent: consul server down +49:29.937 [INFO] TestAgent: shutdown complete +49:29.937 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24271 network=tcp +49:29.937 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24271 network=udp +49:29.937 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24272 network=tcp +49:30.444 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.444 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodeServices_WanTranslation (1.83s) +=== CONT TestRemoteExecWrites_ACLToken +2022-12-04T05:49:29.557Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:29.577Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:29.579 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:89ced28b-3b02-e78b-6f7d-87c05e0fe79b Address:127.0.0.1:23396}]" +49:29.579 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23396 [Follower]" leader= +49:29.584 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-89ced28b-3b02-e78b-6f7d-87c05e0fe79b.dc1 127.0.0.1 +49:29.588 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-89ced28b-3b02-e78b-6f7d-87c05e0fe79b 127.0.0.1 +2022-12-04T05:49:29.588Z [INFO] agent.router: Initializing LAN area manager +49:29.589 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24243 network=udp +49:29.589 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-89ced28b-3b02-e78b-6f7d-87c05e0fe79b.dc1 area=wan +49:29.589 [INFO] TestAgent.server: Adding LAN server: server="Node-89ced28b-3b02-e78b-6f7d-87c05e0fe79b (Addr: tcp/127.0.0.1:23396) (DC: dc1)" +49:29.589 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24243 network=tcp +49:29.589 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24244 network=tcp +49:29.590 [INFO] TestAgent: started state syncer +49:29.590 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23397 network=tcp +49:29.634 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.634 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23396 [Candidate]" term=2 +49:29.634 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.634 [DEBUG] TestAgent.server.raft: vote granted: from=89ced28b-3b02-e78b-6f7d-87c05e0fe79b term=2 tally=1 +49:29.634 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.634 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23396 [Leader]" +49:29.634 [INFO] TestAgent.server: cluster leadership acquired +49:29.634 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23396 +49:29.635 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.635 [INFO] TestAgent.server: New leader elected: payload=Node-89ced28b-3b02-e78b-6f7d-87c05e0fe79b +49:29.637 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.637 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.637 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.637 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.637 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.637 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-89ced28b-3b02-e78b-6f7d-87c05e0fe79b +49:29.637 [INFO] TestAgent.server: member joined, marking health alive: member=Node-89ced28b-3b02-e78b-6f7d-87c05e0fe79b +49:29.637 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.852 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:29.853 [INFO] TestAgent: Synced node info +49:29.977 [INFO] TestAgent: Requesting shutdown +49:29.977 [INFO] TestAgent.server: shutting down server +49:29.977 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.977 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.977 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.977 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.977 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.977 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.977 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.977 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.977 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.977 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:29.977 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:29.977Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.977Z [INFO] agent.router.manager: shutting down +49:29.977 [INFO] TestAgent: consul server down +49:29.977 [INFO] TestAgent: shutdown complete +49:29.977 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24243 network=tcp +49:29.978 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24243 network=udp +49:29.978 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24244 network=tcp +49:30.478 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.478 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodes_Blocking (0.94s) +=== CONT TestHandleRemoteExec +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== CONT TestAgent_JoinWAN_viaMeshGateway + agent_test.go:4395: routing {bob.,}server.dc1.consul => 127.0.0.1:23497 + agent_test.go:4396: routing {betty.,}server.dc2.consul => 127.0.0.1:23413 + agent_test.go:4397: routing {bonnie.,}server.dc3.consul => 127.0.0.1:23449 +=== RUN TestCatalogRegister_Service_InvalidAddress/addr_0.0.0.0 +=== RUN TestCatalogRegister_Service_InvalidAddress/addr_:: +=== RUN TestCatalogRegister_Service_InvalidAddress/addr_[::] +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=1)_compressed=true +2022-12-04T05:49:29.805Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:29.817Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:29.819 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:259a37f6-6a2c-4d75-cf17-94ad27c8e6c2 Address:127.0.0.1:23459}]" +49:29.819 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23459 [Follower]" leader= +49:29.819 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-259a37f6-6a2c-4d75-cf17-94ad27c8e6c2.dc1 127.0.0.1 +49:29.820 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-259a37f6-6a2c-4d75-cf17-94ad27c8e6c2 127.0.0.1 +2022-12-04T05:49:29.820Z [INFO] agent.router: Initializing LAN area manager +49:29.820 [INFO] TestAgent.server: Adding LAN server: server="Node-259a37f6-6a2c-4d75-cf17-94ad27c8e6c2 (Addr: tcp/127.0.0.1:23459) (DC: dc1)" +49:29.820 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-259a37f6-6a2c-4d75-cf17-94ad27c8e6c2.dc1 area=wan +49:29.820 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23391 network=udp +49:29.820 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23391 network=tcp +49:29.821 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23392 network=tcp +49:29.821 [INFO] TestAgent: started state syncer +49:29.821 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23460 network=tcp +49:29.888 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.888 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23459 [Candidate]" term=2 +49:29.888 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.888 [DEBUG] TestAgent.server.raft: vote granted: from=259a37f6-6a2c-4d75-cf17-94ad27c8e6c2 term=2 tally=1 +49:29.888 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.888 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23459 [Leader]" +49:29.888 [INFO] TestAgent.server: cluster leadership acquired +49:29.889 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23459 +49:29.890 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.891 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.891 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.891 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.891 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.891 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.891 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-259a37f6-6a2c-4d75-cf17-94ad27c8e6c2 +49:29.891 [INFO] TestAgent.server: member joined, marking health alive: member=Node-259a37f6-6a2c-4d75-cf17-94ad27c8e6c2 +49:29.892 [INFO] TestAgent.server: New leader elected: payload=Node-259a37f6-6a2c-4d75-cf17-94ad27c8e6c2 +49:29.892 [INFO] TestAgent.server: federation state anti-entropy synced +49:30.072 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:30.073 [INFO] TestAgent: Synced node info +49:30.187 [INFO] TestAgent: Requesting shutdown +49:30.187 [INFO] TestAgent.server: shutting down server +49:30.187 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.187 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.187 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.187 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.187 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.187 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.187 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.187 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.187 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.187 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.187 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:30.188Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.188Z [INFO] agent.router.manager: shutting down +49:30.188 [INFO] TestAgent: consul server down +49:30.188 [INFO] TestAgent: shutdown complete +49:30.188 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23391 network=tcp +49:30.188 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23391 network=udp +49:30.188 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23392 network=tcp +49:30.688 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.688 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodes_Filter (0.89s) +=== CONT TestRemoteExecWrites_ACLDeny +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=1)_compressed=false +2022-12-04T05:49:29.450Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:29.475Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:29.482 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:76e42021-af6b-730f-b1a8-5893291dface Address:127.0.0.1:23354}]" +49:29.482 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-76e42021-af6b-730f-b1a8-5893291dface.dc1 127.0.0.1 +49:29.483 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-76e42021-af6b-730f-b1a8-5893291dface 127.0.0.1 +2022-12-04T05:49:29.483Z [INFO] agent.router: Initializing LAN area manager +49:29.483 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23419 network=udp +49:29.483 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23354 [Follower]" leader= +49:29.484 [INFO] TestAgent.server: Adding LAN server: server="Node-76e42021-af6b-730f-b1a8-5893291dface (Addr: tcp/127.0.0.1:23354) (DC: dc1)" +49:29.484 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-76e42021-af6b-730f-b1a8-5893291dface.dc1 area=wan +49:29.484 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23419 network=tcp +49:29.487 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23420 network=tcp +49:29.487 [INFO] TestAgent: started state syncer +49:29.487 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23355 network=tcp +49:29.556 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.556 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23354 [Candidate]" term=2 +49:29.556 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.556 [DEBUG] TestAgent.server.raft: vote granted: from=76e42021-af6b-730f-b1a8-5893291dface term=2 tally=1 +49:29.556 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.556 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23354 [Leader]" +49:29.557 [INFO] TestAgent.server: cluster leadership acquired +49:29.557 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23354 +49:29.558 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.558 [INFO] TestAgent.server: New leader elected: payload=Node-76e42021-af6b-730f-b1a8-5893291dface +49:29.560 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.560 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.560 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.560 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.560 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.560 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-76e42021-af6b-730f-b1a8-5893291dface +49:29.560 [INFO] TestAgent.server: member joined, marking health alive: member=Node-76e42021-af6b-730f-b1a8-5893291dface +49:29.561 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.718 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:29.719 [INFO] TestAgent: Synced node info +49:30.208 [INFO] TestAgent: Requesting shutdown +49:30.208 [INFO] TestAgent.server: shutting down server +49:30.208 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.208 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.208 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.208 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.208 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.208 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.208 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.208 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.208 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:30.208 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.208 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:30.208Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.208Z [INFO] agent.router.manager: shutting down +49:30.208 [INFO] TestAgent: consul server down +49:30.208 [INFO] TestAgent: shutdown complete +49:30.208 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23419 network=tcp +49:30.208 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23419 network=udp +49:30.208 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23420 network=tcp +49:30.708 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.708 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodes_DistanceSort (1.28s) +=== CONT TestRemoteExecWrites_ACLAgentToken +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=true +2022-12-04T05:49:29.680Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:29.680Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:29.690Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:29.690Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:29.692 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2c464c06-6cc7-b8df-ea95-a268a17a40be Address:127.0.0.1:23389}]" +49:29.692 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23389 [Follower]" leader= +49:29.692 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2c464c06-6cc7-b8df-ea95-a268a17a40be.dc2 127.0.0.1 +49:29.693 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2c464c06-6cc7-b8df-ea95-a268a17a40be 127.0.0.1 +2022-12-04T05:49:29.693Z [INFO] agent.router: Initializing LAN area manager +49:29.693 [INFO] TestAgent.server: Adding LAN server: server="Node-2c464c06-6cc7-b8df-ea95-a268a17a40be (Addr: tcp/127.0.0.1:23389) (DC: dc2)" +49:29.693 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2c464c06-6cc7-b8df-ea95-a268a17a40be.dc2 area=wan +49:29.693 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23328 network=udp +49:29.693 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23328 network=tcp +49:29.693 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23329 network=tcp +49:29.694 [INFO] TestAgent: started state syncer +49:29.694 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23390 network=tcp +49:29.742 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.743 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23389 [Candidate]" term=2 +49:29.743 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.743 [DEBUG] TestAgent.server.raft: vote granted: from=2c464c06-6cc7-b8df-ea95-a268a17a40be term=2 tally=1 +49:29.743 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.743 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23389 [Leader]" +49:29.743 [INFO] TestAgent.server: cluster leadership acquired +49:29.743 [INFO] TestAgent.server: initializing acls +49:29.743 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:29.743 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:29.743 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:29.744 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:29.744 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-2c464c06-6cc7-b8df-ea95-a268a17a40be +49:29.744 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-2c464c06-6cc7-b8df-ea95-a268a17a40be.dc2 +49:29.744 [INFO] TestAgent.server: New leader elected: payload=Node-2c464c06-6cc7-b8df-ea95-a268a17a40be +49:29.744 [INFO] TestAgent.server: Updating LAN server: server="Node-2c464c06-6cc7-b8df-ea95-a268a17a40be (Addr: tcp/127.0.0.1:23389) (DC: dc2)" +49:29.744 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-2c464c06-6cc7-b8df-ea95-a268a17a40be.dc2 area=wan +49:29.744 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.746 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.746 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.746 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.746 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.746 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.746 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2c464c06-6cc7-b8df-ea95-a268a17a40be +49:29.746 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2c464c06-6cc7-b8df-ea95-a268a17a40be +49:29.746 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2c464c06-6cc7-b8df-ea95-a268a17a40be +49:29.747 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.768 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:23283] +49:29.768 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:23283 +49:29.773 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-798a9c17-a405-16bd-f2a9-9317446af857.dc1 127.0.0.1 +49:29.773 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +49:29.773 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-798a9c17-a405-16bd-f2a9-9317446af857.dc1 area=wan +49:29.777 [INFO] TestAgent: Requesting shutdown +49:29.777 [INFO] TestAgent.server: shutting down server +49:29.777 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:29.777 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:29.777 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:29.777 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:29.777 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:29.777 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:29.777 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:29.777 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:29.777 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:29.777 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:29.777 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:29.777 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:29.777 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:29.777 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:29.777 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:29.777 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:29.778Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.778Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:29.778Z [INFO] agent.router.manager: shutting down +49:29.778 [INFO] TestAgent: consul server down +49:29.778 [INFO] TestAgent: shutdown complete +49:29.778 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23328 network=tcp +49:29.778 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23328 network=udp +49:29.778 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23329 network=tcp +49:30.278 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.279 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:29.285Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:29.285Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:29.296Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:29.296Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:29.297 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:798a9c17-a405-16bd-f2a9-9317446af857 Address:127.0.0.1:23284}]" +49:29.297 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23284 [Follower]" leader= +49:29.297 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-798a9c17-a405-16bd-f2a9-9317446af857.dc1 127.0.0.1 +49:29.298 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-798a9c17-a405-16bd-f2a9-9317446af857 127.0.0.1 +2022-12-04T05:49:29.298Z [INFO] agent.router: Initializing LAN area manager +49:29.298 [INFO] TestAgent.server: Adding LAN server: server="Node-798a9c17-a405-16bd-f2a9-9317446af857 (Addr: tcp/127.0.0.1:23284) (DC: dc1)" +49:29.298 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-798a9c17-a405-16bd-f2a9-9317446af857.dc1 area=wan +49:29.298 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23377 network=udp +49:29.298 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23377 network=tcp +49:29.299 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23378 network=tcp +49:29.299 [INFO] TestAgent: started state syncer +49:29.299 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23285 network=tcp +49:29.338 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.338 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23284 [Candidate]" term=2 +49:29.338 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.338 [DEBUG] TestAgent.server.raft: vote granted: from=798a9c17-a405-16bd-f2a9-9317446af857 term=2 tally=1 +49:29.338 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.338 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23284 [Leader]" +49:29.338 [INFO] TestAgent.server: cluster leadership acquired +49:29.339 [INFO] TestAgent.server: New leader elected: payload=Node-798a9c17-a405-16bd-f2a9-9317446af857 +49:29.339 [INFO] TestAgent.server: initializing acls +49:29.340 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:29.341 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:29.341 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:29.341 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:29.341 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-798a9c17-a405-16bd-f2a9-9317446af857 +49:29.342 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-798a9c17-a405-16bd-f2a9-9317446af857.dc1 +49:29.342 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.344 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.344 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.344 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.344 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.344 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.344 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-798a9c17-a405-16bd-f2a9-9317446af857 +49:29.344 [INFO] TestAgent.server: member joined, marking health alive: member=Node-798a9c17-a405-16bd-f2a9-9317446af857 +49:29.344 [INFO] TestAgent.server: Updating LAN server: server="Node-798a9c17-a405-16bd-f2a9-9317446af857 (Addr: tcp/127.0.0.1:23284) (DC: dc1)" +49:29.344 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-798a9c17-a405-16bd-f2a9-9317446af857.dc1 area=wan +49:29.345 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-798a9c17-a405-16bd-f2a9-9317446af857 +49:29.345 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.714 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:29.714 [INFO] TestAgent: Synced node info +49:29.714 [DEBUG] TestAgent: Node info in sync +49:29.768 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:55956 +49:29.772 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2c464c06-6cc7-b8df-ea95-a268a17a40be.dc2 127.0.0.1 +49:29.773 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2c464c06-6cc7-b8df-ea95-a268a17a40be.dc2 area=wan +49:30.279 [INFO] TestAgent: Requesting shutdown +49:30.279 [INFO] TestAgent.server: shutting down server +49:30.279 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:30.279 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.279 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.279 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.279 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.279 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:30.279 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.279 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:30.279 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.279 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.279 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.279 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.279 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.280 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:30.280 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +2022-12-04T05:49:30.280Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.280Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.280Z [INFO] agent.router.manager: shutting down +49:30.280 [INFO] TestAgent: consul server down +49:30.280 [INFO] TestAgent: shutdown complete +49:30.280 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23377 network=tcp +49:30.280 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23377 network=udp +49:30.280 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23378 network=tcp +49:30.781 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.781 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogServiceNodes_WanTranslation (1.51s) +=== CONT TestAgent_ServiceMaintenance_Disable +2022-12-04T05:49:30.179Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.189Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.190 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9a4641e0-8e75-79c3-3065-5acdb11ed5c8 Address:127.0.0.1:23424}]" +49:30.191 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23424 [Follower]" leader= +49:30.191 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9a4641e0-8e75-79c3-3065-5acdb11ed5c8.dc1 127.0.0.1 +49:30.191 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9a4641e0-8e75-79c3-3065-5acdb11ed5c8 127.0.0.1 +2022-12-04T05:49:30.191Z [INFO] agent.router: Initializing LAN area manager +49:30.192 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23440 network=udp +49:30.192 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9a4641e0-8e75-79c3-3065-5acdb11ed5c8.dc1 area=wan +49:30.192 [INFO] TestAgent.server: Adding LAN server: server="Node-9a4641e0-8e75-79c3-3065-5acdb11ed5c8 (Addr: tcp/127.0.0.1:23424) (DC: dc1)" +49:30.192 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23440 network=tcp +49:30.192 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23441 network=tcp +49:30.192 [INFO] TestAgent: started state syncer +49:30.192 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23425 network=tcp +49:30.243 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.243 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23424 [Candidate]" term=2 +49:30.243 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.243 [DEBUG] TestAgent.server.raft: vote granted: from=9a4641e0-8e75-79c3-3065-5acdb11ed5c8 term=2 tally=1 +49:30.243 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.243 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23424 [Leader]" +49:30.243 [INFO] TestAgent.server: cluster leadership acquired +49:30.243 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23424 +49:30.244 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.244 [INFO] TestAgent.server: New leader elected: payload=Node-9a4641e0-8e75-79c3-3065-5acdb11ed5c8 +49:30.246 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.246 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.246 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.246 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.246 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.246 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9a4641e0-8e75-79c3-3065-5acdb11ed5c8 +49:30.246 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9a4641e0-8e75-79c3-3065-5acdb11ed5c8 +49:30.247 [INFO] TestAgent.server: federation state anti-entropy synced +49:30.272 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:30.272 [INFO] TestAgent: Synced node info +49:30.294 [INFO] TestAgent: Requesting shutdown +49:30.294 [INFO] TestAgent.server: shutting down server +49:30.294 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.294 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.294 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.294 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.294 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.294 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.294 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.294 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.294 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.294 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.298 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:30.299Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.299Z [INFO] agent.router.manager: shutting down +49:30.299 [INFO] TestAgent: consul server down +49:30.299 [INFO] TestAgent: shutdown complete +49:30.299 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23440 network=tcp +49:30.299 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23440 network=udp +49:30.299 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23441 network=tcp +49:30.800 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.800 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogServiceNodes_NodeMetaFilter (0.64s) +=== CONT TestAgent_consulConfig_RaftTrailingLogs +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:30.118Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.137Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.139 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:eb6678a7-c3e9-2c90-34dd-93d84164e8f7 Address:127.0.0.1:23319}]" +49:30.139 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23319 [Follower]" leader= +49:30.139 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-eb6678a7-c3e9-2c90-34dd-93d84164e8f7.dc1 127.0.0.1 +49:30.140 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-eb6678a7-c3e9-2c90-34dd-93d84164e8f7 127.0.0.1 +2022-12-04T05:49:30.140Z [INFO] agent.router: Initializing LAN area manager +49:30.140 [INFO] TestAgent.server: Adding LAN server: server="Node-eb6678a7-c3e9-2c90-34dd-93d84164e8f7 (Addr: tcp/127.0.0.1:23319) (DC: dc1)" +49:30.140 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-eb6678a7-c3e9-2c90-34dd-93d84164e8f7.dc1 area=wan +49:30.140 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23370 network=udp +49:30.141 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23370 network=tcp +49:30.141 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23371 network=tcp +49:30.141 [INFO] TestAgent: started state syncer +49:30.141 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23320 network=tcp +49:30.198 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.198 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23319 [Candidate]" term=2 +49:30.198 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.198 [DEBUG] TestAgent.server.raft: vote granted: from=eb6678a7-c3e9-2c90-34dd-93d84164e8f7 term=2 tally=1 +49:30.198 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.198 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23319 [Leader]" +49:30.198 [INFO] TestAgent.server: cluster leadership acquired +49:30.198 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23319 +49:30.199 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.200 [INFO] TestAgent.server: New leader elected: payload=Node-eb6678a7-c3e9-2c90-34dd-93d84164e8f7 +49:30.200 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.200 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.200 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.200 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.200 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.200 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eb6678a7-c3e9-2c90-34dd-93d84164e8f7 +49:30.200 [INFO] TestAgent.server: member joined, marking health alive: member=Node-eb6678a7-c3e9-2c90-34dd-93d84164e8f7 +49:30.201 [INFO] TestAgent.server: federation state anti-entropy synced +49:30.316 [INFO] TestAgent.server.connect: CA provider config updated +49:30.316 [INFO] TestAgent: Requesting shutdown +49:30.316 [INFO] TestAgent.server: shutting down server +49:30.316 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.316 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.316 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.316 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.316 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.317 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.317 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.317 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:30.317 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.317 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.317 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.317 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:30.317Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.317Z [INFO] agent.router.manager: shutting down +49:30.317 [INFO] TestAgent: consul server down +49:30.317 [INFO] TestAgent: shutdown complete +49:30.317 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23370 network=tcp +49:30.317 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23370 network=udp +49:30.317 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23371 network=tcp +49:30.818 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.818 [INFO] TestAgent: Endpoints down +=== RUN TestConnectCAConfig/setting_state_fails +2022-12-04T05:49:30.140Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.156Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.158 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8973bf5e-11f2-fb8c-05e4-3b2293d52fe3 Address:127.0.0.1:23438}]" +49:30.159 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23438 [Follower]" leader= +49:30.159 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8973bf5e-11f2-fb8c-05e4-3b2293d52fe3.dc1 127.0.0.1 +49:30.168 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8973bf5e-11f2-fb8c-05e4-3b2293d52fe3 127.0.0.1 +2022-12-04T05:49:30.168Z [INFO] agent.router: Initializing LAN area manager +49:30.169 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23321 network=udp +49:30.169 [INFO] TestAgent.server: Adding LAN server: server="Node-8973bf5e-11f2-fb8c-05e4-3b2293d52fe3 (Addr: tcp/127.0.0.1:23438) (DC: dc1)" +49:30.169 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8973bf5e-11f2-fb8c-05e4-3b2293d52fe3.dc1 area=wan +49:30.169 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23321 network=tcp +49:30.170 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23322 network=tcp +49:30.170 [INFO] TestAgent: started state syncer +49:30.170 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23439 network=tcp +49:30.226 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.226 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23438 [Candidate]" term=2 +49:30.227 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.227 [DEBUG] TestAgent.server.raft: vote granted: from=8973bf5e-11f2-fb8c-05e4-3b2293d52fe3 term=2 tally=1 +49:30.227 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.227 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23438 [Leader]" +49:30.227 [INFO] TestAgent.server: cluster leadership acquired +49:30.227 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23438 +49:30.228 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.228 [INFO] TestAgent.server: New leader elected: payload=Node-8973bf5e-11f2-fb8c-05e4-3b2293d52fe3 +49:30.229 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.229 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.229 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.229 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.229 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.229 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8973bf5e-11f2-fb8c-05e4-3b2293d52fe3 +49:30.229 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8973bf5e-11f2-fb8c-05e4-3b2293d52fe3 +49:30.230 [INFO] TestAgent.server: federation state anti-entropy synced +49:30.316 [INFO] TestAgent: Requesting shutdown +49:30.316 [INFO] TestAgent.server: shutting down server +49:30.316 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.316 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.316 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.316 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.316 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.316 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.316 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.316 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:30.316 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.316 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.316 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.316 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:30.317Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.317Z [INFO] agent.router.manager: shutting down +49:30.317 [INFO] TestAgent: consul server down +49:30.317 [INFO] TestAgent: shutdown complete +49:30.317 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23321 network=tcp +49:30.317 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23321 network=udp +49:30.317 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23322 network=tcp +49:30.822 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.822 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodes (0.70s) +=== CONT TestAgent_consulConfig_AutoEncryptAllowTLS +2022-12-04T05:49:30.226Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.238Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.240 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0c18b3d9-97aa-ed53-7168-de3c39c22f01 Address:127.0.0.1:23525}]" +49:30.240 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23525 [Follower]" leader= +49:30.240 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0c18b3d9-97aa-ed53-7168-de3c39c22f01.dc1 127.0.0.1 +49:30.241 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0c18b3d9-97aa-ed53-7168-de3c39c22f01 127.0.0.1 +2022-12-04T05:49:30.241Z [INFO] agent.router: Initializing LAN area manager +49:30.241 [INFO] TestAgent.server: Adding LAN server: server="Node-0c18b3d9-97aa-ed53-7168-de3c39c22f01 (Addr: tcp/127.0.0.1:23525) (DC: dc1)" +49:30.241 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0c18b3d9-97aa-ed53-7168-de3c39c22f01.dc1 area=wan +49:30.241 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23513 network=udp +49:30.241 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23513 network=tcp +49:30.241 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23521 network=tcp +49:30.241 [INFO] TestAgent: started state syncer +49:30.241 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23498 network=tcp +49:30.297 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.297 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23525 [Candidate]" term=2 +49:30.297 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.297 [DEBUG] TestAgent.server.raft: vote granted: from=0c18b3d9-97aa-ed53-7168-de3c39c22f01 term=2 tally=1 +49:30.297 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.297 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23525 [Leader]" +49:30.298 [INFO] TestAgent.server: cluster leadership acquired +49:30.298 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23525 +49:30.298 [INFO] TestAgent.server: New leader elected: payload=Node-0c18b3d9-97aa-ed53-7168-de3c39c22f01 +49:30.300 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.314 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.314 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.314 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.314 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.314 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.314 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0c18b3d9-97aa-ed53-7168-de3c39c22f01 +49:30.314 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0c18b3d9-97aa-ed53-7168-de3c39c22f01 +49:30.315 [INFO] TestAgent.server: federation state anti-entropy synced +49:30.344 [WARN] TestAgent: check has interval below minimum: check=http minimum_interval=1s +49:30.344 [WARN] TestAgent: check has interval below minimum: check=grpc minimum_interval=1s +49:30.344 [INFO] TestAgent: Requesting shutdown +49:30.345 [INFO] TestAgent.server: shutting down server +49:30.345 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.345 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.345 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.345 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.345 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.345 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.345 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.345 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:30.345 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.345 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.345 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.345 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:30.346Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.346Z [INFO] agent.router.manager: shutting down +49:30.346 [INFO] TestAgent: consul server down +49:30.346 [INFO] TestAgent: shutdown complete +49:30.346 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23513 network=tcp +49:30.346 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23513 network=udp +49:30.346 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23521 network=tcp +49:30.856 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.856 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RerouteNewHTTPChecks (0.64s) +=== CONT TestAgent_ReloadConfigTLSConfigFailure +2022-12-04T05:49:30.002Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.012Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.014 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fb903bf6-2f9b-41a7-225d-a82b2f0ae0be Address:127.0.0.1:23445}]" +49:30.014 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23445 [Follower]" leader= +49:30.014 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fb903bf6-2f9b-41a7-225d-a82b2f0ae0be.dc1 127.0.0.1 +49:30.015 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fb903bf6-2f9b-41a7-225d-a82b2f0ae0be 127.0.0.1 +2022-12-04T05:49:30.015Z [INFO] agent.router: Initializing LAN area manager +49:30.015 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fb903bf6-2f9b-41a7-225d-a82b2f0ae0be.dc1 area=wan +49:30.015 [INFO] TestAgent.server: Adding LAN server: server="Node-fb903bf6-2f9b-41a7-225d-a82b2f0ae0be (Addr: tcp/127.0.0.1:23445) (DC: dc1)" +49:30.015 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23461 network=udp +49:30.015 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23461 network=tcp +49:30.016 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23462 network=tcp +49:30.016 [INFO] TestAgent: started state syncer +49:30.016 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23446 network=tcp +49:30.057 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.064 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23445 [Candidate]" term=2 +49:30.065 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.065 [DEBUG] TestAgent.server.raft: vote granted: from=fb903bf6-2f9b-41a7-225d-a82b2f0ae0be term=2 tally=1 +49:30.065 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.065 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23445 [Leader]" +49:30.065 [INFO] TestAgent.server: cluster leadership acquired +49:30.065 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23445 +49:30.065 [INFO] TestAgent.server: New leader elected: payload=Node-fb903bf6-2f9b-41a7-225d-a82b2f0ae0be +49:30.066 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.069 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.069 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.069 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.069 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.069 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.069 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fb903bf6-2f9b-41a7-225d-a82b2f0ae0be +49:30.069 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fb903bf6-2f9b-41a7-225d-a82b2f0ae0be +49:30.070 [INFO] TestAgent.server: federation state anti-entropy synced +49:30.319 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:30.319 [INFO] TestAgent: Synced node info +49:30.421 [INFO] TestAgent: Requesting shutdown +49:30.421 [INFO] TestAgent.server: shutting down server +49:30.421 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.421 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.421 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.421 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.421 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.421 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.421 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.421 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.421 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.421 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.421 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:30.421Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.421Z [INFO] agent.router.manager: shutting down +49:30.421 [INFO] TestAgent: consul server down +49:30.421 [INFO] TestAgent: shutdown complete +49:30.422 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23461 network=tcp +49:30.422 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23461 network=udp +49:30.422 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23462 network=tcp +49:30.922 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.922 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodes_MetaFilter (0.96s) +=== CONT TestAgent_ReloadConfigIncomingRPCConfig +2022-12-04T05:49:30.221Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.233Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.235 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e5b6dfff-94fd-026c-2f90-8b6bd18c703d Address:127.0.0.1:23410}]" +49:30.235 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23410 [Follower]" leader= +49:30.236 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e5b6dfff-94fd-026c-2f90-8b6bd18c703d.dc1 127.0.0.1 +49:30.236 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e5b6dfff-94fd-026c-2f90-8b6bd18c703d 127.0.0.1 +2022-12-04T05:49:30.236Z [INFO] agent.router: Initializing LAN area manager +49:30.236 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e5b6dfff-94fd-026c-2f90-8b6bd18c703d.dc1 area=wan +49:30.236 [INFO] TestAgent.server: Adding LAN server: server="Node-e5b6dfff-94fd-026c-2f90-8b6bd18c703d (Addr: tcp/127.0.0.1:23410) (DC: dc1)" +49:30.237 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23433 network=tcp +49:30.237 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23433 network=udp +49:30.237 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23434 network=tcp +49:30.237 [INFO] TestAgent: started state syncer +49:30.237 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23411 network=tcp +49:30.315 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.315 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23410 [Candidate]" term=2 +49:30.315 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.315 [DEBUG] TestAgent.server.raft: vote granted: from=e5b6dfff-94fd-026c-2f90-8b6bd18c703d term=2 tally=1 +49:30.315 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.315 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23410 [Leader]" +49:30.316 [INFO] TestAgent.server: cluster leadership acquired +49:30.316 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23410 +49:30.316 [INFO] TestAgent.server: New leader elected: payload=Node-e5b6dfff-94fd-026c-2f90-8b6bd18c703d +49:30.317 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.318 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.318 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.318 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.318 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.318 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.318 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e5b6dfff-94fd-026c-2f90-8b6bd18c703d +49:30.318 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e5b6dfff-94fd-026c-2f90-8b6bd18c703d +49:30.319 [INFO] TestAgent.server: federation state anti-entropy synced +49:30.332 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:30.333 [INFO] TestAgent: Synced node info +49:30.536 [INFO] TestAgent: Requesting shutdown +49:30.536 [INFO] TestAgent.server: shutting down server +49:30.536 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.536 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.536 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.536 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.536 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.536 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.536 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.536 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.536 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.536 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.537 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:30.537Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.537Z [INFO] agent.router.manager: shutting down +49:30.537 [INFO] TestAgent: consul server down +49:30.537 [INFO] TestAgent: shutdown complete +49:30.537 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23433 network=tcp +49:30.537 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23433 network=udp +49:30.537 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23434 network=tcp +49:31.037 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.037 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogDatacenters (0.83s) +=== CONT TestAgent_ReloadConfigOutgoingRPCConfig +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=false +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:30.180Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.193Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.194 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5c6dcad6-fe81-8a15-c5be-abbf1608e710 Address:127.0.0.1:23431}]" +49:30.194 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23431 [Follower]" leader= +49:30.195 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5c6dcad6-fe81-8a15-c5be-abbf1608e710.dc1 127.0.0.1 +49:30.195 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5c6dcad6-fe81-8a15-c5be-abbf1608e710 127.0.0.1 +2022-12-04T05:49:30.195Z [INFO] agent.router: Initializing LAN area manager +49:30.196 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23426 network=udp +49:30.196 [INFO] TestAgent.server: Adding LAN server: server="Node-5c6dcad6-fe81-8a15-c5be-abbf1608e710 (Addr: tcp/127.0.0.1:23431) (DC: dc1)" +49:30.196 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5c6dcad6-fe81-8a15-c5be-abbf1608e710.dc1 area=wan +49:30.196 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23426 network=tcp +49:30.196 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23427 network=tcp +49:30.196 [INFO] TestAgent: started state syncer +49:30.196 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23432 network=tcp +49:30.264 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.264 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23431 [Candidate]" term=2 +49:30.264 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.264 [DEBUG] TestAgent.server.raft: vote granted: from=5c6dcad6-fe81-8a15-c5be-abbf1608e710 term=2 tally=1 +49:30.264 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.264 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23431 [Leader]" +49:30.264 [INFO] TestAgent.server: cluster leadership acquired +49:30.264 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23431 +49:30.265 [INFO] TestAgent.server: New leader elected: payload=Node-5c6dcad6-fe81-8a15-c5be-abbf1608e710 +49:30.265 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.267 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.267 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.267 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.267 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.267 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.267 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5c6dcad6-fe81-8a15-c5be-abbf1608e710 +49:30.267 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5c6dcad6-fe81-8a15-c5be-abbf1608e710 +49:30.267 [INFO] TestAgent.server: federation state anti-entropy synced +49:30.445 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:30.446 [INFO] TestAgent: Synced node info +49:30.446 [DEBUG] TestAgent: Node info in sync +49:30.592 [INFO] TestAgent: Requesting shutdown +49:30.593 [INFO] TestAgent.server: shutting down server +49:30.593 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.593 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.593 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.593 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.593 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.593 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.593 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.593 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.593 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.593 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.593 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:30.595Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.595Z [INFO] agent.router.manager: shutting down +49:30.595 [INFO] TestAgent: consul server down +49:30.595 [INFO] TestAgent: shutdown complete +49:30.595 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23426 network=tcp +49:30.595 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23426 network=udp +49:30.595 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23427 network=tcp +49:31.096 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.096 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogRegister_Service_InvalidAddress (0.93s) + --- PASS: TestCatalogRegister_Service_InvalidAddress/addr_0.0.0.0 (0.00s) + --- PASS: TestCatalogRegister_Service_InvalidAddress/addr_:: (0.00s) + --- PASS: TestCatalogRegister_Service_InvalidAddress/addr_[::] (0.00s) +=== CONT TestAgent_SecurityChecks +2022-12-04T05:49:30.502Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.531Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.533 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d9d48dea-d070-5680-fb14-b5e0929b4b9e Address:127.0.0.1:23553}]" +49:30.533 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23553 [Follower]" leader= +49:30.534 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d9d48dea-d070-5680-fb14-b5e0929b4b9e.dc1 127.0.0.1 +49:30.534 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d9d48dea-d070-5680-fb14-b5e0929b4b9e 127.0.0.1 +2022-12-04T05:49:30.534Z [INFO] agent.router: Initializing LAN area manager +49:30.535 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23562 network=udp +49:30.535 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d9d48dea-d070-5680-fb14-b5e0929b4b9e.dc1 area=wan +49:30.535 [INFO] TestAgent.server: Adding LAN server: server="Node-d9d48dea-d070-5680-fb14-b5e0929b4b9e (Addr: tcp/127.0.0.1:23553) (DC: dc1)" +49:30.535 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23562 network=tcp +49:30.535 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23549 network=tcp +49:30.536 [INFO] TestAgent: started state syncer +49:30.536 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23491 network=tcp +49:30.593 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.593 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23553 [Candidate]" term=2 +49:30.593 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.593 [DEBUG] TestAgent.server.raft: vote granted: from=d9d48dea-d070-5680-fb14-b5e0929b4b9e term=2 tally=1 +49:30.593 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.593 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23553 [Leader]" +49:30.593 [INFO] TestAgent.server: cluster leadership acquired +49:30.593 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23553 +49:30.594 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.595 [INFO] TestAgent.server: New leader elected: payload=Node-d9d48dea-d070-5680-fb14-b5e0929b4b9e +49:30.596 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.596 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.596 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.596 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.596 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.596 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d9d48dea-d070-5680-fb14-b5e0929b4b9e +49:30.596 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d9d48dea-d070-5680-fb14-b5e0929b4b9e +49:30.597 [INFO] TestAgent.server: federation state anti-entropy synced +49:30.694 [DEBUG] TestAgent: received remote exec event: id=78fd4521-c989-8e97-a9e4-161b108921e8 +49:30.695 [INFO] TestAgent: remote exec script: script=uptime +49:30.699 [INFO] TestAgent: Requesting shutdown +49:30.699 [INFO] TestAgent.server: shutting down server +49:30.699 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.699 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.699 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.699 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.699 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.699 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.699 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.699 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.699 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.699 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:30.699 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.699 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:30.700Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.700Z [INFO] agent.router.manager: shutting down +49:30.700 [INFO] TestAgent: consul server down +49:30.700 [INFO] TestAgent: shutdown complete +49:30.700 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23562 network=tcp +49:30.700 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23562 network=udp +49:30.700 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23549 network=tcp +49:31.201 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.201 [INFO] TestAgent: Endpoints down +--- PASS: TestHandleRemoteExec (0.72s) +=== CONT TestAgent_loadTokens +2022-12-04T05:49:30.473Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:30.473Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.486Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:30.486Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.493 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:60efc1ae-78b1-6b1e-d500-eb880a5a3d83 Address:127.0.0.1:23539}]" +49:30.493 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-60efc1ae-78b1-6b1e-d500-eb880a5a3d83.dc1 127.0.0.1 +49:30.494 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-60efc1ae-78b1-6b1e-d500-eb880a5a3d83 127.0.0.1 +2022-12-04T05:49:30.494Z [INFO] agent.router: Initializing LAN area manager +49:30.495 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23555 network=udp +49:30.495 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23539 [Follower]" leader= +49:30.495 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-60efc1ae-78b1-6b1e-d500-eb880a5a3d83.dc1 area=wan +49:30.495 [INFO] TestAgent.server: Adding LAN server: server="Node-60efc1ae-78b1-6b1e-d500-eb880a5a3d83 (Addr: tcp/127.0.0.1:23539) (DC: dc1)" +49:30.495 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23555 network=tcp +49:30.497 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23535 network=tcp +49:30.497 [INFO] TestAgent: started state syncer +49:30.497 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23561 network=tcp +49:30.537 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.537 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23539 [Candidate]" term=2 +49:30.537 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.537 [DEBUG] TestAgent.server.raft: vote granted: from=60efc1ae-78b1-6b1e-d500-eb880a5a3d83 term=2 tally=1 +49:30.537 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.537 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23539 [Leader]" +49:30.537 [INFO] TestAgent.server: cluster leadership acquired +49:30.537 [INFO] TestAgent.server: New leader elected: payload=Node-60efc1ae-78b1-6b1e-d500-eb880a5a3d83 +49:30.538 [INFO] TestAgent.server: initializing acls +49:30.538 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:30.538 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:30.538 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:30.539 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:30.539 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:30.539 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:30.539 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-60efc1ae-78b1-6b1e-d500-eb880a5a3d83 +49:30.539 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-60efc1ae-78b1-6b1e-d500-eb880a5a3d83.dc1 +49:30.539 [INFO] TestAgent.server: Updating LAN server: server="Node-60efc1ae-78b1-6b1e-d500-eb880a5a3d83 (Addr: tcp/127.0.0.1:23539) (DC: dc1)" +49:30.539 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-60efc1ae-78b1-6b1e-d500-eb880a5a3d83.dc1 area=wan +49:30.539 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.541 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.541 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.541 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.541 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.541 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.541 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-60efc1ae-78b1-6b1e-d500-eb880a5a3d83 +49:30.541 [INFO] TestAgent.server: member joined, marking health alive: member=Node-60efc1ae-78b1-6b1e-d500-eb880a5a3d83 +49:30.542 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-60efc1ae-78b1-6b1e-d500-eb880a5a3d83 +49:30.542 [INFO] TestAgent.server: federation state anti-entropy synced +49:30.630 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:30.631 [INFO] TestAgent: Synced node info +49:30.638 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:30.638 [DEBUG] TestAgent: Node info in sync +49:30.638 [DEBUG] TestAgent: Node info in sync +49:30.746 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-60efc1ae-78b1-6b1e-d500-eb880a5a3d83 +49:30.746 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-60efc1ae-78b1-6b1e-d500-eb880a5a3d83 +49:30.747 [INFO] TestAgent: Requesting shutdown +49:30.747 [INFO] TestAgent.server: shutting down server +49:30.747 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.747 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:30.747 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:30.747 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.747 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.747 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.747 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.747 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.747 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.748 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.748 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:30.748 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:30.748 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.748 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.748 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:30.748Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.748Z [INFO] agent.router.manager: shutting down +49:30.748 [INFO] TestAgent: consul server down +49:30.748 [INFO] TestAgent: shutdown complete +49:30.748 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23555 network=tcp +49:30.748 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23555 network=udp +49:30.748 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23535 network=tcp +49:31.248 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.248 [INFO] TestAgent: Endpoints down +--- PASS: TestRemoteExecWrites_ACLToken (0.80s) +=== CONT TestAgent_reloadWatchesHTTPS +2022-12-04T05:49:30.038Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:30.038Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.053Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:30.053Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.055 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5d17bf1b-df81-f86a-62b8-7ca2db186b48 Address:127.0.0.1:23368}]" +49:30.055 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23368 [Follower]" leader= +49:30.056 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5d17bf1b-df81-f86a-62b8-7ca2db186b48.dc2 127.0.0.1 +49:30.056 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5d17bf1b-df81-f86a-62b8-7ca2db186b48 127.0.0.1 +2022-12-04T05:49:30.056Z [INFO] agent.router: Initializing LAN area manager +49:30.057 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5d17bf1b-df81-f86a-62b8-7ca2db186b48.dc2 area=wan +49:30.057 [INFO] TestAgent.server: Adding LAN server: server="Node-5d17bf1b-df81-f86a-62b8-7ca2db186b48 (Addr: tcp/127.0.0.1:23368) (DC: dc2)" +49:30.057 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23447 network=udp +49:30.057 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23447 network=tcp +49:30.065 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23448 network=tcp +49:30.065 [INFO] TestAgent: started state syncer +49:30.065 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23369 network=tcp +49:30.091 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.091 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23368 [Candidate]" term=2 +49:30.091 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.091 [DEBUG] TestAgent.server.raft: vote granted: from=5d17bf1b-df81-f86a-62b8-7ca2db186b48 term=2 tally=1 +49:30.091 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.091 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23368 [Leader]" +49:30.092 [INFO] TestAgent.server: cluster leadership acquired +49:30.092 [INFO] TestAgent.server: New leader elected: payload=Node-5d17bf1b-df81-f86a-62b8-7ca2db186b48 +49:30.092 [INFO] TestAgent.server: initializing acls +49:30.092 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:30.092 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:30.092 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:30.092 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:30.093 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5d17bf1b-df81-f86a-62b8-7ca2db186b48 +49:30.093 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5d17bf1b-df81-f86a-62b8-7ca2db186b48.dc2 +49:30.093 [INFO] TestAgent.server: Updating LAN server: server="Node-5d17bf1b-df81-f86a-62b8-7ca2db186b48 (Addr: tcp/127.0.0.1:23368) (DC: dc2)" +49:30.093 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5d17bf1b-df81-f86a-62b8-7ca2db186b48.dc2 area=wan +49:30.093 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.094 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.094 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.094 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.094 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.094 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.095 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5d17bf1b-df81-f86a-62b8-7ca2db186b48 +49:30.095 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5d17bf1b-df81-f86a-62b8-7ca2db186b48 +49:30.095 [INFO] TestAgent.server: federation state anti-entropy synced +49:30.095 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5d17bf1b-df81-f86a-62b8-7ca2db186b48 +49:30.147 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:30.152 [INFO] TestAgent: Synced node info +49:30.255 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:23325] +49:30.256 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:23325 +49:30.257 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bd4c61bb-ebe0-3110-84f9-9938d103d7c7.dc1 127.0.0.1 +49:30.257 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +49:30.257 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bd4c61bb-ebe0-3110-84f9-9938d103d7c7.dc1 area=wan +49:30.262 [INFO] TestAgent: Requesting shutdown +49:30.262 [INFO] TestAgent.server: shutting down server +49:30.262 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.262 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.262 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.262 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.262 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:30.262 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:30.262 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.262 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.262 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.262 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.262 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.262 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.262 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:30.262 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:30.262 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:30.262Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.262Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.262Z [INFO] agent.router.manager: shutting down +49:30.262 [INFO] TestAgent: consul server down +49:30.262 [INFO] TestAgent: shutdown complete +49:30.262 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23447 network=tcp +49:30.262 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23447 network=udp +49:30.262 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23448 network=tcp +49:30.764 [INFO] TestAgent: Waiting for endpoints to shut down +49:30.764 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:29.618Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:29.618Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:29.645Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:29.645Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:29.649 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bd4c61bb-ebe0-3110-84f9-9938d103d7c7 Address:127.0.0.1:23326}]" +49:29.650 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bd4c61bb-ebe0-3110-84f9-9938d103d7c7.dc1 127.0.0.1 +49:29.651 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bd4c61bb-ebe0-3110-84f9-9938d103d7c7 127.0.0.1 +2022-12-04T05:49:29.651Z [INFO] agent.router: Initializing LAN area manager +49:29.651 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23398 network=udp +49:29.651 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23326 [Follower]" leader= +49:29.652 [INFO] TestAgent.server: Adding LAN server: server="Node-bd4c61bb-ebe0-3110-84f9-9938d103d7c7 (Addr: tcp/127.0.0.1:23326) (DC: dc1)" +49:29.652 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23398 network=tcp +49:29.652 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bd4c61bb-ebe0-3110-84f9-9938d103d7c7.dc1 area=wan +49:29.653 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23399 network=tcp +49:29.653 [INFO] TestAgent: started state syncer +49:29.654 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23327 network=tcp +49:29.703 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:29.712 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:29.712 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23326 [Candidate]" term=2 +49:29.712 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:29.712 [DEBUG] TestAgent.server.raft: vote granted: from=bd4c61bb-ebe0-3110-84f9-9938d103d7c7 term=2 tally=1 +49:29.712 [INFO] TestAgent.server.raft: election won: tally=1 +49:29.712 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23326 [Leader]" +49:29.712 [INFO] TestAgent.server: cluster leadership acquired +49:29.712 [INFO] TestAgent.server: New leader elected: payload=Node-bd4c61bb-ebe0-3110-84f9-9938d103d7c7 +49:29.712 [INFO] TestAgent.server: initializing acls +49:29.713 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:29.713 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:29.713 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:29.713 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:29.713 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-bd4c61bb-ebe0-3110-84f9-9938d103d7c7 +49:29.713 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-bd4c61bb-ebe0-3110-84f9-9938d103d7c7.dc1 +49:29.713 [INFO] TestAgent.server: Updating LAN server: server="Node-bd4c61bb-ebe0-3110-84f9-9938d103d7c7 (Addr: tcp/127.0.0.1:23326) (DC: dc1)" +49:29.713 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-bd4c61bb-ebe0-3110-84f9-9938d103d7c7.dc1 area=wan +49:29.714 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:29.715 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:29.715 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:29.715 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:29.715 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:29.715 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:29.715 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bd4c61bb-ebe0-3110-84f9-9938d103d7c7 +49:29.715 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bd4c61bb-ebe0-3110-84f9-9938d103d7c7 +49:29.716 [INFO] TestAgent.server: federation state anti-entropy synced +49:29.716 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bd4c61bb-ebe0-3110-84f9-9938d103d7c7 +49:30.041 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:30.041 [INFO] TestAgent: Synced node info +49:30.041 [DEBUG] TestAgent: Node info in sync +49:30.256 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:60260 +49:30.257 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5d17bf1b-df81-f86a-62b8-7ca2db186b48.dc2 127.0.0.1 +49:30.257 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5d17bf1b-df81-f86a-62b8-7ca2db186b48.dc2 area=wan +49:30.764 [INFO] TestAgent: Requesting shutdown +49:30.764 [INFO] TestAgent.server: shutting down server +49:30.764 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.764 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.764 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:30.764 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:30.764 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.764 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.764 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.764 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.764 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.764 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.764 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.764 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:30.764 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:30.764 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.765 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:30.765Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.765Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.765Z [INFO] agent.router.manager: shutting down +49:30.765 [INFO] TestAgent: consul server down +49:30.765 [INFO] TestAgent: shutdown complete +49:30.765 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23398 network=tcp +49:30.765 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23398 network=udp +49:30.765 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23399 network=tcp +49:31.268 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.268 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogNodes_WanTranslation (1.66s) +=== CONT TestAgent_reloadWatches +2022-12-04T05:49:30.330Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.349Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.352 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a823913a-4373-5d68-f3f0-9625dfd488b7 Address:127.0.0.1:23504}]" +49:30.353 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a823913a-4373-5d68-f3f0-9625dfd488b7.dc1 127.0.0.1 +49:30.354 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a823913a-4373-5d68-f3f0-9625dfd488b7 127.0.0.1 +2022-12-04T05:49:30.354Z [INFO] agent.router: Initializing LAN area manager +49:30.354 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23499 network=udp +49:30.354 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23504 [Follower]" leader= +49:30.356 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a823913a-4373-5d68-f3f0-9625dfd488b7.dc1 area=wan +49:30.356 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23499 network=tcp +49:30.358 [INFO] TestAgent.server: Adding LAN server: server="Node-a823913a-4373-5d68-f3f0-9625dfd488b7 (Addr: tcp/127.0.0.1:23504) (DC: dc1)" +49:30.358 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23500 network=tcp +49:30.358 [INFO] TestAgent: started state syncer +49:30.359 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23554 network=tcp +49:30.393 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.393 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23504 [Candidate]" term=2 +49:30.394 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.394 [DEBUG] TestAgent.server.raft: vote granted: from=a823913a-4373-5d68-f3f0-9625dfd488b7 term=2 tally=1 +49:30.394 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.394 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23504 [Leader]" +49:30.394 [INFO] TestAgent.server: cluster leadership acquired +49:30.394 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23504 +49:30.396 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.396 [INFO] TestAgent.server: New leader elected: payload=Node-a823913a-4373-5d68-f3f0-9625dfd488b7 +49:30.398 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.398 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.398 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.398 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.398 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.398 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a823913a-4373-5d68-f3f0-9625dfd488b7 +49:30.398 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a823913a-4373-5d68-f3f0-9625dfd488b7 +49:30.399 [INFO] TestAgent.server: federation state anti-entropy synced +49:30.612 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:30.613 [INFO] TestAgent: Synced node info +49:30.786 [INFO] TestAgent: Requesting shutdown +49:30.786 [INFO] TestAgent.server: shutting down server +49:30.786 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.786 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.786 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.786 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.786 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.786 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.786 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.786 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.786 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.786 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.786 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:30.786Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.786Z [INFO] agent.router.manager: shutting down +49:30.786 [INFO] TestAgent: consul server down +49:30.786 [INFO] TestAgent: shutdown complete +49:30.786 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23499 network=tcp +49:30.786 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23499 network=udp +49:30.786 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23500 network=tcp +49:31.286 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.286 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogDeregister (0.97s) +=== CONT TestAgent_GetCoordinate +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:30.719Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:30.719Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.728Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:30.729Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.730 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d8dfc4a6-d4e8-e58f-a59f-cdaf709b21f5 Address:127.0.0.1:23483}]" +49:30.730 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23483 [Follower]" leader= +49:30.730 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d8dfc4a6-d4e8-e58f-a59f-cdaf709b21f5.dc1 127.0.0.1 +49:30.731 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d8dfc4a6-d4e8-e58f-a59f-cdaf709b21f5 127.0.0.1 +2022-12-04T05:49:30.731Z [INFO] agent.router: Initializing LAN area manager +49:30.731 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d8dfc4a6-d4e8-e58f-a59f-cdaf709b21f5.dc1 area=wan +49:30.731 [INFO] TestAgent.server: Adding LAN server: server="Node-d8dfc4a6-d4e8-e58f-a59f-cdaf709b21f5 (Addr: tcp/127.0.0.1:23483) (DC: dc1)" +49:30.731 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23506 network=udp +49:30.731 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23506 network=tcp +49:30.732 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23479 network=tcp +49:30.732 [INFO] TestAgent: started state syncer +49:30.732 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23456 network=tcp +49:30.782 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:30.782 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.782 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23483 [Candidate]" term=2 +49:30.782 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.782 [DEBUG] TestAgent.server.raft: vote granted: from=d8dfc4a6-d4e8-e58f-a59f-cdaf709b21f5 term=2 tally=1 +49:30.782 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.782 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23483 [Leader]" +49:30.783 [INFO] TestAgent.server: cluster leadership acquired +49:30.783 [INFO] TestAgent.server: initializing acls +49:30.783 [INFO] TestAgent.server: New leader elected: payload=Node-d8dfc4a6-d4e8-e58f-a59f-cdaf709b21f5 +49:30.783 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:30.783 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:30.784 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:30.785 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:30.785 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:30.785 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:30.785 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d8dfc4a6-d4e8-e58f-a59f-cdaf709b21f5 +49:30.785 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d8dfc4a6-d4e8-e58f-a59f-cdaf709b21f5.dc1 +49:30.786 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.786 [INFO] TestAgent.server: Updating LAN server: server="Node-d8dfc4a6-d4e8-e58f-a59f-cdaf709b21f5 (Addr: tcp/127.0.0.1:23483) (DC: dc1)" +49:30.786 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d8dfc4a6-d4e8-e58f-a59f-cdaf709b21f5.dc1 area=wan +49:30.787 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.787 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.787 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.787 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.787 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.787 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d8dfc4a6-d4e8-e58f-a59f-cdaf709b21f5 +49:30.787 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d8dfc4a6-d4e8-e58f-a59f-cdaf709b21f5 +49:30.788 [INFO] TestAgent.server: federation state anti-entropy synced +49:30.788 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d8dfc4a6-d4e8-e58f-a59f-cdaf709b21f5 +49:30.805 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d8dfc4a6-d4e8-e58f-a59f-cdaf709b21f5 +49:30.805 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d8dfc4a6-d4e8-e58f-a59f-cdaf709b21f5 +49:30.808 [INFO] TestAgent: Requesting shutdown +49:30.808 [INFO] TestAgent.server: shutting down server +49:30.808 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:30.808 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:30.808 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.808 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.808 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.808 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.808 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.808 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:30.808 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:30.808 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.808 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.808 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.808 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:30.808 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.808 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.808 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:30.808Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.808Z [INFO] agent.router.manager: shutting down +49:30.808 [INFO] TestAgent: consul server down +49:30.808 [INFO] TestAgent: shutdown complete +49:30.808 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23506 network=tcp +49:30.808 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23506 network=udp +49:30.808 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23479 network=tcp +49:31.308 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.309 [INFO] TestAgent: Endpoints down +--- PASS: TestRemoteExecWrites_ACLAgentToken (0.60s) +=== CONT TestAgent_purgeCheckState +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=true +2022-12-04T05:49:30.699Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:30.699Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.709Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:30.709Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.710 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:09c9c71b-e757-f54d-6c4d-7fb9ad40e007 Address:127.0.0.1:23455}]" +49:30.710 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23455 [Follower]" leader= +49:30.711 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-09c9c71b-e757-f54d-6c4d-7fb9ad40e007.dc1 127.0.0.1 +49:30.711 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-09c9c71b-e757-f54d-6c4d-7fb9ad40e007 127.0.0.1 +2022-12-04T05:49:30.711Z [INFO] agent.router: Initializing LAN area manager +49:30.711 [INFO] TestAgent.server: Adding LAN server: server="Node-09c9c71b-e757-f54d-6c4d-7fb9ad40e007 (Addr: tcp/127.0.0.1:23455) (DC: dc1)" +49:30.712 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23492 network=udp +49:30.712 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-09c9c71b-e757-f54d-6c4d-7fb9ad40e007.dc1 area=wan +49:30.712 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23492 network=tcp +49:30.712 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23451 network=tcp +49:30.712 [INFO] TestAgent: started state syncer +49:30.712 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23505 network=tcp +49:30.764 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.764 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23455 [Candidate]" term=2 +49:30.764 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:30.764 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.764 [DEBUG] TestAgent.server.raft: vote granted: from=09c9c71b-e757-f54d-6c4d-7fb9ad40e007 term=2 tally=1 +49:30.764 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.764 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23455 [Leader]" +49:30.764 [INFO] TestAgent.server: cluster leadership acquired +49:30.765 [INFO] TestAgent.server: initializing acls +49:30.765 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:30.765 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:30.765 [INFO] TestAgent.server: New leader elected: payload=Node-09c9c71b-e757-f54d-6c4d-7fb9ad40e007 +49:30.765 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:30.766 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:30.766 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:30.766 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:30.766 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-09c9c71b-e757-f54d-6c4d-7fb9ad40e007 +49:30.766 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-09c9c71b-e757-f54d-6c4d-7fb9ad40e007.dc1 +49:30.766 [INFO] TestAgent.server: Updating LAN server: server="Node-09c9c71b-e757-f54d-6c4d-7fb9ad40e007 (Addr: tcp/127.0.0.1:23455) (DC: dc1)" +49:30.766 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-09c9c71b-e757-f54d-6c4d-7fb9ad40e007.dc1 area=wan +49:30.767 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.769 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.769 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.769 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.769 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.769 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.769 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-09c9c71b-e757-f54d-6c4d-7fb9ad40e007 +49:30.769 [INFO] TestAgent.server: member joined, marking health alive: member=Node-09c9c71b-e757-f54d-6c4d-7fb9ad40e007 +49:30.770 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-09c9c71b-e757-f54d-6c4d-7fb9ad40e007 +49:30.770 [INFO] TestAgent.server: federation state anti-entropy synced +49:30.867 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-09c9c71b-e757-f54d-6c4d-7fb9ad40e007 +49:30.867 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-09c9c71b-e757-f54d-6c4d-7fb9ad40e007 +49:30.868 [ERROR] TestAgent: failed to ack remote exec job: error="Permission denied" +49:30.869 [ERROR] TestAgent: failed to write output for remote exec job: error="Permission denied" +49:30.869 [ERROR] TestAgent: failed to write output for remote exec job: error="Permission denied" +49:30.869 [INFO] TestAgent: Requesting shutdown +49:30.869 [INFO] TestAgent.server: shutting down server +49:30.869 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:30.869 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:30.869 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.869 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.869 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.869 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.869 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.869 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:30.869 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.869 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:30.869 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:30.870 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:30.870 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.870 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.870 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.870 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:30.870Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.870Z [INFO] agent.router.manager: shutting down +49:30.870 [INFO] TestAgent: consul server down +49:30.870 [INFO] TestAgent: shutdown complete +49:30.870 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23492 network=tcp +49:30.870 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23492 network=udp +49:30.870 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23451 network=tcp +49:31.371 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.371 [INFO] TestAgent: Endpoints down +--- PASS: TestRemoteExecWrites_ACLDeny (0.68s) +=== CONT TestAgent_loadCheckState +2022-12-04T05:49:30.811Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.821Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.822 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:615a81c1-6839-1a1f-f3c5-e73f4fe66dad Address:127.0.0.1:23511}]" +49:30.822 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-615a81c1-6839-1a1f-f3c5-e73f4fe66dad.dc1 127.0.0.1 +49:30.823 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23511 [Follower]" leader= +49:30.823 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-615a81c1-6839-1a1f-f3c5-e73f4fe66dad 127.0.0.1 +2022-12-04T05:49:30.823Z [INFO] agent.router: Initializing LAN area manager +49:30.823 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23471 network=udp +49:30.823 [INFO] TestAgent.server: Adding LAN server: server="Node-615a81c1-6839-1a1f-f3c5-e73f4fe66dad (Addr: tcp/127.0.0.1:23511) (DC: dc1)" +49:30.823 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-615a81c1-6839-1a1f-f3c5-e73f4fe66dad.dc1 area=wan +49:30.823 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23471 network=tcp +49:30.824 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23507 network=tcp +49:30.824 [INFO] TestAgent: started state syncer +49:30.824 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23526 network=tcp +49:30.873 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.873 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23511 [Candidate]" term=2 +49:30.873 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.873 [DEBUG] TestAgent.server.raft: vote granted: from=615a81c1-6839-1a1f-f3c5-e73f4fe66dad term=2 tally=1 +49:30.873 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.873 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23511 [Leader]" +49:30.873 [INFO] TestAgent.server: cluster leadership acquired +49:30.874 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23511 +49:30.874 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.875 [INFO] TestAgent.server: New leader elected: payload=Node-615a81c1-6839-1a1f-f3c5-e73f4fe66dad +49:30.875 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.875 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.875 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.875 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.875 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.875 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-615a81c1-6839-1a1f-f3c5-e73f4fe66dad +49:30.875 [INFO] TestAgent.server: member joined, marking health alive: member=Node-615a81c1-6839-1a1f-f3c5-e73f4fe66dad +49:30.876 [INFO] TestAgent.server: federation state anti-entropy synced +49:30.913 [INFO] TestAgent: Requesting shutdown +49:30.913 [INFO] TestAgent.server: shutting down server +49:30.913 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:30.913 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:30.913 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:30.913 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:30.913 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:30.914 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:30.914 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:30.914 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" +49:30.914 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:30.914 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:30.914 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:30.914 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:30.915Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:30.915Z [INFO] agent.router.manager: shutting down +49:30.915 [INFO] TestAgent: consul server down +49:30.915 [INFO] TestAgent: shutdown complete +49:30.915 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23471 network=tcp +49:30.915 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23471 network=udp +49:30.915 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23507 network=tcp +49:31.416 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.416 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_consulConfig_RaftTrailingLogs (0.62s) +=== CONT TestAgent_persistCheckState +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=false +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=33)_compressed=true +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:30.833Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.844Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.852 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b4ae71c3-5f4d-4d64-e82e-beeeaa07d125 Address:127.0.0.1:23560}]" +49:30.853 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23560 [Follower]" leader= +49:30.853 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b4ae71c3-5f4d-4d64-e82e-beeeaa07d125.dc1 127.0.0.1 +49:30.855 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b4ae71c3-5f4d-4d64-e82e-beeeaa07d125 127.0.0.1 +2022-12-04T05:49:30.855Z [INFO] agent.router: Initializing LAN area manager +49:30.855 [INFO] TestAgent.server: Adding LAN server: server="Node-b4ae71c3-5f4d-4d64-e82e-beeeaa07d125 (Addr: tcp/127.0.0.1:23560) (DC: dc1)" +49:30.855 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23534 network=udp +49:30.855 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b4ae71c3-5f4d-4d64-e82e-beeeaa07d125.dc1 area=wan +49:30.856 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23534 network=tcp +49:30.857 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23556 network=tcp +49:30.857 [INFO] TestAgent: started state syncer +49:30.858 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23540 network=tcp +49:30.918 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.918 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23560 [Candidate]" term=2 +49:30.918 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.918 [DEBUG] TestAgent.server.raft: vote granted: from=b4ae71c3-5f4d-4d64-e82e-beeeaa07d125 term=2 tally=1 +49:30.918 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.918 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23560 [Leader]" +49:30.918 [INFO] TestAgent.server: cluster leadership acquired +49:30.918 [INFO] TestAgent.server: New leader elected: payload=Node-b4ae71c3-5f4d-4d64-e82e-beeeaa07d125 +49:30.918 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23560 +49:30.919 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.921 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.921 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.921 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.921 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.921 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.921 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b4ae71c3-5f4d-4d64-e82e-beeeaa07d125 +49:30.921 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b4ae71c3-5f4d-4d64-e82e-beeeaa07d125 +49:30.922 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.050 [INFO] TestAgent: Requesting shutdown +49:31.050 [INFO] TestAgent.server: shutting down server +49:31.050 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:31.051 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:31.051 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:31.051 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:31.051 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:31.051 [INFO] TestAgent.server: shutting down Connect CA roots monitor +49:31.051 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:31.051 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:31.051 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:31.051 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:31.051 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:31.051 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:31.051 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:31.051Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:31.051Z [INFO] agent.router.manager: shutting down +49:31.051 [INFO] TestAgent: consul server down +49:31.051 [INFO] TestAgent: shutdown complete +49:31.051 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23534 network=tcp +49:31.051 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23534 network=udp +49:31.051 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23556 network=tcp +49:31.552 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.552 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_consulConfig_AutoEncryptAllowTLS (0.73s) +=== CONT TestAgent_loadChecks_checkFails +2022-12-04T05:49:31.097Z [WARN] TestAgent_ReloadConfigIncomingRPCConfig: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.938Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.958Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.960 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:66b385ab-59f8-6f33-7e36-6d3372d89df1 Address:127.0.0.1:23581}]" +49:30.960 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23581 [Follower]" leader= +49:30.961 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-66b385ab-59f8-6f33-7e36-6d3372d89df1.dc1 127.0.0.1 +49:30.961 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-66b385ab-59f8-6f33-7e36-6d3372d89df1 127.0.0.1 +2022-12-04T05:49:30.961Z [INFO] agent.router: Initializing LAN area manager +49:30.961 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23548 network=udp +49:30.962 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-66b385ab-59f8-6f33-7e36-6d3372d89df1.dc1 area=wan +49:30.962 [INFO] TestAgent.server: Adding LAN server: server="Node-66b385ab-59f8-6f33-7e36-6d3372d89df1 (Addr: tcp/127.0.0.1:23581) (DC: dc1)" +49:30.962 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23548 network=tcp +49:30.962 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23577 network=tcp +49:30.962 [INFO] TestAgent: started state syncer +49:30.962 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23610 network=tcp +49:31.024 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:31.024 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23581 [Candidate]" term=2 +49:31.024 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:31.024 [DEBUG] TestAgent.server.raft: vote granted: from=66b385ab-59f8-6f33-7e36-6d3372d89df1 term=2 tally=1 +49:31.024 [INFO] TestAgent.server.raft: election won: tally=1 +49:31.024 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23581 [Leader]" +49:31.024 [INFO] TestAgent.server: cluster leadership acquired +49:31.025 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23581 +49:31.025 [INFO] TestAgent.server: New leader elected: payload=Node-66b385ab-59f8-6f33-7e36-6d3372d89df1 +49:31.026 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:31.027 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:31.027 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:31.027 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:31.027 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:31.027 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:31.027 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-66b385ab-59f8-6f33-7e36-6d3372d89df1 +49:31.027 [INFO] TestAgent.server: member joined, marking health alive: member=Node-66b385ab-59f8-6f33-7e36-6d3372d89df1 +49:31.028 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.098 [INFO] TestAgent: Requesting shutdown +49:31.098 [INFO] TestAgent.server: shutting down server +49:31.098 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:31.098 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:31.098 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:31.098 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:31.098Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:31.098Z [INFO] agent.router.manager: shutting down +49:31.098 [INFO] TestAgent: consul server down +49:31.098 [INFO] TestAgent: shutdown complete +49:31.098 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23548 network=tcp +49:31.099 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23548 network=udp +49:31.099 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23577 network=tcp +49:31.599 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.599 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ReloadConfigIncomingRPCConfig (0.68s) +=== CONT TestAgent_checkStateSnapshot +2022-12-04T05:49:30.830Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.840Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.841 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2b4c527b-e859-46b8-46f6-a0d3a2f8b5d5 Address:127.0.0.1:23532}]" +49:30.841 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23532 [Follower]" leader= +49:30.842 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2b4c527b-e859-46b8-46f6-a0d3a2f8b5d5.dc1 127.0.0.1 +49:30.842 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2b4c527b-e859-46b8-46f6-a0d3a2f8b5d5 127.0.0.1 +2022-12-04T05:49:30.842Z [INFO] agent.router: Initializing LAN area manager +49:30.842 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23527 network=udp +49:30.842 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2b4c527b-e859-46b8-46f6-a0d3a2f8b5d5.dc1 area=wan +49:30.842 [INFO] TestAgent.server: Adding LAN server: server="Node-2b4c527b-e859-46b8-46f6-a0d3a2f8b5d5 (Addr: tcp/127.0.0.1:23532) (DC: dc1)" +49:30.842 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23527 network=tcp +49:30.843 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23528 network=tcp +49:30.843 [INFO] TestAgent: started state syncer +49:30.843 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23533 network=tcp +49:30.900 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.900 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23532 [Candidate]" term=2 +49:30.901 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.901 [DEBUG] TestAgent.server.raft: vote granted: from=2b4c527b-e859-46b8-46f6-a0d3a2f8b5d5 term=2 tally=1 +49:30.901 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.901 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23532 [Leader]" +49:30.901 [INFO] TestAgent.server: cluster leadership acquired +49:30.901 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23532 +49:30.902 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.904 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.904 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.904 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.904 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.904 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.904 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2b4c527b-e859-46b8-46f6-a0d3a2f8b5d5 +49:30.904 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2b4c527b-e859-46b8-46f6-a0d3a2f8b5d5 +49:30.904 [INFO] TestAgent.server: New leader elected: payload=Node-2b4c527b-e859-46b8-46f6-a0d3a2f8b5d5 +49:30.905 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.052 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:31.052 [INFO] TestAgent: Synced node info +49:31.164 [INFO] TestAgent: Requesting shutdown +49:31.164 [INFO] TestAgent.server: shutting down server +49:31.164 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:31.164 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:31.164 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:31.164 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:31.164 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:31.165 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:31.165 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:31.165 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:31.165 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:31.165 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:31.165 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:31.165Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:31.165Z [INFO] agent.router.manager: shutting down +49:31.165 [INFO] TestAgent: consul server down +49:31.165 [INFO] TestAgent: shutdown complete +49:31.165 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23527 network=tcp +49:31.165 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23527 network=udp +49:31.165 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23528 network=tcp +49:31.666 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.666 [INFO] TestAgent: Endpoints down +=== RUN TestConnectCAConfig/updating_config_with_same_state +=== RUN TestAgent_loadTokens/original-configuration +=== RUN TestAgent_loadTokens/updated-configuration +=== RUN TestAgent_loadTokens/persisted-tokens +=== RUN TestAgent_loadTokens/persisted-tokens-override +=== RUN TestAgent_loadTokens/partial-persisted +=== RUN TestAgent_loadTokens/persistence-error-not-json +=== RUN TestAgent_loadTokens/persistence-error-wrong-top-level +2022-12-04T05:49:30.796Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.808Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.810 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:38fac513-c88e-42cb-3cb9-b11acfe012fe Address:127.0.0.1:23469}]" +49:30.810 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23469 [Follower]" leader= +49:30.810 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-38fac513-c88e-42cb-3cb9-b11acfe012fe.dc1 127.0.0.1 +49:30.811 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-38fac513-c88e-42cb-3cb9-b11acfe012fe 127.0.0.1 +2022-12-04T05:49:30.811Z [INFO] agent.router: Initializing LAN area manager +49:30.811 [INFO] TestAgent.server: Adding LAN server: server="Node-38fac513-c88e-42cb-3cb9-b11acfe012fe (Addr: tcp/127.0.0.1:23469) (DC: dc1)" +49:30.811 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-38fac513-c88e-42cb-3cb9-b11acfe012fe.dc1 area=wan +49:30.811 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23457 network=udp +49:30.811 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23457 network=tcp +49:30.812 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23465 network=tcp +49:30.812 [INFO] TestAgent: started state syncer +49:30.812 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23470 network=tcp +49:30.870 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.870 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23469 [Candidate]" term=2 +49:30.870 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.870 [DEBUG] TestAgent.server.raft: vote granted: from=38fac513-c88e-42cb-3cb9-b11acfe012fe term=2 tally=1 +49:30.870 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.870 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23469 [Leader]" +49:30.871 [INFO] TestAgent.server: cluster leadership acquired +49:30.871 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23469 +49:30.872 [INFO] TestAgent.server: New leader elected: payload=Node-38fac513-c88e-42cb-3cb9-b11acfe012fe +49:30.873 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.874 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.874 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.874 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.874 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.874 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.874 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-38fac513-c88e-42cb-3cb9-b11acfe012fe +49:30.874 [INFO] TestAgent.server: member joined, marking health alive: member=Node-38fac513-c88e-42cb-3cb9-b11acfe012fe +49:30.875 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.188 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:31.189 [INFO] TestAgent: Synced node info +49:31.189 [DEBUG] TestAgent: Node info in sync +49:31.211 [INFO] TestAgent: Service entered maintenance mode: service=test +49:31.211 [DEBUG] TestAgent: removed check: check=_service_maintenance:test +49:31.211 [INFO] TestAgent: Service left maintenance mode: service=test +49:31.211 [DEBUG] TestAgent: Node info in sync +49:31.212 [INFO] TestAgent: Synced service: service=test +49:31.212 [INFO] TestAgent: Deregistered check: check=_service_maintenance:test +49:31.212 [INFO] TestAgent: Requesting shutdown +49:31.213 [INFO] TestAgent.server: shutting down server +49:31.213 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:31.213 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:31.213 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:31.213 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:31.213 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:31.213 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:31.213 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:31.213 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:31.213 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:31.213 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:31.213 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:31.213Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:31.213Z [INFO] agent.router.manager: shutting down +49:31.213 [INFO] TestAgent: consul server down +49:31.213 [INFO] TestAgent: shutdown complete +49:31.213 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23457 network=tcp +49:31.213 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23457 network=udp +49:31.213 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23465 network=tcp +49:31.714 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.714 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ServiceMaintenance_Disable (0.93s) +=== CONT TestAgent_NodeMaintenanceMode +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:31.336Z [WARN] TestAgent_ReloadConfigTLSConfigFailure: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.879Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.896Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.899 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8aabc4a9-7494-f167-2c44-17a0518cbbcd Address:127.0.0.1:23546}]" +49:30.900 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23546 [Follower]" leader= +49:30.908 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8aabc4a9-7494-f167-2c44-17a0518cbbcd.dc1 127.0.0.1 +49:30.909 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8aabc4a9-7494-f167-2c44-17a0518cbbcd 127.0.0.1 +2022-12-04T05:49:30.909Z [INFO] agent.router: Initializing LAN area manager +49:30.910 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23541 network=udp +49:30.912 [INFO] TestAgent.server: Adding LAN server: server="Node-8aabc4a9-7494-f167-2c44-17a0518cbbcd (Addr: tcp/127.0.0.1:23546) (DC: dc1)" +49:30.912 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23541 network=tcp +49:30.913 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8aabc4a9-7494-f167-2c44-17a0518cbbcd.dc1 area=wan +49:30.915 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23542 network=tcp +49:30.915 [INFO] TestAgent: started state syncer +49:30.915 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23547 network=tcp +49:30.955 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.955 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23546 [Candidate]" term=2 +49:30.955 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:30.955 [DEBUG] TestAgent.server.raft: vote granted: from=8aabc4a9-7494-f167-2c44-17a0518cbbcd term=2 tally=1 +49:30.955 [INFO] TestAgent.server.raft: election won: tally=1 +49:30.955 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23546 [Leader]" +49:30.955 [INFO] TestAgent.server: cluster leadership acquired +49:30.955 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23546 +49:30.956 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.956 [INFO] TestAgent.server: New leader elected: payload=Node-8aabc4a9-7494-f167-2c44-17a0518cbbcd +49:30.957 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.957 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:30.957 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:30.957 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:30.957 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:30.957 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8aabc4a9-7494-f167-2c44-17a0518cbbcd +49:30.957 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8aabc4a9-7494-f167-2c44-17a0518cbbcd +49:30.958 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.301 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:31.301 [INFO] TestAgent: Synced node info +49:31.301 [DEBUG] TestAgent: Node info in sync +49:31.336 [INFO] TestAgent: Requesting shutdown +49:31.336 [INFO] TestAgent.server: shutting down server +49:31.336 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:31.337 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:31.337 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:31.337Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:31.337Z [INFO] agent.router.manager: shutting down +49:31.337 [INFO] TestAgent: consul server down +49:31.337 [INFO] TestAgent: shutdown complete +49:31.337 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23541 network=tcp +49:31.337 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23541 network=udp +49:31.337 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23542 network=tcp +49:31.838 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.838 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ReloadConfigTLSConfigFailure (0.98s) +=== CONT TestAgent_AddCheck_restoresSnapshot +2022-12-04T05:49:31.290Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.302Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:31.305 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:416852a2-8ce9-096a-1b87-bd170ef358ae Address:127.0.0.1:23658}]" +49:31.305 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23658 [Follower]" leader= +49:31.306 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-416852a2-8ce9-096a-1b87-bd170ef358ae.dc1 127.0.0.1 +49:31.307 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-416852a2-8ce9-096a-1b87-bd170ef358ae 127.0.0.1 +2022-12-04T05:49:31.307Z [INFO] agent.router: Initializing LAN area manager +49:31.307 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23639 network=udp +49:31.308 [INFO] TestAgent.server: Adding LAN server: server="Node-416852a2-8ce9-096a-1b87-bd170ef358ae (Addr: tcp/127.0.0.1:23658) (DC: dc1)" +49:31.308 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-416852a2-8ce9-096a-1b87-bd170ef358ae.dc1 area=wan +49:31.308 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23639 network=tcp +49:31.309 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23654 network=tcp +49:31.309 [INFO] TestAgent: started state syncer +49:31.309 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23631 network=tcp +49:31.365 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:31.365 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23658 [Candidate]" term=2 +49:31.366 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:31.366 [DEBUG] TestAgent.server.raft: vote granted: from=416852a2-8ce9-096a-1b87-bd170ef358ae term=2 tally=1 +49:31.366 [INFO] TestAgent.server.raft: election won: tally=1 +49:31.366 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23658 [Leader]" +49:31.366 [INFO] TestAgent.server: cluster leadership acquired +49:31.366 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23658 +49:31.368 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:31.368 [INFO] TestAgent.server: New leader elected: payload=Node-416852a2-8ce9-096a-1b87-bd170ef358ae +49:31.370 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:31.370 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:31.370 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:31.370 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:31.370 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:31.370 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-416852a2-8ce9-096a-1b87-bd170ef358ae +49:31.370 [INFO] TestAgent.server: member joined, marking health alive: member=Node-416852a2-8ce9-096a-1b87-bd170ef358ae +49:31.371 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.444 [INFO] TestAgent: Requesting shutdown +49:31.444 [INFO] TestAgent.server: shutting down server +49:31.444 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:31.444 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:31.444 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:31.444 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:31.444 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:31.444 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:31.444 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:31.444 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:31.444 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:31.444 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:31.444 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:31.444 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:31.444Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:31.444Z [INFO] agent.router.manager: shutting down +49:31.445 [INFO] TestAgent: consul server down +49:31.445 [INFO] TestAgent: shutdown complete +49:31.445 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23639 network=tcp +49:31.445 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23639 network=udp +49:31.445 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23654 network=tcp +49:31.945 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.945 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_reloadWatches (0.68s) +=== CONT TestAgent_Service_MaintenanceMode +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=33)_compressed=false +2022-12-04T05:49:31.273Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.290Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:31.292 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0f288ede-9b92-65c3-0583-c92eb23fd335 Address:127.0.0.1:23623}]" +49:31.292 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23623 [Follower]" leader= +49:31.292 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0f288ede-9b92-65c3-0583-c92eb23fd335.dc1 127.0.0.1 +49:31.293 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0f288ede-9b92-65c3-0583-c92eb23fd335 127.0.0.1 +2022-12-04T05:49:31.293Z [INFO] agent.router: Initializing LAN area manager +49:31.293 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23660 network=udp +49:31.293 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23660 network=tcp +49:31.293 [INFO] TestAgent.server: Adding LAN server: server="Node-0f288ede-9b92-65c3-0583-c92eb23fd335 (Addr: tcp/127.0.0.1:23623) (DC: dc1)" +49:31.293 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0f288ede-9b92-65c3-0583-c92eb23fd335.dc1 area=wan +49:31.294 [INFO] TestAgent: Started HTTPS server: address=127.0.0.1:23620 network=tcp +49:31.294 [INFO] TestAgent: started state syncer +49:31.294 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23638 network=tcp +49:31.333 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:31.333 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23623 [Candidate]" term=2 +49:31.333 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:31.333 [DEBUG] TestAgent.server.raft: vote granted: from=0f288ede-9b92-65c3-0583-c92eb23fd335 term=2 tally=1 +49:31.333 [INFO] TestAgent.server.raft: election won: tally=1 +49:31.333 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23623 [Leader]" +49:31.333 [INFO] TestAgent.server: cluster leadership acquired +49:31.333 [INFO] TestAgent.server: New leader elected: payload=Node-0f288ede-9b92-65c3-0583-c92eb23fd335 +49:31.333 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23623 +49:31.334 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:31.335 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:31.335 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:31.335 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:31.335 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:31.335 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:31.335 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0f288ede-9b92-65c3-0583-c92eb23fd335 +49:31.335 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0f288ede-9b92-65c3-0583-c92eb23fd335 +49:31.336 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.427 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:31.428 [INFO] TestAgent: Synced node info +49:31.464 [INFO] TestAgent: Requesting shutdown +49:31.464 [INFO] TestAgent.server: shutting down server +49:31.464 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:31.464 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:31.464 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:31.464 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:31.464 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:31.464 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:31.464 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:31.464 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:31.464 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:31.464 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:31.464 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:31.464Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:31.464Z [INFO] agent.router.manager: shutting down +49:31.464 [INFO] TestAgent: consul server down +49:31.464 [INFO] TestAgent: shutdown complete +49:31.464 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23660 network=tcp +49:31.464 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23660 network=udp +49:31.464 [INFO] TestAgent: Stopping server: protocol=HTTPS address=127.0.0.1:23620 network=tcp +49:31.965 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.965 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_reloadWatchesHTTPS (0.72s) +=== CONT TestAgent_unloadChecks +2022-12-04T05:49:31.466Z [WARN] TestAgent_ReloadConfigOutgoingRPCConfig: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.048Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.058Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:31.060 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c178e61f-7b43-8a06-073d-210c21182187 Address:127.0.0.1:23588}]" +49:31.060 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23588 [Follower]" leader= +49:31.061 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c178e61f-7b43-8a06-073d-210c21182187.dc1 127.0.0.1 +49:31.061 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c178e61f-7b43-8a06-073d-210c21182187 127.0.0.1 +2022-12-04T05:49:31.061Z [INFO] agent.router: Initializing LAN area manager +49:31.062 [INFO] TestAgent.server: Adding LAN server: server="Node-c178e61f-7b43-8a06-073d-210c21182187 (Addr: tcp/127.0.0.1:23588) (DC: dc1)" +49:31.062 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c178e61f-7b43-8a06-073d-210c21182187.dc1 area=wan +49:31.062 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23611 network=udp +49:31.062 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23611 network=tcp +49:31.062 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23584 network=tcp +49:31.062 [INFO] TestAgent: started state syncer +49:31.062 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23575 network=tcp +49:31.108 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:31.108 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23588 [Candidate]" term=2 +49:31.109 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:31.109 [DEBUG] TestAgent.server.raft: vote granted: from=c178e61f-7b43-8a06-073d-210c21182187 term=2 tally=1 +49:31.109 [INFO] TestAgent.server.raft: election won: tally=1 +49:31.109 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23588 [Leader]" +49:31.109 [INFO] TestAgent.server: cluster leadership acquired +49:31.109 [INFO] TestAgent.server: New leader elected: payload=Node-c178e61f-7b43-8a06-073d-210c21182187 +49:31.109 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23588 +49:31.110 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:31.112 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:31.112 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:31.112 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:31.112 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:31.112 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:31.113 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c178e61f-7b43-8a06-073d-210c21182187 +49:31.113 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c178e61f-7b43-8a06-073d-210c21182187 +49:31.114 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.436 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:31.436 [INFO] TestAgent: Synced node info +49:31.467 [INFO] TestAgent: Requesting shutdown +49:31.467 [INFO] TestAgent.server: shutting down server +49:31.467 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:31.467 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:31.467 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:31.467Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:31.467Z [INFO] agent.router.manager: shutting down +49:31.467 [INFO] TestAgent: consul server down +49:31.467 [INFO] TestAgent: shutdown complete +49:31.467 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23611 network=tcp +49:31.467 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23611 network=udp +49:31.467 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23584 network=tcp +49:31.967 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.967 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ReloadConfigOutgoingRPCConfig (0.93s) +=== CONT TestAgent_loadChecks_token +2022-12-04T05:49:31.332Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.360Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:31.363 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6205bf29-c7fd-cbbe-84a8-c2cc04265ff2 Address:127.0.0.1:23567}]" +49:31.363 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6205bf29-c7fd-cbbe-84a8-c2cc04265ff2.dc1 127.0.0.1 +49:31.364 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23567 [Follower]" leader= +49:31.372 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6205bf29-c7fd-cbbe-84a8-c2cc04265ff2 127.0.0.1 +2022-12-04T05:49:31.372Z [INFO] agent.router: Initializing LAN area manager +49:31.373 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23590 network=udp +49:31.373 [INFO] TestAgent.server: Adding LAN server: server="Node-6205bf29-c7fd-cbbe-84a8-c2cc04265ff2 (Addr: tcp/127.0.0.1:23567) (DC: dc1)" +49:31.373 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6205bf29-c7fd-cbbe-84a8-c2cc04265ff2.dc1 area=wan +49:31.373 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23590 network=tcp +49:31.375 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23563 network=tcp +49:31.375 [INFO] TestAgent: started state syncer +49:31.375 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23596 network=tcp +49:31.430 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:31.430 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23567 [Candidate]" term=2 +49:31.430 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:31.430 [DEBUG] TestAgent.server.raft: vote granted: from=6205bf29-c7fd-cbbe-84a8-c2cc04265ff2 term=2 tally=1 +49:31.430 [INFO] TestAgent.server.raft: election won: tally=1 +49:31.430 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23567 [Leader]" +49:31.431 [INFO] TestAgent.server: cluster leadership acquired +49:31.431 [INFO] TestAgent.server: New leader elected: payload=Node-6205bf29-c7fd-cbbe-84a8-c2cc04265ff2 +49:31.431 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23567 +49:31.432 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:31.433 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:31.433 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:31.433 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:31.433 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:31.433 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:31.433 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6205bf29-c7fd-cbbe-84a8-c2cc04265ff2 +49:31.433 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6205bf29-c7fd-cbbe-84a8-c2cc04265ff2 +49:31.434 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.479 [INFO] TestAgent: Requesting shutdown +49:31.480 [INFO] TestAgent.server: shutting down server +49:31.480 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:31.480 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:31.480 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:31.480 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:31.480 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:31.480 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:31.480 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:31.480 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:31.480 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:31.480 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:31.480 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:31.480 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:31.480Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:31.480Z [INFO] agent.router.manager: shutting down +49:31.480 [INFO] TestAgent: consul server down +49:31.480 [INFO] TestAgent: shutdown complete +49:31.480 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23590 network=tcp +49:31.480 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23590 network=udp +49:31.480 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23563 network=tcp +49:31.980 [INFO] TestAgent: Waiting for endpoints to shut down +49:31.980 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_purgeCheckState (0.67s) +=== CONT TestAgent_DeregisterPersistedSidecarAfterRestart +--- PASS: TestAgent_SecurityChecks (0.94s) +=== CONT TestAgent_PurgeCheckOnDuplicate +2022-12-04T05:49:31.429Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.440Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:31.441 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f3551b37-9d74-82b3-a861-b53628a69e1f Address:127.0.0.1:23644}]" +49:31.442 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23644 [Follower]" leader= +49:31.442 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f3551b37-9d74-82b3-a861-b53628a69e1f.dc1 127.0.0.1 +49:31.443 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f3551b37-9d74-82b3-a861-b53628a69e1f 127.0.0.1 +2022-12-04T05:49:31.443Z [INFO] agent.router: Initializing LAN area manager +49:31.443 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f3551b37-9d74-82b3-a861-b53628a69e1f.dc1 area=wan +49:31.443 [INFO] TestAgent.server: Adding LAN server: server="Node-f3551b37-9d74-82b3-a861-b53628a69e1f (Addr: tcp/127.0.0.1:23644) (DC: dc1)" +49:31.443 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23618 network=udp +49:31.443 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23618 network=tcp +49:31.444 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23640 network=tcp +49:31.444 [INFO] TestAgent: started state syncer +49:31.444 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23568 network=tcp +49:31.510 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:31.510 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23644 [Candidate]" term=2 +49:31.510 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:31.510 [DEBUG] TestAgent.server.raft: vote granted: from=f3551b37-9d74-82b3-a861-b53628a69e1f term=2 tally=1 +49:31.510 [INFO] TestAgent.server.raft: election won: tally=1 +49:31.510 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23644 [Leader]" +49:31.510 [INFO] TestAgent.server: cluster leadership acquired +49:31.511 [INFO] TestAgent.server: New leader elected: payload=Node-f3551b37-9d74-82b3-a861-b53628a69e1f +49:31.511 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23644 +49:31.511 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:31.513 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:31.513 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:31.513 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:31.513 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:31.513 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:31.513 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f3551b37-9d74-82b3-a861-b53628a69e1f +49:31.513 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f3551b37-9d74-82b3-a861-b53628a69e1f +49:31.514 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.548 [INFO] TestAgent: Requesting shutdown +49:31.548 [INFO] TestAgent.server: shutting down server +49:31.548 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:31.548 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:31.548 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:31.548 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:31.548 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:31.548 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:31.548 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:31.548 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:31.549 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:31.549 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:31.549 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:31.549 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:31.549Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:31.549Z [INFO] agent.router.manager: shutting down +49:31.549 [INFO] TestAgent: consul server down +49:31.549 [INFO] TestAgent: shutdown complete +49:31.549 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23618 network=tcp +49:31.549 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23618 network=udp +49:31.549 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23640 network=tcp +49:32.049 [INFO] TestAgent: Waiting for endpoints to shut down +49:32.049 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_persistCheckState (0.63s) +=== CONT TestAgent_PurgeCheck +[WARN] freeport: 10 out of 59 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:31.389Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.401Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:31.402 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1323c00e-8b86-934d-7197-79df34a4e625 Address:127.0.0.1:23574}]" +49:31.403 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23574 [Follower]" leader= +49:31.403 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1323c00e-8b86-934d-7197-79df34a4e625.dc1 127.0.0.1 +49:31.404 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1323c00e-8b86-934d-7197-79df34a4e625 127.0.0.1 +2022-12-04T05:49:31.404Z [INFO] agent.router: Initializing LAN area manager +49:31.404 [INFO] TestAgent.server: Adding LAN server: server="Node-1323c00e-8b86-934d-7197-79df34a4e625 (Addr: tcp/127.0.0.1:23574) (DC: dc1)" +49:31.404 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1323c00e-8b86-934d-7197-79df34a4e625.dc1 area=wan +49:31.404 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23597 network=udp +49:31.404 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23597 network=tcp +49:31.404 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23570 network=tcp +49:31.405 [INFO] TestAgent: started state syncer +49:31.405 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23617 network=tcp +49:31.459 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:31.459 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23574 [Candidate]" term=2 +49:31.459 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:31.459 [DEBUG] TestAgent.server.raft: vote granted: from=1323c00e-8b86-934d-7197-79df34a4e625 term=2 tally=1 +49:31.459 [INFO] TestAgent.server.raft: election won: tally=1 +49:31.459 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23574 [Leader]" +49:31.460 [INFO] TestAgent.server: cluster leadership acquired +49:31.460 [INFO] TestAgent.server: New leader elected: payload=Node-1323c00e-8b86-934d-7197-79df34a4e625 +49:31.460 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23574 +49:31.460 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:31.462 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:31.462 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:31.462 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:31.462 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:31.462 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:31.462 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1323c00e-8b86-934d-7197-79df34a4e625 +49:31.462 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1323c00e-8b86-934d-7197-79df34a4e625 +49:31.462 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.593 [DEBUG] TestAgent: check state expired, not restoring: check=check1 +49:31.593 [INFO] TestAgent: Requesting shutdown +49:31.594 [INFO] TestAgent.server: shutting down server +49:31.594 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:31.594 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:31.594 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:31.594 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:31.594 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:31.594 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:31.594 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:31.594 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:31.594 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:31.594 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:31.594 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:31.594 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:49:31.594Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:31.594Z [INFO] agent.router.manager: shutting down +49:31.594 [INFO] TestAgent: consul server down +49:31.594 [INFO] TestAgent: shutdown complete +49:31.594 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23597 network=tcp +49:31.594 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23597 network=udp +49:31.594 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23570 network=tcp +49:32.094 [INFO] TestAgent: Waiting for endpoints to shut down +49:32.094 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_loadCheckState (0.72s) +=== CONT TestAgent_PersistCheck +2022-12-04T05:49:31.231Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.246Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:31.248 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cb4fb5c6-32a0-34a1-4c65-325311765289 Address:127.0.0.1:23602}]" +49:31.248 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23602 [Follower]" leader= +49:31.249 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cb4fb5c6-32a0-34a1-4c65-325311765289.dc1 127.0.0.1 +49:31.249 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cb4fb5c6-32a0-34a1-4c65-325311765289 127.0.0.1 +2022-12-04T05:49:31.249Z [INFO] agent.router: Initializing LAN area manager +49:31.250 [INFO] TestAgent.server: Adding LAN server: server="Node-cb4fb5c6-32a0-34a1-4c65-325311765289 (Addr: tcp/127.0.0.1:23602) (DC: dc1)" +49:31.250 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23583 network=udp +49:31.250 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cb4fb5c6-32a0-34a1-4c65-325311765289.dc1 area=wan +49:31.250 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23583 network=tcp +49:31.250 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23598 network=tcp +49:31.250 [INFO] TestAgent: started state syncer +49:31.250 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23659 network=tcp +49:31.300 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:31.309 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:31.309 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23602 [Candidate]" term=2 +49:31.310 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:31.310 [DEBUG] TestAgent.server.raft: vote granted: from=cb4fb5c6-32a0-34a1-4c65-325311765289 term=2 tally=1 +49:31.310 [INFO] TestAgent.server.raft: election won: tally=1 +49:31.310 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23602 [Leader]" +49:31.310 [INFO] TestAgent.server: cluster leadership acquired +49:31.310 [INFO] TestAgent.server: initializing acls +49:31.310 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:31.311 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:31.311 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:31.311 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:31.311 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-cb4fb5c6-32a0-34a1-4c65-325311765289 +49:31.311 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-cb4fb5c6-32a0-34a1-4c65-325311765289.dc1 +49:31.312 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:31.312 [INFO] TestAgent.server: New leader elected: payload=Node-cb4fb5c6-32a0-34a1-4c65-325311765289 +49:31.312 [INFO] TestAgent.server: Updating LAN server: server="Node-cb4fb5c6-32a0-34a1-4c65-325311765289 (Addr: tcp/127.0.0.1:23602) (DC: dc1)" +49:31.312 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-cb4fb5c6-32a0-34a1-4c65-325311765289.dc1 area=wan +49:31.314 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:31.314 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:31.314 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:31.314 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:31.314 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:31.314 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cb4fb5c6-32a0-34a1-4c65-325311765289 +49:31.314 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cb4fb5c6-32a0-34a1-4c65-325311765289 +49:31.314 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cb4fb5c6-32a0-34a1-4c65-325311765289 +49:31.315 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.685 [WARN] TestAgent: "default" token present in both the configuration and persisted token store, using the persisted token +49:31.685 [WARN] TestAgent: "agent" token present in both the configuration and persisted token store, using the persisted token +49:31.685 [WARN] TestAgent: "agent_master" token present in both the configuration and persisted token store, using the persisted token +49:31.685 [WARN] TestAgent: "replication" token present in both the configuration and persisted token store, using the persisted token +49:31.685 [WARN] TestAgent: "default" token present in both the configuration and persisted token store, using the persisted token +49:31.685 [WARN] TestAgent: "agent" token present in both the configuration and persisted token store, using the persisted token +49:31.685 [WARN] TestAgent: "agent_master" token present in both the configuration and persisted token store, using the persisted token +49:31.685 [WARN] TestAgent: "replication" token present in both the configuration and persisted token store, using the persisted token +49:31.686 [WARN] TestAgent: "agent" token present in both the configuration and persisted token store, using the persisted token +49:31.686 [WARN] TestAgent: "agent_master" token present in both the configuration and persisted token store, using the persisted token +49:31.686 [WARN] TestAgent: unable to load persisted tokens: error="failed to decode tokens file "/tmp/consul-test/TestAgent_loadTokens-TestAgent-agent728177745/acl-tokens.json": invalid character '\x01' looking for beginning of value" +49:31.686 [WARN] TestAgent: unable to load persisted tokens: error="failed to decode tokens file "/tmp/consul-test/TestAgent_loadTokens-TestAgent-agent728177745/acl-tokens.json": json: cannot unmarshal array into Go value of type agent.persistedTokens" +49:31.686 [INFO] TestAgent: Requesting shutdown +49:31.686 [INFO] TestAgent.server: shutting down server +49:31.686 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:31.686 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:31.686 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:31.686 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:31.686 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:31.686 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:31.686 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:31.686 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:31.686 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:31.686 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:31.686 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:31.686 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:31.686 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:31.686 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:31.686 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:31.687 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:31.687Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:31.687Z [INFO] agent.router.manager: shutting down +49:31.687 [INFO] TestAgent: consul server down +49:31.687 [INFO] TestAgent: shutdown complete +49:31.687 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23583 network=tcp +49:31.687 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23583 network=udp +49:31.687 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23598 network=tcp +49:32.189 [INFO] TestAgent: Waiting for endpoints to shut down +49:32.189 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_loadTokens (0.99s) + --- PASS: TestAgent_loadTokens/original-configuration (0.00s) + --- PASS: TestAgent_loadTokens/updated-configuration (0.00s) + --- PASS: TestAgent_loadTokens/persisted-tokens (0.00s) + --- PASS: TestAgent_loadTokens/persisted-tokens-override (0.00s) + --- PASS: TestAgent_loadTokens/partial-persisted (0.00s) + --- PASS: TestAgent_loadTokens/persistence-error-not-json (0.00s) + --- PASS: TestAgent_loadTokens/persistence-error-wrong-top-level (0.00s) +=== CONT TestAgent_updateTTLCheck +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:31.615Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.625Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:31.626 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c6302214-b3fc-9cc2-cefa-f1068903d609 Address:127.0.0.1:23686}]" +49:31.627 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23686 [Follower]" leader= +49:31.627 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c6302214-b3fc-9cc2-cefa-f1068903d609.dc1 127.0.0.1 +49:31.627 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c6302214-b3fc-9cc2-cefa-f1068903d609 127.0.0.1 +2022-12-04T05:49:31.627Z [INFO] agent.router: Initializing LAN area manager +49:31.628 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23681 network=udp +49:31.628 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c6302214-b3fc-9cc2-cefa-f1068903d609.dc1 area=wan +49:31.628 [INFO] TestAgent.server: Adding LAN server: server="Node-c6302214-b3fc-9cc2-cefa-f1068903d609 (Addr: tcp/127.0.0.1:23686) (DC: dc1)" +49:31.628 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23681 network=tcp +49:31.628 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23682 network=tcp +49:31.628 [INFO] TestAgent: started state syncer +49:31.629 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23652 network=tcp +49:31.664 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:31.664 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23686 [Candidate]" term=2 +49:31.664 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:31.664 [DEBUG] TestAgent.server.raft: vote granted: from=c6302214-b3fc-9cc2-cefa-f1068903d609 term=2 tally=1 +49:31.664 [INFO] TestAgent.server.raft: election won: tally=1 +49:31.664 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23686 [Leader]" +49:31.664 [INFO] TestAgent.server: cluster leadership acquired +49:31.664 [INFO] TestAgent.server: New leader elected: payload=Node-c6302214-b3fc-9cc2-cefa-f1068903d609 +49:31.664 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23686 +49:31.665 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:31.667 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:31.667 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:31.667 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:31.667 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:31.667 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:31.667 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c6302214-b3fc-9cc2-cefa-f1068903d609 +49:31.667 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c6302214-b3fc-9cc2-cefa-f1068903d609 +49:31.668 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.819 [DEBUG] TestAgent: removed check: check=service:redis +49:31.820 [DEBUG] TestAgent: restored health check from file: check=service:redis file=/tmp/consul-test/TestAgent_checkStateSnapshot-TestAgent-agent836910105/checks/60a2ef12de014a05ecdc850d9aab46da +49:31.820 [INFO] TestAgent: Requesting shutdown +49:31.820 [INFO] TestAgent.server: shutting down server +49:31.820 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:31.820 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:31.820 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:31.820 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:31.820 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:31.820 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:31.820 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:31.820 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:31.820 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:31.820 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:31.820Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:31.820Z [INFO] agent.router.manager: shutting down +49:31.821 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:31.821 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:31.821 [INFO] TestAgent: consul server down +49:31.821 [INFO] TestAgent: shutdown complete +49:31.821 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23681 network=tcp +49:31.821 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23681 network=udp +49:31.821 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23682 network=tcp +49:32.322 [INFO] TestAgent: Waiting for endpoints to shut down +49:32.322 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_checkStateSnapshot (0.72s) +=== CONT TestAgent_HTTPCheck_TLSSkipVerify +2022-12-04T05:49:31.565Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.574Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:31.575 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e37c1e53-6c46-7162-1768-d26bbd0cea3d Address:127.0.0.1:23595}]" +49:31.575 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23595 [Follower]" leader= +49:31.576 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e37c1e53-6c46-7162-1768-d26bbd0cea3d.dc1 127.0.0.1 +49:31.576 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e37c1e53-6c46-7162-1768-d26bbd0cea3d 127.0.0.1 +2022-12-04T05:49:31.576Z [INFO] agent.router: Initializing LAN area manager +49:31.576 [INFO] TestAgent.server: Adding LAN server: server="Node-e37c1e53-6c46-7162-1768-d26bbd0cea3d (Addr: tcp/127.0.0.1:23595) (DC: dc1)" +49:31.576 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e37c1e53-6c46-7162-1768-d26bbd0cea3d.dc1 area=wan +49:31.577 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23569 network=udp +49:31.577 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23569 network=tcp +49:31.577 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23591 network=tcp +49:31.577 [INFO] TestAgent: started state syncer +49:31.577 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23680 network=tcp +49:31.619 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:31.619 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23595 [Candidate]" term=2 +49:31.619 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:31.619 [DEBUG] TestAgent.server.raft: vote granted: from=e37c1e53-6c46-7162-1768-d26bbd0cea3d term=2 tally=1 +49:31.619 [INFO] TestAgent.server.raft: election won: tally=1 +49:31.619 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23595 [Leader]" +49:31.619 [INFO] TestAgent.server: cluster leadership acquired +49:31.619 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23595 +49:31.619 [INFO] TestAgent.server: New leader elected: payload=Node-e37c1e53-6c46-7162-1768-d26bbd0cea3d +49:31.620 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:31.621 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:31.621 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:31.621 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:31.621 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:31.621 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:31.621 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e37c1e53-6c46-7162-1768-d26bbd0cea3d +49:31.621 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e37c1e53-6c46-7162-1768-d26bbd0cea3d +49:31.622 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.811 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:31.812 [INFO] TestAgent: Synced node info +49:31.812 [DEBUG] TestAgent: Node info in sync +49:31.878 [WARN] TestAgent: Failed to restore check: check=service:redis error="ServiceID "nope" does not exist" +49:31.879 [DEBUG] TestAgent: restored health check from file: check=service:redis file=/tmp/consul-test/TestAgent_loadChecks_checkFails-TestAgent-agent231654690/checks/60a2ef12de014a05ecdc850d9aab46da +49:31.879 [INFO] TestAgent: Requesting shutdown +49:31.879 [INFO] TestAgent.server: shutting down server +49:31.879 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:31.879 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:31.879 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:31.879 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:31.879 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:31.879 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:31.879 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:31.879 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:31.879 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:31.879 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:31.879 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:31.879Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:31.879Z [INFO] agent.router.manager: shutting down +49:31.879 [INFO] TestAgent: consul server down +49:31.879 [INFO] TestAgent: shutdown complete +49:31.879 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23569 network=tcp +49:31.879 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23569 network=udp +49:31.880 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23591 network=tcp +49:32.381 [INFO] TestAgent: Waiting for endpoints to shut down +49:32.381 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_loadChecks_checkFails (0.83s) +=== CONT TestAgent_RemoveCheck +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=33)_compressed=true +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=33)_compressed=false +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=true#01 +2022-12-04T05:49:31.677Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.707Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:31.710 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9f23742d-c6ac-91a2-785e-1ac243083d16 Address:127.0.0.1:23665}]" +49:31.710 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23665 [Follower]" leader= +49:31.711 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9f23742d-c6ac-91a2-785e-1ac243083d16.dc1 127.0.0.1 +49:31.711 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9f23742d-c6ac-91a2-785e-1ac243083d16 127.0.0.1 +2022-12-04T05:49:31.711Z [INFO] agent.router: Initializing LAN area manager +49:31.711 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23653 network=udp +49:31.711 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9f23742d-c6ac-91a2-785e-1ac243083d16.dc1 area=wan +49:31.712 [INFO] TestAgent.server: Adding LAN server: server="Node-9f23742d-c6ac-91a2-785e-1ac243083d16 (Addr: tcp/127.0.0.1:23665) (DC: dc1)" +49:31.712 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23653 network=tcp +49:31.712 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23661 network=tcp +49:31.712 [INFO] TestAgent: started state syncer +49:31.712 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23624 network=tcp +49:31.761 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:31.761 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23665 [Candidate]" term=2 +49:31.762 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:31.762 [DEBUG] TestAgent.server.raft: vote granted: from=9f23742d-c6ac-91a2-785e-1ac243083d16 term=2 tally=1 +49:31.762 [INFO] TestAgent.server.raft: election won: tally=1 +49:31.762 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23665 [Leader]" +49:31.762 [INFO] TestAgent.server: cluster leadership acquired +49:31.762 [INFO] TestAgent.server: New leader elected: payload=Node-9f23742d-c6ac-91a2-785e-1ac243083d16 +49:31.762 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23665 +49:31.763 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:31.764 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:31.764 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:31.764 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:31.764 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:31.764 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:31.764 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9f23742d-c6ac-91a2-785e-1ac243083d16 +49:31.764 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9f23742d-c6ac-91a2-785e-1ac243083d16 +49:31.765 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.888 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:31.889 [INFO] TestAgent: Synced node info +49:31.889 [DEBUG] TestAgent: Node info in sync +49:31.937 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:31.937 [DEBUG] TestAgent: Node info in sync +49:32.119 [INFO] TestAgent.server.connect: CA provider config updated +49:32.120 [INFO] TestAgent: Requesting shutdown +49:32.120 [INFO] TestAgent.server: shutting down server +49:32.120 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:32.120 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:32.120 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:32.120 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:32.120 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:32.120 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:32.120 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:32.120 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:32.120 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:32.120 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:32.120 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:32.120Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:32.120Z [INFO] agent.router.manager: shutting down +49:32.120 [INFO] TestAgent: consul server down +49:32.120 [INFO] TestAgent: shutdown complete +49:32.120 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23653 network=tcp +49:32.120 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23653 network=udp +49:32.120 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23661 network=tcp +49:32.623 [INFO] TestAgent: Waiting for endpoints to shut down +49:32.623 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectCAConfig (4.97s) + --- PASS: TestConnectCAConfig/basic (0.83s) + --- PASS: TestConnectCAConfig/basic_with_IntermediateCertTTL (0.93s) + --- PASS: TestConnectCAConfig/force_without_cross_sign_CamelCase (0.68s) + --- PASS: TestConnectCAConfig/force_without_cross_sign_snake_case (0.72s) + --- PASS: TestConnectCAConfig/setting_state_fails (0.85s) + --- PASS: TestConnectCAConfig/updating_config_with_same_state (0.96s) +=== CONT TestAgent_AddCheck_Alias_userAndSetToken +2022-12-04T05:49:31.854Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.866Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:31.868 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:89507d04-d5b4-c3cf-71b7-feb4d16b13b0 Address:127.0.0.1:23637}]" +49:31.869 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23637 [Follower]" leader= +49:31.869 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-89507d04-d5b4-c3cf-71b7-feb4d16b13b0.dc1 127.0.0.1 +49:31.870 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-89507d04-d5b4-c3cf-71b7-feb4d16b13b0 127.0.0.1 +2022-12-04T05:49:31.870Z [INFO] agent.router: Initializing LAN area manager +49:31.870 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23667 network=udp +49:31.870 [INFO] TestAgent.server: Adding LAN server: server="Node-89507d04-d5b4-c3cf-71b7-feb4d16b13b0 (Addr: tcp/127.0.0.1:23637) (DC: dc1)" +49:31.870 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-89507d04-d5b4-c3cf-71b7-feb4d16b13b0.dc1 area=wan +49:31.870 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23667 network=tcp +49:31.871 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23633 network=tcp +49:31.871 [INFO] TestAgent: started state syncer +49:31.871 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23603 network=tcp +49:31.933 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:31.933 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23637 [Candidate]" term=2 +49:31.933 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:31.933 [DEBUG] TestAgent.server.raft: vote granted: from=89507d04-d5b4-c3cf-71b7-feb4d16b13b0 term=2 tally=1 +49:31.933 [INFO] TestAgent.server.raft: election won: tally=1 +49:31.933 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23637 [Leader]" +49:31.933 [INFO] TestAgent.server: cluster leadership acquired +49:31.933 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23637 +49:31.935 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:31.938 [INFO] TestAgent.server: New leader elected: payload=Node-89507d04-d5b4-c3cf-71b7-feb4d16b13b0 +49:31.939 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:31.939 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:31.939 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:31.939 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:31.939 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:31.939 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-89507d04-d5b4-c3cf-71b7-feb4d16b13b0 +49:31.939 [INFO] TestAgent.server: member joined, marking health alive: member=Node-89507d04-d5b4-c3cf-71b7-feb4d16b13b0 +49:31.940 [INFO] TestAgent.server: federation state anti-entropy synced +49:32.140 [INFO] TestAgent: Requesting shutdown +49:32.140 [INFO] TestAgent.server: shutting down server +49:32.140 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:32.140 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:32.140 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:32.140 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:32.140 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:32.140 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:32.140 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:32.140 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:32.141 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:32.141 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:32.141 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:32.141 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:49:32.141Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:32.141Z [INFO] agent.router.manager: shutting down +49:32.141 [INFO] TestAgent: consul server down +49:32.141 [INFO] TestAgent: shutdown complete +49:32.141 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23667 network=tcp +49:32.141 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23667 network=udp +49:32.141 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23633 network=tcp +49:32.641 [INFO] TestAgent: Waiting for endpoints to shut down +49:32.641 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_restoresSnapshot (0.80s) +=== CONT TestAgent_AddCheck_Alias_userToken +2022-12-04T05:49:31.738Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.755Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:31.757 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cc8ab0d1-5650-2f07-fb23-5a91ec988644 Address:127.0.0.1:23679}]" +49:31.757 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23679 [Follower]" leader= +49:31.757 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cc8ab0d1-5650-2f07-fb23-5a91ec988644.dc1 127.0.0.1 +49:31.758 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cc8ab0d1-5650-2f07-fb23-5a91ec988644 127.0.0.1 +2022-12-04T05:49:31.758Z [INFO] agent.router: Initializing LAN area manager +49:31.758 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23625 network=udp +49:31.758 [INFO] TestAgent.server: Adding LAN server: server="Node-cc8ab0d1-5650-2f07-fb23-5a91ec988644 (Addr: tcp/127.0.0.1:23679) (DC: dc1)" +49:31.758 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23625 network=tcp +49:31.758 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cc8ab0d1-5650-2f07-fb23-5a91ec988644.dc1 area=wan +49:31.759 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23675 network=tcp +49:31.759 [INFO] TestAgent: started state syncer +49:31.759 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23666 network=tcp +49:31.807 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:31.807 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23679 [Candidate]" term=2 +49:31.807 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:31.807 [DEBUG] TestAgent.server.raft: vote granted: from=cc8ab0d1-5650-2f07-fb23-5a91ec988644 term=2 tally=1 +49:31.807 [INFO] TestAgent.server.raft: election won: tally=1 +49:31.807 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23679 [Leader]" +49:31.807 [INFO] TestAgent.server: cluster leadership acquired +49:31.807 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23679 +49:31.808 [INFO] TestAgent.server: New leader elected: payload=Node-cc8ab0d1-5650-2f07-fb23-5a91ec988644 +49:31.808 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:31.809 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:31.809 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:31.809 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:31.809 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:31.809 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:31.809 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cc8ab0d1-5650-2f07-fb23-5a91ec988644 +49:31.809 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cc8ab0d1-5650-2f07-fb23-5a91ec988644 +49:31.810 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.979 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:31.979 [INFO] TestAgent: Synced node info +49:31.979 [DEBUG] TestAgent: Node info in sync +49:32.139 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:32.139 [DEBUG] TestAgent: Node info in sync +49:32.157 [DEBUG] TestAgent: Node info in sync +49:32.158 [INFO] TestAgent: Node entered maintenance mode +49:32.159 [INFO] TestAgent: Synced check: check=_node_maintenance +49:32.159 [DEBUG] TestAgent: removed check: check=_node_maintenance +49:32.159 [INFO] TestAgent: Node left maintenance mode +49:32.159 [INFO] TestAgent: Node entered maintenance mode +49:32.159 [INFO] TestAgent: Requesting shutdown +49:32.159 [INFO] TestAgent.server: shutting down server +49:32.159 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:32.159 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:32.159 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:32.159 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:32.159 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:32.160 [DEBUG] TestAgent: Node info in sync +49:32.160 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:32.160 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:32.160 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:32.160 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:32.160 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:32.160 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:32.160Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:32.160Z [INFO] agent.router.manager: shutting down +49:32.161 [INFO] TestAgent: consul server down +49:32.161 [INFO] TestAgent: shutdown complete +49:32.161 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23625 network=tcp +49:32.161 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23625 network=udp +49:32.161 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23675 network=tcp +49:32.161 [INFO] TestAgent: Synced check: check=_node_maintenance +49:32.661 [INFO] TestAgent: Waiting for endpoints to shut down +49:32.661 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_NodeMaintenanceMode (0.95s) +=== CONT TestAgent_AddCheck_Alias_setToken +2022-12-04T05:49:31.961Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.972Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:31.974 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e42e7794-07b7-620e-f13f-b38f6ce8e8e6 Address:127.0.0.1:23609}]" +49:31.974 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23609 [Follower]" leader= +49:31.975 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e42e7794-07b7-620e-f13f-b38f6ce8e8e6.dc1 127.0.0.1 +49:31.975 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e42e7794-07b7-620e-f13f-b38f6ce8e8e6 127.0.0.1 +2022-12-04T05:49:31.975Z [INFO] agent.router: Initializing LAN area manager +49:31.975 [INFO] TestAgent.server: Adding LAN server: server="Node-e42e7794-07b7-620e-f13f-b38f6ce8e8e6 (Addr: tcp/127.0.0.1:23609) (DC: dc1)" +49:31.975 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e42e7794-07b7-620e-f13f-b38f6ce8e8e6.dc1 area=wan +49:31.975 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23604 network=udp +49:31.975 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23604 network=tcp +49:31.976 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23605 network=tcp +49:31.976 [INFO] TestAgent: started state syncer +49:31.976 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23645 network=tcp +49:32.020 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:32.020 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23609 [Candidate]" term=2 +49:32.020 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:32.020 [DEBUG] TestAgent.server.raft: vote granted: from=e42e7794-07b7-620e-f13f-b38f6ce8e8e6 term=2 tally=1 +49:32.020 [INFO] TestAgent.server.raft: election won: tally=1 +49:32.020 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23609 [Leader]" +49:32.020 [INFO] TestAgent.server: cluster leadership acquired +49:32.021 [INFO] TestAgent.server: New leader elected: payload=Node-e42e7794-07b7-620e-f13f-b38f6ce8e8e6 +49:32.021 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23609 +49:32.022 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:32.023 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:32.023 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:32.023 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:32.023 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:32.023 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:32.023 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e42e7794-07b7-620e-f13f-b38f6ce8e8e6 +49:32.023 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e42e7794-07b7-620e-f13f-b38f6ce8e8e6 +49:32.024 [INFO] TestAgent.server: federation state anti-entropy synced +49:32.161 [INFO] TestAgent: Service entered maintenance mode: service=redis +49:32.162 [DEBUG] TestAgent: removed check: check=_service_maintenance:redis +49:32.162 [INFO] TestAgent: Service left maintenance mode: service=redis +49:32.162 [INFO] TestAgent: Service entered maintenance mode: service=redis +49:32.162 [INFO] TestAgent: Requesting shutdown +49:32.162 [INFO] TestAgent.server: shutting down server +49:32.162 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:32.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:32.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:32.162 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:32.162 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:32.162 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:32.162 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:32.162 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:32.162 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:32.162 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:32.162 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:32.162 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:32.162Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:32.162Z [INFO] agent.router.manager: shutting down +49:32.162 [INFO] TestAgent: consul server down +49:32.162 [INFO] TestAgent: shutdown complete +49:32.162 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23604 network=tcp +49:32.162 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23604 network=udp +49:32.162 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23605 network=tcp +49:32.667 [INFO] TestAgent: Waiting for endpoints to shut down +49:32.667 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Service_MaintenanceMode (0.72s) +=== CONT TestAgent_AddCheck_Alias +2022-12-04T05:49:31.981Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.993Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:31.994 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:39b9ced4-89d4-11eb-c6d7-5d4ddf3973a8 Address:127.0.0.1:23728}]" +49:31.995 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-39b9ced4-89d4-11eb-c6d7-5d4ddf3973a8.dc1 127.0.0.1 +49:31.995 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-39b9ced4-89d4-11eb-c6d7-5d4ddf3973a8 127.0.0.1 +2022-12-04T05:49:31.995Z [INFO] agent.router: Initializing LAN area manager +49:31.995 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23728 [Follower]" leader= +49:31.996 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23674 network=udp +49:31.996 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-39b9ced4-89d4-11eb-c6d7-5d4ddf3973a8.dc1 area=wan +49:31.996 [INFO] TestAgent.server: Adding LAN server: server="Node-39b9ced4-89d4-11eb-c6d7-5d4ddf3973a8 (Addr: tcp/127.0.0.1:23728) (DC: dc1)" +49:31.996 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23674 network=tcp +49:31.997 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23724 network=tcp +49:31.997 [INFO] TestAgent: started state syncer +49:31.997 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23694 network=tcp +49:32.052 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:32.052 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23728 [Candidate]" term=2 +49:32.052 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:32.052 [DEBUG] TestAgent.server.raft: vote granted: from=39b9ced4-89d4-11eb-c6d7-5d4ddf3973a8 term=2 tally=1 +49:32.052 [INFO] TestAgent.server.raft: election won: tally=1 +49:32.052 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23728 [Leader]" +49:32.052 [INFO] TestAgent.server: cluster leadership acquired +49:32.052 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23728 +49:32.053 [INFO] TestAgent.server: New leader elected: payload=Node-39b9ced4-89d4-11eb-c6d7-5d4ddf3973a8 +49:32.053 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:32.069 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:32.071 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:32.071 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:32.071 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:32.071 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:32.071 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-39b9ced4-89d4-11eb-c6d7-5d4ddf3973a8 +49:32.071 [INFO] TestAgent.server: member joined, marking health alive: member=Node-39b9ced4-89d4-11eb-c6d7-5d4ddf3973a8 +49:32.076 [INFO] TestAgent.server: federation state anti-entropy synced +49:32.119 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:32.120 [INFO] TestAgent: Synced node info +49:32.120 [INFO] TestAgent: Synced check: check=rabbitmq +49:32.213 [INFO] TestAgent: Requesting shutdown +49:32.213 [INFO] TestAgent.server: shutting down server +49:32.213 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:32.213 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:32.213 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:32.213 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:32.213 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:32.214 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:32.214 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:32.214 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:32.214 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:32.214 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:32.214 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:32.214Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:32.214Z [INFO] agent.router.manager: shutting down +49:32.214 [INFO] TestAgent: consul server down +49:32.214 [INFO] TestAgent: shutdown complete +49:32.214 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23674 network=tcp +49:32.214 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23674 network=udp +49:32.214 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23724 network=tcp +49:32.716 [INFO] TestAgent: Waiting for endpoints to shut down +49:32.716 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_loadChecks_token (0.77s) +=== CONT TestAgent_AddCheck_GRPC +2022-12-04T05:49:31.976Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.988Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:31.990 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b286d1b2-a993-8b71-d831-f893345f4682 Address:127.0.0.1:23651}]" +49:31.990 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23651 [Follower]" leader= +49:31.990 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b286d1b2-a993-8b71-d831-f893345f4682.dc1 127.0.0.1 +49:31.991 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b286d1b2-a993-8b71-d831-f893345f4682 127.0.0.1 +2022-12-04T05:49:31.991Z [INFO] agent.router: Initializing LAN area manager +49:31.991 [INFO] TestAgent.server: Adding LAN server: server="Node-b286d1b2-a993-8b71-d831-f893345f4682 (Addr: tcp/127.0.0.1:23651) (DC: dc1)" +49:31.991 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b286d1b2-a993-8b71-d831-f893345f4682.dc1 area=wan +49:31.991 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23646 network=udp +49:31.991 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23646 network=tcp +49:31.992 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23647 network=tcp +49:31.992 [INFO] TestAgent: started state syncer +49:31.993 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23673 network=tcp +49:32.045 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:32.045 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23651 [Candidate]" term=2 +49:32.045 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:32.045 [DEBUG] TestAgent.server.raft: vote granted: from=b286d1b2-a993-8b71-d831-f893345f4682 term=2 tally=1 +49:32.045 [INFO] TestAgent.server.raft: election won: tally=1 +49:32.045 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23651 [Leader]" +49:32.045 [INFO] TestAgent.server: cluster leadership acquired +49:32.045 [INFO] TestAgent.server: New leader elected: payload=Node-b286d1b2-a993-8b71-d831-f893345f4682 +49:32.045 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23651 +49:32.046 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:32.047 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:32.047 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:32.047 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:32.047 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:32.047 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:32.048 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b286d1b2-a993-8b71-d831-f893345f4682 +49:32.048 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b286d1b2-a993-8b71-d831-f893345f4682 +49:32.049 [INFO] TestAgent.server: federation state anti-entropy synced +49:32.152 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:32.153 [INFO] TestAgent: Synced node info +49:32.153 [DEBUG] TestAgent: Node info in sync +49:32.299 [DEBUG] TestAgent: removed check: check=service:redis +49:32.299 [INFO] TestAgent: Requesting shutdown +49:32.299 [INFO] TestAgent.server: shutting down server +49:32.299 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:32.299 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:32.299 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:32.299 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:32.299 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:32.299 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:32.300 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:32.300 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:32.300 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:32.300 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:32.300 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:32.300Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:32.300Z [INFO] agent.router.manager: shutting down +49:32.300 [INFO] TestAgent: consul server down +49:32.300 [INFO] TestAgent: shutdown complete +49:32.300 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23646 network=tcp +49:32.300 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23646 network=udp +49:32.300 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23647 network=tcp +49:32.800 [INFO] TestAgent: Waiting for endpoints to shut down +49:32.801 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_unloadChecks (0.84s) +=== CONT TestAgent_AddCheck_ExecRemoteDisable +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:32.094Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:32.115Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:32.117 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0532e1dc-de57-2812-a1b2-226a4236107a Address:127.0.0.1:23707}]" +49:32.118 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23707 [Follower]" leader= +49:32.118 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0532e1dc-de57-2812-a1b2-226a4236107a.dc1 127.0.0.1 +49:32.119 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0532e1dc-de57-2812-a1b2-226a4236107a 127.0.0.1 +2022-12-04T05:49:32.119Z [INFO] agent.router: Initializing LAN area manager +49:32.119 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23827 network=udp +49:32.119 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0532e1dc-de57-2812-a1b2-226a4236107a.dc1 area=wan +49:32.119 [INFO] TestAgent.server: Adding LAN server: server="Node-0532e1dc-de57-2812-a1b2-226a4236107a (Addr: tcp/127.0.0.1:23707) (DC: dc1)" +49:32.119 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23827 network=tcp +49:32.120 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23703 network=tcp +49:32.120 [INFO] TestAgent: started state syncer +49:32.120 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23715 network=tcp +49:32.178 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:32.178 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23707 [Candidate]" term=2 +49:32.179 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:32.179 [DEBUG] TestAgent.server.raft: vote granted: from=0532e1dc-de57-2812-a1b2-226a4236107a term=2 tally=1 +49:32.179 [INFO] TestAgent.server.raft: election won: tally=1 +49:32.179 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23707 [Leader]" +49:32.179 [INFO] TestAgent.server: cluster leadership acquired +49:32.179 [INFO] TestAgent.server: New leader elected: payload=Node-0532e1dc-de57-2812-a1b2-226a4236107a +49:32.179 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23707 +49:32.180 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:32.182 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:32.182 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:32.182 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:32.182 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:32.182 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:32.182 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0532e1dc-de57-2812-a1b2-226a4236107a +49:32.182 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0532e1dc-de57-2812-a1b2-226a4236107a +49:32.183 [INFO] TestAgent.server: federation state anti-entropy synced +49:32.343 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:32.344 [INFO] TestAgent: Synced node info +49:32.433 [DEBUG] TestAgent: removed check: check=mem +49:32.433 [DEBUG] TestAgent: removed check: check=mem +49:32.433 [INFO] TestAgent: Requesting shutdown +49:32.433 [INFO] TestAgent.server: shutting down server +49:32.433 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:32.433 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:32.433 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:32.433 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:32.433 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:32.433 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:32.433 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:32.433 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:32.433 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:32.433 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:32.433 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:32.434Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:32.434Z [INFO] agent.router.manager: shutting down +49:32.434 [INFO] TestAgent: consul server down +49:32.434 [INFO] TestAgent: shutdown complete +49:32.434 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23827 network=tcp +49:32.434 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23827 network=udp +49:32.434 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23703 network=tcp +49:32.935 [INFO] TestAgent: Waiting for endpoints to shut down +49:32.935 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_PurgeCheck (0.89s) +=== CONT TestAgent_AddCheck_ExecDisable +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=false#01 +2022-12-04T05:49:32.186Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:32.231Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:32.233 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3a501bbc-3de0-f95d-8635-3518cb1d06db Address:127.0.0.1:23776}]" +49:32.234 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3a501bbc-3de0-f95d-8635-3518cb1d06db.dc1 127.0.0.1 +49:32.234 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23776 [Follower]" leader= +49:32.235 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3a501bbc-3de0-f95d-8635-3518cb1d06db 127.0.0.1 +2022-12-04T05:49:32.235Z [INFO] agent.router: Initializing LAN area manager +49:32.235 [INFO] TestAgent.server: Adding LAN server: server="Node-3a501bbc-3de0-f95d-8635-3518cb1d06db (Addr: tcp/127.0.0.1:23776) (DC: dc1)" +49:32.235 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3a501bbc-3de0-f95d-8635-3518cb1d06db.dc1 area=wan +49:32.235 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23751 network=udp +49:32.235 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23751 network=tcp +49:32.236 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23772 network=tcp +49:32.236 [INFO] TestAgent: started state syncer +49:32.236 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23729 network=tcp +49:32.299 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:32.299 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23776 [Candidate]" term=2 +49:32.299 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:32.299 [DEBUG] TestAgent.server.raft: vote granted: from=3a501bbc-3de0-f95d-8635-3518cb1d06db term=2 tally=1 +49:32.299 [INFO] TestAgent.server.raft: election won: tally=1 +49:32.299 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23776 [Leader]" +49:32.299 [INFO] TestAgent.server: cluster leadership acquired +49:32.300 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23776 +49:32.300 [INFO] TestAgent.server: New leader elected: payload=Node-3a501bbc-3de0-f95d-8635-3518cb1d06db +49:32.301 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:32.303 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:32.303 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:32.303 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:32.303 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:32.303 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:32.303 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3a501bbc-3de0-f95d-8635-3518cb1d06db +49:32.303 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3a501bbc-3de0-f95d-8635-3518cb1d06db +49:32.304 [INFO] TestAgent.server: federation state anti-entropy synced +49:32.490 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:32.490 [INFO] TestAgent: Synced node info +49:32.516 [INFO] TestAgent: Requesting shutdown +49:32.516 [INFO] TestAgent.server: shutting down server +49:32.516 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:32.516 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:32.516 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:32.516 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:32.516 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:32.516 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:32.516 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:32.516 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:32.516 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:32.516 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:32.516 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:32.516Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:32.516Z [INFO] agent.router.manager: shutting down +49:32.517 [INFO] TestAgent: consul server down +49:32.517 [INFO] TestAgent: shutdown complete +49:32.517 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23751 network=tcp +49:32.517 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23751 network=udp +49:32.517 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23772 network=tcp +49:33.017 [INFO] TestAgent: Waiting for endpoints to shut down +49:33.017 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:31.298Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:31.316Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:31.319 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dc16a475-2703-7bf1-15c1-894cb60ea674 Address:127.0.0.1:23616}]" +49:31.320 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23616 [Follower]" leader= +49:31.320 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dc16a475-2703-7bf1-15c1-894cb60ea674.dc1 127.0.0.1 +49:31.321 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dc16a475-2703-7bf1-15c1-894cb60ea674 127.0.0.1 +2022-12-04T05:49:31.321Z [INFO] agent.router: Initializing LAN area manager +49:31.322 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23632 network=udp +49:31.322 [INFO] TestAgent.server: Adding LAN server: server="Node-dc16a475-2703-7bf1-15c1-894cb60ea674 (Addr: tcp/127.0.0.1:23616) (DC: dc1)" +49:31.322 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dc16a475-2703-7bf1-15c1-894cb60ea674.dc1 area=wan +49:31.324 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23632 network=tcp +49:31.325 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23612 network=tcp +49:31.325 [INFO] TestAgent: started state syncer +49:31.325 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23589 network=tcp +49:31.373 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:31.373 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23616 [Candidate]" term=2 +49:31.374 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:31.374 [DEBUG] TestAgent.server.raft: vote granted: from=dc16a475-2703-7bf1-15c1-894cb60ea674 term=2 tally=1 +49:31.374 [INFO] TestAgent.server.raft: election won: tally=1 +49:31.374 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23616 [Leader]" +49:31.374 [INFO] TestAgent.server: cluster leadership acquired +49:31.374 [INFO] TestAgent.server: New leader elected: payload=Node-dc16a475-2703-7bf1-15c1-894cb60ea674 +49:31.374 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23616 +49:31.375 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:31.378 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:31.378 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:31.378 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:31.378 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:31.378 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:31.378 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dc16a475-2703-7bf1-15c1-894cb60ea674 +49:31.378 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dc16a475-2703-7bf1-15c1-894cb60ea674 +49:31.379 [INFO] TestAgent.server: federation state anti-entropy synced +49:31.671 [INFO] TestAgent: Requesting shutdown +49:31.672 [INFO] TestAgent.server: shutting down server +49:31.672 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:31.672 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:31.672 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:31.672 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:31.672 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:31.672 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:31.672 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:31.672 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:31.672 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:31.672 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:31.672 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:31.672 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:31.672Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:31.672Z [INFO] agent.router.manager: shutting down +49:31.672 [INFO] TestAgent: consul server down +49:31.672 [INFO] TestAgent: shutdown complete +49:31.672 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23632 network=tcp +49:31.672 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23632 network=udp +49:31.672 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23612 network=tcp +49:32.172 [INFO] TestAgent: Waiting for endpoints to shut down +49:32.172 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_GetCoordinate (1.73s) +=== CONT TestAgent_AddCheck_RestoreState +2022-12-04T05:49:32.524Z [WARN] agent: Node name "Node 27a3aab3-b070-d1a0-2706-1d6920f57b38" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2022-12-04T05:49:32.538Z [WARN] agent.auto_config: Node name "Node 27a3aab3-b070-d1a0-2706-1d6920f57b38" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +49:32.539 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node 27a3aab3-b070-d1a0-2706-1d6920f57b38 127.0.0.1 +2022-12-04T05:49:32.539Z [INFO] agent.router: Initializing LAN area manager +49:32.539 [DEBUG] Agent2: restored service definition from file: service=svc-sidecar-proxy file=/tmp/consul-test/TestAgent_DeregisterPersistedSidecarAfterRestart-TestAgent-agent426068234/services/18ed17ea501b78b6980ecfa920fcf57a +49:32.539 [DEBUG] Agent2: restored service definition from file: service=svc file=/tmp/consul-test/TestAgent_DeregisterPersistedSidecarAfterRestart-TestAgent-agent426068234/services/961e38b5146c0b07d078f53dec9b699a +49:32.539 [INFO] Agent2: Started DNS server: address=127.0.0.1:23737 network=udp +49:32.540 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node +49:32.540 [INFO] Agent2: Started DNS server: address=127.0.0.1:23737 network=tcp +2022-12-04T05:49:32.540Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.540Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.540Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.540Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.540Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.540Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.540Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.540Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.540Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.540Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.540Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.540Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.540Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.541Z [WARN] agent.router.manager: No servers available +49:32.541 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +49:32.541 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +49:32.541 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:32.541Z [WARN] agent.router.manager: No servers available +49:32.541 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:32.540Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.541Z [WARN] agent.router.manager: No servers available +49:32.541 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +49:32.541 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +49:32.541 [INFO] Agent2: Started HTTP server: address=127.0.0.1:23689 network=tcp +2022-12-04T05:49:32.541Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.541Z [WARN] agent.router.manager: No servers available +49:32.541 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:32.541Z [WARN] agent.router.manager: No servers available +49:32.541 [INFO] Agent2: started state syncer +49:32.541 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:32.541Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.541Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.541Z [WARN] agent.router.manager: No servers available +49:32.541 [INFO] Agent2: Started gRPC server: address=127.0.0.1:23484 network=tcp +2022-12-04T05:49:32.541Z [WARN] agent.router.manager: No servers available +49:32.541 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=intentions error="error filling agent cache: No known Consul servers" +49:32.541 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=roots error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:32.541Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.541Z [WARN] agent.router.manager: No servers available +2022-12-04T05:49:32.541Z [WARN] agent.router.manager: No servers available +49:32.541 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:32.541Z [WARN] agent.router.manager: No servers available +49:32.541 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=leaf error="error filling agent cache: No known Consul servers" +2022-12-04T05:49:32.541Z [WARN] agent.router.manager: No servers available +49:32.541 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" +49:32.541 [DEBUG] Agent2: removed service: service=svc +49:32.541 [DEBUG] Agent2: removed service: service=svc-sidecar-proxy +49:32.541 [INFO] Agent2: Requesting shutdown +49:32.541 [INFO] Agent2.client: shutting down client +49:32.541 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave +49:32.541 [ERROR] Agent2.proxycfg: watch error: service_id=svc-sidecar-proxy id=service-http-checks:svc error="error filling agent cache: Internal cache failure: service 'svc' not in agent state" +49:32.541 [INFO] Agent2: consul client down +49:32.541 [INFO] Agent2: shutdown complete +49:32.541 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:23737 network=tcp +49:32.542 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:23737 network=udp +49:32.542 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:23689 network=tcp +49:33.042 [INFO] Agent2: Waiting for endpoints to shut down +49:33.042 [INFO] Agent2: Endpoints down +2022-12-04T05:49:31.995Z [WARN] agent: Node name "Node 27a3aab3-b070-d1a0-2706-1d6920f57b38" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2022-12-04T05:49:32.008Z [WARN] agent.auto_config: Node name "Node 27a3aab3-b070-d1a0-2706-1d6920f57b38" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +49:32.008 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node 27a3aab3-b070-d1a0-2706-1d6920f57b38 127.0.0.1 +2022-12-04T05:49:32.008Z [INFO] agent.router: Initializing LAN area manager +49:32.009 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23695 network=udp +49:32.009 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23695 network=tcp +49:32.009 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23486 network=tcp +49:32.010 [INFO] TestAgent: started state syncer +49:32.010 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23701 network=tcp +2022-12-04T05:49:32.010Z [WARN] agent.router.manager: No servers available +49:32.010 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +49:32.010 [INFO] TestAgent: Requesting shutdown +49:32.010 [INFO] TestAgent.client: shutting down client +49:32.010 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +49:32.011 [INFO] TestAgent: consul client down +49:32.011 [INFO] TestAgent: shutdown complete +49:32.011 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23695 network=tcp +49:32.011 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23695 network=udp +49:32.011 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23486 network=tcp +49:32.511 [INFO] TestAgent: Waiting for endpoints to shut down +49:32.511 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_DeregisterPersistedSidecarAfterRestart (1.06s) +=== CONT TestAgent_AddCheck_MissingService +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:32.234Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:32.247Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:32.250 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4657d714-ece1-9660-d8c7-aae5f5266c38 Address:127.0.0.1:23735}]" +49:32.250 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4657d714-ece1-9660-d8c7-aae5f5266c38.dc1 127.0.0.1 +49:32.250 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23735 [Follower]" leader= +49:32.251 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4657d714-ece1-9660-d8c7-aae5f5266c38 127.0.0.1 +2022-12-04T05:49:32.251Z [INFO] agent.router: Initializing LAN area manager +49:32.251 [INFO] TestAgent.server: Adding LAN server: server="Node-4657d714-ece1-9660-d8c7-aae5f5266c38 (Addr: tcp/127.0.0.1:23735) (DC: dc1)" +49:32.251 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4657d714-ece1-9660-d8c7-aae5f5266c38.dc1 area=wan +49:32.251 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23730 network=udp +49:32.251 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23730 network=tcp +49:32.252 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23731 network=tcp +49:32.252 [INFO] TestAgent: started state syncer +49:32.252 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23722 network=tcp +49:32.287 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:32.287 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23735 [Candidate]" term=2 +49:32.288 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:32.288 [DEBUG] TestAgent.server.raft: vote granted: from=4657d714-ece1-9660-d8c7-aae5f5266c38 term=2 tally=1 +49:32.288 [INFO] TestAgent.server.raft: election won: tally=1 +49:32.288 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23735 [Leader]" +49:32.288 [INFO] TestAgent.server: cluster leadership acquired +49:32.288 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23735 +49:32.288 [INFO] TestAgent.server: New leader elected: payload=Node-4657d714-ece1-9660-d8c7-aae5f5266c38 +49:32.289 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:32.290 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:32.290 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:32.290 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:32.290 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:32.290 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:32.290 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4657d714-ece1-9660-d8c7-aae5f5266c38 +49:32.290 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4657d714-ece1-9660-d8c7-aae5f5266c38 +49:32.291 [INFO] TestAgent.server: federation state anti-entropy synced +49:32.496 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:32.497 [INFO] TestAgent: Synced node info +49:32.570 [DEBUG] TestAgent: Check status updated: check=mem status=passing +49:32.570 [DEBUG] TestAgent: Check status updated: check=mem status=critical +49:32.570 [INFO] TestAgent: Requesting shutdown +49:32.570 [INFO] TestAgent.server: shutting down server +49:32.571 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:32.571 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:32.571 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:32.571 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:32.571 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:32.571 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:32.571 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:32.571 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:32.571 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:32.571 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:32.571 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:32.571Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:32.571Z [INFO] agent.router.manager: shutting down +49:32.571 [INFO] TestAgent: consul server down +49:32.571 [INFO] TestAgent: shutdown complete +49:32.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23730 network=tcp +49:32.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23730 network=udp +49:32.571 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23731 network=tcp +49:33.072 [INFO] TestAgent: Waiting for endpoints to shut down +49:33.072 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_updateTTLCheck (0.87s) +=== CONT TestAgent_AddCheck_MinInterval +2022-12-04T05:49:32.610Z [WARN] agent: Node name "Node 71353d29-f836-3d9f-c877-673a1917d5b9" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2022-12-04T05:49:32.610Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2022-12-04T05:49:32.641Z [WARN] agent.auto_config: Node name "Node 71353d29-f836-3d9f-c877-673a1917d5b9" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2022-12-04T05:49:32.641Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +49:32.642 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node 71353d29-f836-3d9f-c877-673a1917d5b9 127.0.0.1 +2022-12-04T05:49:32.642Z [INFO] agent.router: Initializing LAN area manager +49:32.642 [DEBUG] Agent2: check exists, not restoring from file: check=mem file=/tmp/consul-test/TestAgent_PurgeCheckOnDuplicate-TestAgent-agent730319585/checks/afc4fc7e48a0710a1dc94ef3e8bc5764 +49:32.642 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node +49:32.642 [INFO] Agent2: Started DNS server: address=127.0.0.1:23485 network=udp +49:32.642 [INFO] Agent2: Started DNS server: address=127.0.0.1:23485 network=tcp +49:32.643 [INFO] Agent2: Started HTTP server: address=127.0.0.1:23668 network=tcp +49:32.643 [INFO] Agent2: started state syncer +49:32.644 [INFO] Agent2: Requesting shutdown +49:32.644 [INFO] Agent2.client: shutting down client +49:32.644 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave +49:32.644 [INFO] Agent2: Started gRPC server: address=127.0.0.1:23708 network=tcp +49:32.644 [ERROR] Agent2: gRPC server failed: error="grpc: the server has been stopped" +2022-12-04T05:49:32.644Z [WARN] agent.router.manager: No servers available +49:32.644 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" +49:32.644 [INFO] Agent2: consul client down +49:32.644 [INFO] Agent2: shutdown complete +49:32.644 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:23485 network=tcp +49:32.644 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:23485 network=udp +49:32.644 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:23668 network=tcp +49:33.147 [INFO] Agent2: Waiting for endpoints to shut down +49:33.147 [INFO] Agent2: Endpoints down +2022-12-04T05:49:32.045Z [WARN] agent: Node name "Node 71353d29-f836-3d9f-c877-673a1917d5b9" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2022-12-04T05:49:32.045Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2022-12-04T05:49:32.086Z [WARN] agent.auto_config: Node name "Node 71353d29-f836-3d9f-c877-673a1917d5b9" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2022-12-04T05:49:32.086Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +49:32.087 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node 71353d29-f836-3d9f-c877-673a1917d5b9 127.0.0.1 +2022-12-04T05:49:32.087Z [INFO] agent.router: Initializing LAN area manager +49:32.087 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23702 network=udp +49:32.087 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23702 network=tcp +49:32.089 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23717 network=tcp +49:32.089 [INFO] TestAgent: started state syncer +49:32.090 [INFO] TestAgent: Requesting shutdown +49:32.090 [INFO] TestAgent.client: shutting down client +49:32.090 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +49:32.090 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23826 network=tcp +49:32.090 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" +2022-12-04T05:49:32.090Z [WARN] agent.router.manager: No servers available +49:32.090 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +49:32.090 [INFO] TestAgent: consul client down +49:32.090 [INFO] TestAgent: shutdown complete +49:32.090 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23702 network=tcp +49:32.090 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23702 network=udp +49:32.090 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23717 network=tcp +49:32.591 [INFO] TestAgent: Waiting for endpoints to shut down +49:32.591 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_PurgeCheckOnDuplicate (1.12s) +=== CONT TestAgent_AddCheck_StartPassing +2022-12-04T05:49:32.652Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2022-12-04T05:49:32.666Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +49:32.666 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node-b5b19db2-3c65-d7eb-994d-07a297a3cced 127.0.0.1 +2022-12-04T05:49:32.667Z [INFO] agent.router: Initializing LAN area manager +49:32.667 [INFO] Agent2.client.serf.lan: serf: Attempting re-join to previously known node: Node-8c5b832c-43a6-92aa-6a10-00b7dd3cf877: 127.0.0.1:23761 +49:32.667 [DEBUG] Agent2: restored health check from file: check=mem file=/tmp/consul-test/TestAgent_PersistCheck-TestAgent-agent865838011/checks/afc4fc7e48a0710a1dc94ef3e8bc5764 +49:32.667 [INFO] Agent2: Started DNS server: address=127.0.0.1:23688 network=udp +49:32.667 [DEBUG] Agent2.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:23761: connect: connection refused +49:32.667 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node +49:32.667 [INFO] Agent2: Started DNS server: address=127.0.0.1:23688 network=tcp +49:32.668 [INFO] Agent2: Started HTTP server: address=127.0.0.1:23793 network=tcp +49:32.668 [INFO] Agent2: started state syncer +49:32.668 [INFO] Agent2: Started gRPC server: address=127.0.0.1:23791 network=tcp +49:32.669 [INFO] Agent2: Requesting shutdown +2022-12-04T05:49:32.669Z [WARN] agent.router.manager: No servers available +49:32.669 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" +49:32.669 [INFO] Agent2.client: shutting down client +49:32.669 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave +49:32.669 [INFO] Agent2: consul client down +49:32.669 [INFO] Agent2: shutdown complete +49:32.669 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:23688 network=tcp +49:32.669 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:23688 network=udp +49:32.669 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:23793 network=tcp +49:33.172 [INFO] Agent2: Waiting for endpoints to shut down +49:33.172 [INFO] Agent2: Endpoints down +2022-12-04T05:49:32.117Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2022-12-04T05:49:32.131Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +49:32.132 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-8c5b832c-43a6-92aa-6a10-00b7dd3cf877 127.0.0.1 +2022-12-04T05:49:32.132Z [INFO] agent.router: Initializing LAN area manager +49:32.132 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23716 network=udp +49:32.132 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23716 network=tcp +49:32.133 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23759 network=tcp +49:32.133 [INFO] TestAgent: started state syncer +49:32.133 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23750 network=tcp +2022-12-04T05:49:32.134Z [WARN] agent.router.manager: No servers available +49:32.134 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +49:32.134 [INFO] TestAgent: Requesting shutdown +49:32.134 [INFO] TestAgent.client: shutting down client +49:32.134 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +49:32.134 [INFO] TestAgent: consul client down +49:32.134 [INFO] TestAgent: shutdown complete +49:32.134 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23716 network=tcp +49:32.134 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23716 network=udp +49:32.135 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23759 network=tcp +49:32.635 [INFO] TestAgent: Waiting for endpoints to shut down +49:32.635 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_PersistCheck (1.08s) +=== CONT TestAgent_AddCheck +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=true#01 +2022-12-04T05:49:32.680Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:32.695Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:32.697 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:98b25caa-8366-e2f7-9ae9-1ef213eeb95c Address:127.0.0.1:23476}]" +49:32.697 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-98b25caa-8366-e2f7-9ae9-1ef213eeb95c.dc1 127.0.0.1 +49:32.698 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23476 [Follower]" leader= +49:32.698 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-98b25caa-8366-e2f7-9ae9-1ef213eeb95c 127.0.0.1 +2022-12-04T05:49:32.698Z [INFO] agent.router: Initializing LAN area manager +49:32.698 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23478 network=udp +49:32.698 [INFO] TestAgent.server: Adding LAN server: server="Node-98b25caa-8366-e2f7-9ae9-1ef213eeb95c (Addr: tcp/127.0.0.1:23476) (DC: dc1)" +49:32.699 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-98b25caa-8366-e2f7-9ae9-1ef213eeb95c.dc1 area=wan +49:32.699 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23478 network=tcp +49:32.699 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23472 network=tcp +49:32.699 [INFO] TestAgent: started state syncer +49:32.699 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23805 network=tcp +49:32.745 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:32.745 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23476 [Candidate]" term=2 +49:32.745 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:32.745 [DEBUG] TestAgent.server.raft: vote granted: from=98b25caa-8366-e2f7-9ae9-1ef213eeb95c term=2 tally=1 +49:32.745 [INFO] TestAgent.server.raft: election won: tally=1 +49:32.745 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23476 [Leader]" +49:32.746 [INFO] TestAgent.server: cluster leadership acquired +49:32.746 [INFO] TestAgent.server: New leader elected: payload=Node-98b25caa-8366-e2f7-9ae9-1ef213eeb95c +49:32.746 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23476 +49:32.747 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:32.750 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:32.750 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:32.750 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:32.750 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:32.750 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:32.750 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-98b25caa-8366-e2f7-9ae9-1ef213eeb95c +49:32.750 [INFO] TestAgent.server: member joined, marking health alive: member=Node-98b25caa-8366-e2f7-9ae9-1ef213eeb95c +49:32.751 [INFO] TestAgent.server: federation state anti-entropy synced +49:32.814 [INFO] TestAgent: Requesting shutdown +49:32.814 [INFO] TestAgent.server: shutting down server +49:32.814 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:32.814 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:32.814 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:32.814 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:32.814 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:32.815 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:32.815 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:32.815 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:32.815 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:32.815 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:32.815 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:32.815 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:32.815Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:32.815Z [INFO] agent.router.manager: shutting down +49:32.815 [INFO] TestAgent: consul server down +49:32.815 [INFO] TestAgent: shutdown complete +49:32.815 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23478 network=tcp +49:32.815 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23478 network=udp +49:32.815 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23472 network=tcp +49:33.316 [INFO] TestAgent: Waiting for endpoints to shut down +49:33.316 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_Alias (0.65s) +=== CONT TestAgent_IndexChurn +=== RUN TestAgent_IndexChurn/no_tags +2022-12-04T05:49:32.674Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:32.686Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:32.688 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b6372585-a13a-4a4a-d8b6-65c173938220 Address:127.0.0.1:23742}]" +49:32.689 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23742 [Follower]" leader= +49:32.689 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b6372585-a13a-4a4a-d8b6-65c173938220.dc1 127.0.0.1 +49:32.690 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b6372585-a13a-4a4a-d8b6-65c173938220 127.0.0.1 +2022-12-04T05:49:32.690Z [INFO] agent.router: Initializing LAN area manager +49:32.690 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b6372585-a13a-4a4a-d8b6-65c173938220.dc1 area=wan +49:32.690 [INFO] TestAgent.server: Adding LAN server: server="Node-b6372585-a13a-4a4a-d8b6-65c173938220 (Addr: tcp/127.0.0.1:23742) (DC: dc1)" +49:32.690 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23771 network=udp +49:32.690 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23771 network=tcp +49:32.691 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23738 network=tcp +49:32.691 [INFO] TestAgent: started state syncer +49:32.691 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23477 network=tcp +49:32.734 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:32.734 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23742 [Candidate]" term=2 +49:32.734 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:32.734 [DEBUG] TestAgent.server.raft: vote granted: from=b6372585-a13a-4a4a-d8b6-65c173938220 term=2 tally=1 +49:32.734 [INFO] TestAgent.server.raft: election won: tally=1 +49:32.734 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23742 [Leader]" +49:32.734 [INFO] TestAgent.server: cluster leadership acquired +49:32.734 [INFO] TestAgent.server: New leader elected: payload=Node-b6372585-a13a-4a4a-d8b6-65c173938220 +49:32.735 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23742 +49:32.736 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:32.738 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:32.738 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:32.738 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:32.738 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:32.738 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:32.738 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b6372585-a13a-4a4a-d8b6-65c173938220 +49:32.738 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b6372585-a13a-4a4a-d8b6-65c173938220 +49:32.739 [INFO] TestAgent.server: federation state anti-entropy synced +49:32.813 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:32.814 [INFO] TestAgent: Synced node info +49:32.814 [DEBUG] TestAgent: Node info in sync +49:32.814 [INFO] TestAgent: Requesting shutdown +49:32.814 [INFO] TestAgent.server: shutting down server +49:32.814 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:32.814 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:32.814 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:32.814 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:32.814 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:32.814 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:32.814 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:32.815 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:32.815 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:32.815 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:32.815 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:32.815Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:32.815Z [INFO] agent.router.manager: shutting down +49:32.815 [INFO] TestAgent: consul server down +49:32.815 [INFO] TestAgent: shutdown complete +49:32.815 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23771 network=tcp +49:32.815 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23771 network=udp +49:32.815 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23738 network=tcp +49:33.317 [INFO] TestAgent: Waiting for endpoints to shut down +49:33.317 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_Alias_setToken (0.66s) +=== CONT TestAgent_ReconnectConfigWanDisabled +2022-12-04T05:49:32.412Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:32.412Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2022-12-04T05:49:32.436Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:32.436Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +49:32.438 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:624dea3a-ad1b-1d6f-f83f-3687ccd2e323 Address:127.0.0.1:23783}]" +49:32.438 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23783 [Follower]" leader= +49:32.439 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-624dea3a-ad1b-1d6f-f83f-3687ccd2e323.dc1 127.0.0.1 +49:32.439 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-624dea3a-ad1b-1d6f-f83f-3687ccd2e323 127.0.0.1 +2022-12-04T05:49:32.439Z [INFO] agent.router: Initializing LAN area manager +49:32.440 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23778 network=udp +49:32.440 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23778 network=tcp +49:32.440 [INFO] TestAgent.server: Adding LAN server: server="Node-624dea3a-ad1b-1d6f-f83f-3687ccd2e323 (Addr: tcp/127.0.0.1:23783) (DC: dc1)" +49:32.440 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-624dea3a-ad1b-1d6f-f83f-3687ccd2e323.dc1 area=wan +49:32.440 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23779 network=tcp +49:32.440 [INFO] TestAgent: started state syncer +49:32.440 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23743 network=tcp +49:32.483 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:32.483 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23783 [Candidate]" term=2 +49:32.483 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:32.483 [DEBUG] TestAgent.server.raft: vote granted: from=624dea3a-ad1b-1d6f-f83f-3687ccd2e323 term=2 tally=1 +49:32.483 [INFO] TestAgent.server.raft: election won: tally=1 +49:32.483 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23783 [Leader]" +49:32.484 [INFO] TestAgent.server: cluster leadership acquired +49:32.484 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23783 +49:32.484 [INFO] TestAgent.server: New leader elected: payload=Node-624dea3a-ad1b-1d6f-f83f-3687ccd2e323 +49:32.485 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:32.486 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:32.486 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:32.486 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:32.486 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:32.486 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:32.486 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-624dea3a-ad1b-1d6f-f83f-3687ccd2e323 +49:32.486 [INFO] TestAgent.server: member joined, marking health alive: member=Node-624dea3a-ad1b-1d6f-f83f-3687ccd2e323 +49:32.487 [INFO] TestAgent.server: federation state anti-entropy synced +49:32.834 [DEBUG] TestAgent: removed check: check=mem +49:32.834 [DEBUG] TestAgent: removed check: check=mem +49:32.834 [INFO] TestAgent: Requesting shutdown +49:32.835 [INFO] TestAgent.server: shutting down server +49:32.835 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:32.835 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:32.835 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:32.835 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:32.835 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:32.835 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:32.835 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:32.835 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:32.835 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:32.835 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:32.835 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:32.836 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:32.837 [INFO] TestAgent: consul server down +49:32.837 [INFO] TestAgent: shutdown complete +49:32.837 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23778 network=tcp +49:32.837 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23778 network=udp +2022-12-04T05:49:32.836Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:32.836Z [INFO] agent.router.manager: shutting down +49:32.837 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23779 network=tcp +49:33.337 [INFO] TestAgent: Waiting for endpoints to shut down +49:33.337 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RemoveCheck (0.96s) +=== CONT TestAgent_ReconnectConfigSettings +2022-12-04T05:49:32.657Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:32.671Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:32.672 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:15979115-4376-798f-5db4-2f453b0a50e3 Address:127.0.0.1:23825}]" +49:32.673 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23825 [Follower]" leader= +49:32.673 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-15979115-4376-798f-5db4-2f453b0a50e3.dc1 127.0.0.1 +49:32.674 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-15979115-4376-798f-5db4-2f453b0a50e3 127.0.0.1 +2022-12-04T05:49:32.674Z [INFO] agent.router: Initializing LAN area manager +49:32.674 [INFO] TestAgent.server: Adding LAN server: server="Node-15979115-4376-798f-5db4-2f453b0a50e3 (Addr: tcp/127.0.0.1:23825) (DC: dc1)" +49:32.674 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-15979115-4376-798f-5db4-2f453b0a50e3.dc1 area=wan +49:32.674 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23792 network=udp +49:32.674 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23792 network=tcp +49:32.675 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23821 network=tcp +49:32.675 [INFO] TestAgent: started state syncer +49:32.675 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23784 network=tcp +49:32.712 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:32.712 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23825 [Candidate]" term=2 +49:32.712 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:32.712 [DEBUG] TestAgent.server.raft: vote granted: from=15979115-4376-798f-5db4-2f453b0a50e3 term=2 tally=1 +49:32.712 [INFO] TestAgent.server.raft: election won: tally=1 +49:32.712 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23825 [Leader]" +49:32.712 [INFO] TestAgent.server: cluster leadership acquired +49:32.712 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23825 +49:32.713 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:32.715 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:32.715 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:32.715 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:32.715 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:32.715 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:32.715 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-15979115-4376-798f-5db4-2f453b0a50e3 +49:32.715 [INFO] TestAgent.server: member joined, marking health alive: member=Node-15979115-4376-798f-5db4-2f453b0a50e3 +49:32.716 [INFO] TestAgent.server: New leader elected: payload=Node-15979115-4376-798f-5db4-2f453b0a50e3 +49:32.732 [INFO] TestAgent.server: federation state anti-entropy synced +49:32.876 [INFO] TestAgent: Requesting shutdown +49:32.876 [INFO] TestAgent.server: shutting down server +49:32.877 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:32.877 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:32.877 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:32.877 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:32.877 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:32.877 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:32.877 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:32.877 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:32.877 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:32.877 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:32.877 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:32.877 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:32.877Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:32.877Z [INFO] agent.router.manager: shutting down +49:32.877 [INFO] TestAgent: consul server down +49:32.877 [INFO] TestAgent: shutdown complete +49:32.877 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23792 network=tcp +49:32.878 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23792 network=udp +49:32.878 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23821 network=tcp +49:33.380 [INFO] TestAgent: Waiting for endpoints to shut down +49:33.380 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_Alias_userToken (0.74s) +=== CONT TestAgent_TokenStore +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=false#01 +2022-12-04T05:49:32.643Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:32.656Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:32.658 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3f0e8879-dd7a-4c82-c21e-a8f1f79b208d Address:127.0.0.1:23700}]" +49:32.658 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3f0e8879-dd7a-4c82-c21e-a8f1f79b208d.dc1 127.0.0.1 +49:32.659 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23700 [Follower]" leader= +49:32.659 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3f0e8879-dd7a-4c82-c21e-a8f1f79b208d 127.0.0.1 +2022-12-04T05:49:32.659Z [INFO] agent.router: Initializing LAN area manager +49:32.659 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23709 network=udp +49:32.660 [INFO] TestAgent.server: Adding LAN server: server="Node-3f0e8879-dd7a-4c82-c21e-a8f1f79b208d (Addr: tcp/127.0.0.1:23700) (DC: dc1)" +49:32.660 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23709 network=tcp +49:32.660 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3f0e8879-dd7a-4c82-c21e-a8f1f79b208d.dc1 area=wan +49:32.660 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23696 network=tcp +49:32.660 [INFO] TestAgent: started state syncer +49:32.661 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23687 network=tcp +49:32.698 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:32.698 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23700 [Candidate]" term=2 +49:32.698 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:32.698 [DEBUG] TestAgent.server.raft: vote granted: from=3f0e8879-dd7a-4c82-c21e-a8f1f79b208d term=2 tally=1 +49:32.698 [INFO] TestAgent.server.raft: election won: tally=1 +49:32.698 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23700 [Leader]" +49:32.698 [INFO] TestAgent.server: cluster leadership acquired +49:32.698 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23700 +49:32.699 [INFO] TestAgent.server: New leader elected: payload=Node-3f0e8879-dd7a-4c82-c21e-a8f1f79b208d +49:32.699 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:32.701 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:32.701 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:32.701 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:32.701 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:32.701 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:32.701 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3f0e8879-dd7a-4c82-c21e-a8f1f79b208d +49:32.701 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3f0e8879-dd7a-4c82-c21e-a8f1f79b208d +49:32.702 [INFO] TestAgent.server: federation state anti-entropy synced +49:32.934 [INFO] TestAgent: Requesting shutdown +49:32.934 [INFO] TestAgent.server: shutting down server +49:32.934 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:32.934 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:32.934 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:32.934 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:32.934 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:32.934 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:32.934 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:32.934 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:32.934 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:32.934 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:32.934 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:32.934 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:32.935Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:32.935Z [INFO] agent.router.manager: shutting down +49:32.935 [INFO] TestAgent: consul server down +49:32.935 [INFO] TestAgent: shutdown complete +49:32.935 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23709 network=tcp +49:32.935 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23709 network=udp +49:32.935 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23696 network=tcp +49:33.437 [INFO] TestAgent: Waiting for endpoints to shut down +49:33.437 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_Alias_userAndSetToken (0.81s) +=== CONT TestAgent_RPCPing +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=255)_compressed=true#01 +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:32.752Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:32.773Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:32.777 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d2d581f4-4bf4-6435-397c-5570b7bb6582 Address:127.0.0.1:23756}]" +49:32.777 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23756 [Follower]" leader= +49:32.778 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d2d581f4-4bf4-6435-397c-5570b7bb6582.dc1 127.0.0.1 +49:32.778 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d2d581f4-4bf4-6435-397c-5570b7bb6582 127.0.0.1 +2022-12-04T05:49:32.778Z [INFO] agent.router: Initializing LAN area manager +49:32.778 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23806 network=udp +49:32.779 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d2d581f4-4bf4-6435-397c-5570b7bb6582.dc1 area=wan +49:32.779 [INFO] TestAgent.server: Adding LAN server: server="Node-d2d581f4-4bf4-6435-397c-5570b7bb6582 (Addr: tcp/127.0.0.1:23756) (DC: dc1)" +49:32.779 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23806 network=tcp +49:32.779 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23752 network=tcp +49:32.779 [INFO] TestAgent: started state syncer +49:32.780 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23854 network=tcp +49:32.824 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:32.824 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23756 [Candidate]" term=2 +49:32.825 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:32.825 [DEBUG] TestAgent.server.raft: vote granted: from=d2d581f4-4bf4-6435-397c-5570b7bb6582 term=2 tally=1 +49:32.825 [INFO] TestAgent.server.raft: election won: tally=1 +49:32.825 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23756 [Leader]" +49:32.825 [INFO] TestAgent.server: cluster leadership acquired +49:32.825 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23756 +49:32.826 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:32.829 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:32.829 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:32.829 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:32.829 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:32.829 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:32.829 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d2d581f4-4bf4-6435-397c-5570b7bb6582 +49:32.829 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d2d581f4-4bf4-6435-397c-5570b7bb6582 +49:32.829 [INFO] TestAgent.server: New leader elected: payload=Node-d2d581f4-4bf4-6435-397c-5570b7bb6582 +49:32.830 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.056 [INFO] TestAgent: Requesting shutdown +49:33.056 [INFO] TestAgent.server: shutting down server +49:33.056 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:33.056 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:33.056 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:33.056 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:33.056 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:33.056 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:33.056 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:33.056 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:33.056 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:33.056 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:33.056 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:33.056 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:33.057Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:33.057Z [INFO] agent.router.manager: shutting down +49:33.057 [INFO] TestAgent: consul server down +49:33.057 [INFO] TestAgent: shutdown complete +49:33.057 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23806 network=tcp +49:33.057 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23806 network=udp +49:33.057 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23752 network=tcp +49:33.558 [INFO] TestAgent: Waiting for endpoints to shut down +49:33.558 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_GRPC (0.82s) +=== CONT TestAgent_StartStop +2022-12-04T05:49:32.971Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:32.991Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:32.993 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3bebc662-7d08-f77b-0024-20245c1a94c9 Address:127.0.0.1:23818}]" +49:32.993 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23818 [Follower]" leader= +49:32.994 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3bebc662-7d08-f77b-0024-20245c1a94c9.dc1 127.0.0.1 +49:32.994 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3bebc662-7d08-f77b-0024-20245c1a94c9 127.0.0.1 +2022-12-04T05:49:32.994Z [INFO] agent.router: Initializing LAN area manager +49:32.995 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23848 network=udp +49:32.995 [INFO] TestAgent.server: Adding LAN server: server="Node-3bebc662-7d08-f77b-0024-20245c1a94c9 (Addr: tcp/127.0.0.1:23818) (DC: dc1)" +49:32.995 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3bebc662-7d08-f77b-0024-20245c1a94c9.dc1 area=wan +49:32.995 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23848 network=tcp +49:32.996 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23814 network=tcp +49:32.996 [INFO] TestAgent: started state syncer +49:32.996 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23819 network=tcp +49:33.055 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.055 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23818 [Candidate]" term=2 +49:33.056 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.056 [DEBUG] TestAgent.server.raft: vote granted: from=3bebc662-7d08-f77b-0024-20245c1a94c9 term=2 tally=1 +49:33.056 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.056 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23818 [Leader]" +49:33.056 [INFO] TestAgent.server: cluster leadership acquired +49:33.056 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23818 +49:33.056 [INFO] TestAgent.server: New leader elected: payload=Node-3bebc662-7d08-f77b-0024-20245c1a94c9 +49:33.057 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.059 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.059 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.059 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.059 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.059 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.059 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3bebc662-7d08-f77b-0024-20245c1a94c9 +49:33.059 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3bebc662-7d08-f77b-0024-20245c1a94c9 +49:33.059 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.074 [INFO] TestAgent: Requesting shutdown +49:33.074 [INFO] TestAgent.server: shutting down server +49:33.074 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:33.074 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:33.074 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:33.074 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:33.074 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:33.074 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:33.074 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:33.074 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:33.074 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:33.074 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:33.074 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:33.074 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:33.075Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:33.075Z [INFO] agent.router.manager: shutting down +49:33.075 [INFO] TestAgent: consul server down +49:33.075 [INFO] TestAgent: shutdown complete +49:33.075 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23848 network=tcp +49:33.075 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23848 network=udp +49:33.075 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23814 network=tcp +49:33.577 [INFO] TestAgent: Waiting for endpoints to shut down +49:33.577 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_ExecDisable (0.64s) +=== CONT TestAgent_Services_ExposeConfig +2022-12-04T05:49:32.831Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:32.851Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:32.853 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:eb768ebc-9aa6-f4e9-17d2-a6d6dda4da52 Address:127.0.0.1:23832}]" +49:32.853 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23832 [Follower]" leader= +49:32.853 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-eb768ebc-9aa6-f4e9-17d2-a6d6dda4da52.dc1 127.0.0.1 +49:32.854 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-eb768ebc-9aa6-f4e9-17d2-a6d6dda4da52 127.0.0.1 +2022-12-04T05:49:32.854Z [INFO] agent.router: Initializing LAN area manager +49:32.854 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23869 network=udp +49:32.854 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-eb768ebc-9aa6-f4e9-17d2-a6d6dda4da52.dc1 area=wan +49:32.854 [INFO] TestAgent.server: Adding LAN server: server="Node-eb768ebc-9aa6-f4e9-17d2-a6d6dda4da52 (Addr: tcp/127.0.0.1:23832) (DC: dc1)" +49:32.854 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23869 network=tcp +49:32.855 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23828 network=tcp +49:32.855 [INFO] TestAgent: started state syncer +49:32.855 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23847 network=tcp +49:32.921 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:32.921 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23832 [Candidate]" term=2 +49:32.921 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:32.921 [DEBUG] TestAgent.server.raft: vote granted: from=eb768ebc-9aa6-f4e9-17d2-a6d6dda4da52 term=2 tally=1 +49:32.921 [INFO] TestAgent.server.raft: election won: tally=1 +49:32.921 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23832 [Leader]" +49:32.921 [INFO] TestAgent.server: cluster leadership acquired +49:32.921 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23832 +49:32.922 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:32.924 [INFO] TestAgent.server: New leader elected: payload=Node-eb768ebc-9aa6-f4e9-17d2-a6d6dda4da52 +49:32.924 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:32.924 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:32.924 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:32.924 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:32.924 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:32.924 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eb768ebc-9aa6-f4e9-17d2-a6d6dda4da52 +49:32.924 [INFO] TestAgent.server: member joined, marking health alive: member=Node-eb768ebc-9aa6-f4e9-17d2-a6d6dda4da52 +49:32.925 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.113 [INFO] TestAgent: Requesting shutdown +49:33.113 [INFO] TestAgent.server: shutting down server +49:33.113 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:33.113 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:33.113 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:33.113 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:33.113 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:33.114 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:33.114 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:33.114 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:33.114 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:33.114 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:33.114 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:33.114 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:33.114Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:33.114Z [INFO] agent.router.manager: shutting down +49:33.114 [INFO] TestAgent: consul server down +49:33.114 [INFO] TestAgent: shutdown complete +49:33.114 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23869 network=tcp +49:33.114 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23869 network=udp +49:33.114 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23828 network=tcp +49:33.616 [INFO] TestAgent: Waiting for endpoints to shut down +49:33.616 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_ExecRemoteDisable (0.82s) +=== CONT TestAgent_HostBadACL +2022-12-04T05:49:33.056Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.056Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2022-12-04T05:49:33.068Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.068Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +49:33.070 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c41e972a-3d83-3952-97b9-90a0fae0024f Address:127.0.0.1:23846}]" +49:33.070 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c41e972a-3d83-3952-97b9-90a0fae0024f.dc1 127.0.0.1 +49:33.070 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23846 [Follower]" leader= +49:33.071 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c41e972a-3d83-3952-97b9-90a0fae0024f 127.0.0.1 +2022-12-04T05:49:33.071Z [INFO] agent.router: Initializing LAN area manager +49:33.071 [INFO] TestAgent.server: Adding LAN server: server="Node-c41e972a-3d83-3952-97b9-90a0fae0024f (Addr: tcp/127.0.0.1:23846) (DC: dc1)" +49:33.071 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c41e972a-3d83-3952-97b9-90a0fae0024f.dc1 area=wan +49:33.071 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23841 network=tcp +49:33.071 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23841 network=udp +49:33.073 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23842 network=tcp +49:33.073 [INFO] TestAgent: started state syncer +49:33.073 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23882 network=tcp +49:33.134 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.134 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23846 [Candidate]" term=2 +49:33.135 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.135 [DEBUG] TestAgent.server.raft: vote granted: from=c41e972a-3d83-3952-97b9-90a0fae0024f term=2 tally=1 +49:33.135 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.135 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23846 [Leader]" +49:33.135 [INFO] TestAgent.server: cluster leadership acquired +49:33.135 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23846 +49:33.136 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.136 [INFO] TestAgent.server: New leader elected: payload=Node-c41e972a-3d83-3952-97b9-90a0fae0024f +49:33.137 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.137 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.137 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.137 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.137 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.137 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c41e972a-3d83-3952-97b9-90a0fae0024f +49:33.137 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c41e972a-3d83-3952-97b9-90a0fae0024f +49:33.138 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.157 [INFO] TestAgent: Requesting shutdown +49:33.158 [INFO] TestAgent.server: shutting down server +49:33.158 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:33.158 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:33.158 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:33.158 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:33.158 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:33.158 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:33.158 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:33.158 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:33.158 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:33.158 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:33.158 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:33.158 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:33.158Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:33.158Z [INFO] agent.router.manager: shutting down +49:33.159 [INFO] TestAgent: consul server down +49:33.159 [INFO] TestAgent: shutdown complete +49:33.159 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23841 network=tcp +49:33.159 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23841 network=udp +49:33.159 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23842 network=tcp +49:33.660 [INFO] TestAgent: Waiting for endpoints to shut down +49:33.660 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_MissingService (0.62s) +=== CONT TestAgent_Host +2022-12-04T05:49:33.030Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.042Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:33.044 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:efc4b62c-7c71-98d7-738b-1f8f0edd5180 Address:127.0.0.1:23804}]" +49:33.045 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-efc4b62c-7c71-98d7-738b-1f8f0edd5180.dc1 127.0.0.1 +49:33.045 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23804 [Follower]" leader= +49:33.045 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-efc4b62c-7c71-98d7-738b-1f8f0edd5180 127.0.0.1 +2022-12-04T05:49:33.045Z [INFO] agent.router: Initializing LAN area manager +49:33.046 [INFO] TestAgent.server: Adding LAN server: server="Node-efc4b62c-7c71-98d7-738b-1f8f0edd5180 (Addr: tcp/127.0.0.1:23804) (DC: dc1)" +49:33.046 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-efc4b62c-7c71-98d7-738b-1f8f0edd5180.dc1 area=wan +49:33.046 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23820 network=udp +49:33.046 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23820 network=tcp +49:33.046 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23800 network=tcp +49:33.046 [INFO] TestAgent: started state syncer +49:33.046 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23840 network=tcp +49:33.088 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.088 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23804 [Candidate]" term=2 +49:33.088 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.089 [DEBUG] TestAgent.server.raft: vote granted: from=efc4b62c-7c71-98d7-738b-1f8f0edd5180 term=2 tally=1 +49:33.089 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.089 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23804 [Leader]" +49:33.089 [INFO] TestAgent.server: cluster leadership acquired +49:33.089 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23804 +49:33.091 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.092 [INFO] TestAgent.server: New leader elected: payload=Node-efc4b62c-7c71-98d7-738b-1f8f0edd5180 +49:33.093 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.093 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.093 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.093 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.093 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.093 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-efc4b62c-7c71-98d7-738b-1f8f0edd5180 +49:33.093 [INFO] TestAgent.server: member joined, marking health alive: member=Node-efc4b62c-7c71-98d7-738b-1f8f0edd5180 +49:33.095 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.265 [INFO] TestAgent: Requesting shutdown +49:33.265 [INFO] TestAgent.server: shutting down server +49:33.265 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:33.265 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:33.265 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:33.265 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:33.265 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:33.265 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:33.265 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:33.265 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:33.265 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:33.265 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:33.265 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:33.265 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:33.266Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:33.266Z [INFO] agent.router.manager: shutting down +49:33.266 [INFO] TestAgent: consul server down +49:33.266 [INFO] TestAgent: shutdown complete +49:33.266 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23820 network=tcp +49:33.266 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23820 network=udp +49:33.266 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23800 network=tcp +49:33.768 [INFO] TestAgent: Waiting for endpoints to shut down +49:33.768 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_RestoreState (0.75s) +=== CONT TestAgentConnectAuthorize_defaultAllow +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:33.196Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.196Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2022-12-04T05:49:33.211Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.211Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +49:33.213 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3fa34967-ee4f-0f65-e736-0feaa90c0969 Address:127.0.0.1:23895}]" +49:33.213 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23895 [Follower]" leader= +49:33.214 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3fa34967-ee4f-0f65-e736-0feaa90c0969.dc1 127.0.0.1 +49:33.214 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3fa34967-ee4f-0f65-e736-0feaa90c0969 127.0.0.1 +2022-12-04T05:49:33.214Z [INFO] agent.router: Initializing LAN area manager +49:33.214 [INFO] TestAgent.server: Adding LAN server: server="Node-3fa34967-ee4f-0f65-e736-0feaa90c0969 (Addr: tcp/127.0.0.1:23895) (DC: dc1)" +49:33.214 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3fa34967-ee4f-0f65-e736-0feaa90c0969.dc1 area=wan +49:33.215 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23813 network=udp +49:33.215 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23813 network=tcp +49:33.215 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23891 network=tcp +49:33.215 [INFO] TestAgent: started state syncer +49:33.215 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23757 network=tcp +49:33.283 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.283 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23895 [Candidate]" term=2 +49:33.283 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.283 [DEBUG] TestAgent.server.raft: vote granted: from=3fa34967-ee4f-0f65-e736-0feaa90c0969 term=2 tally=1 +49:33.283 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.283 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23895 [Leader]" +49:33.283 [INFO] TestAgent.server: cluster leadership acquired +49:33.284 [INFO] TestAgent.server: New leader elected: payload=Node-3fa34967-ee4f-0f65-e736-0feaa90c0969 +49:33.284 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23895 +49:33.285 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.287 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.287 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.287 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.287 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.287 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.287 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3fa34967-ee4f-0f65-e736-0feaa90c0969 +49:33.287 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3fa34967-ee4f-0f65-e736-0feaa90c0969 +49:33.292 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.333 [INFO] TestAgent: Requesting shutdown +49:33.333 [INFO] TestAgent.server: shutting down server +49:33.333 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:33.333 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:33.333 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:33.333 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:33.333 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:33.333 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:33.333 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:33.333 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:33.333 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:33.333 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:33.333 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:33.333 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:33.333Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:33.333Z [INFO] agent.router.manager: shutting down +49:33.333 [INFO] TestAgent: consul server down +49:33.333 [INFO] TestAgent: shutdown complete +49:33.333 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23813 network=tcp +49:33.333 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23813 network=udp +49:33.333 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23891 network=tcp +49:33.836 [INFO] TestAgent: Waiting for endpoints to shut down +49:33.836 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck (0.66s) +=== CONT TestAgentConnectAuthorize_defaultDeny +2022-12-04T05:49:33.085Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.085Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2022-12-04T05:49:33.117Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.117Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +49:33.123 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ba125108-0b42-437a-d341-55b97a0fd6af Address:127.0.0.1:23888}]" +49:33.123 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23888 [Follower]" leader= +49:33.124 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ba125108-0b42-437a-d341-55b97a0fd6af.dc1 127.0.0.1 +49:33.124 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ba125108-0b42-437a-d341-55b97a0fd6af 127.0.0.1 +2022-12-04T05:49:33.124Z [INFO] agent.router: Initializing LAN area manager +49:33.125 [INFO] TestAgent.server: Adding LAN server: server="Node-ba125108-0b42-437a-d341-55b97a0fd6af (Addr: tcp/127.0.0.1:23888) (DC: dc1)" +49:33.125 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23862 network=udp +49:33.125 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23862 network=tcp +49:33.125 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ba125108-0b42-437a-d341-55b97a0fd6af.dc1 area=wan +49:33.125 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23884 network=tcp +49:33.125 [INFO] TestAgent: started state syncer +49:33.126 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23896 network=tcp +49:33.170 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.170 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23888 [Candidate]" term=2 +49:33.170 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.171 [DEBUG] TestAgent.server.raft: vote granted: from=ba125108-0b42-437a-d341-55b97a0fd6af term=2 tally=1 +49:33.171 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.171 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23888 [Leader]" +49:33.171 [INFO] TestAgent.server: cluster leadership acquired +49:33.171 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23888 +49:33.172 [INFO] TestAgent.server: New leader elected: payload=Node-ba125108-0b42-437a-d341-55b97a0fd6af +49:33.172 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.175 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.175 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.175 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.175 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.175 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.175 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba125108-0b42-437a-d341-55b97a0fd6af +49:33.175 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ba125108-0b42-437a-d341-55b97a0fd6af +49:33.180 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.391 [WARN] TestAgent: check has interval below minimum: check=mem minimum_interval=1s +49:33.391 [INFO] TestAgent: Requesting shutdown +49:33.391 [INFO] TestAgent.server: shutting down server +49:33.391 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:33.391 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:33.391 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:33.391 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:33.391 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:33.391 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:33.391 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:33.391 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:33.391 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:33.391 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:33.391 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:33.391 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:33.391Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:33.391Z [INFO] agent.router.manager: shutting down +49:33.391 [INFO] TestAgent: consul server down +49:33.391 [INFO] TestAgent: shutdown complete +49:33.391 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23862 network=tcp +49:33.391 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23862 network=udp +49:33.392 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23884 network=tcp +49:33.893 [INFO] TestAgent: Waiting for endpoints to shut down +49:33.893 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_MinInterval (0.82s) +=== CONT TestAgentConnectAuthorize_serviceWrite +2022-12-04T05:49:32.360Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:32.371Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:32.372 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0335adec-f37d-87e3-63ad-42e34bebab08 Address:127.0.0.1:23790}]" +49:32.373 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0335adec-f37d-87e3-63ad-42e34bebab08.dc1 127.0.0.1 +49:32.373 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0335adec-f37d-87e3-63ad-42e34bebab08 127.0.0.1 +2022-12-04T05:49:32.373Z [INFO] agent.router: Initializing LAN area manager +49:32.374 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23785 network=udp +49:32.374 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23790 [Follower]" leader= +49:32.374 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0335adec-f37d-87e3-63ad-42e34bebab08.dc1 area=wan +49:32.374 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23785 network=tcp +49:32.374 [INFO] TestAgent.server: Adding LAN server: server="Node-0335adec-f37d-87e3-63ad-42e34bebab08 (Addr: tcp/127.0.0.1:23790) (DC: dc1)" +49:32.375 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23786 network=tcp +49:32.375 [INFO] TestAgent: started state syncer +49:32.375 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23766 network=tcp +49:32.417 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:32.417 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23790 [Candidate]" term=2 +49:32.417 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:32.417 [DEBUG] TestAgent.server.raft: vote granted: from=0335adec-f37d-87e3-63ad-42e34bebab08 term=2 tally=1 +49:32.417 [INFO] TestAgent.server.raft: election won: tally=1 +49:32.417 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23790 [Leader]" +49:32.417 [INFO] TestAgent.server: cluster leadership acquired +49:32.418 [INFO] TestAgent.server: New leader elected: payload=Node-0335adec-f37d-87e3-63ad-42e34bebab08 +49:32.418 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23790 +49:32.419 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:32.421 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:32.421 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:32.421 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:32.421 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:32.421 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:32.421 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0335adec-f37d-87e3-63ad-42e34bebab08 +49:32.421 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0335adec-f37d-87e3-63ad-42e34bebab08 +49:32.422 [INFO] TestAgent.server: federation state anti-entropy synced +49:32.704 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:32.705 [INFO] TestAgent: Synced node info +49:32.733 [WARN] TestAgent: check has interval below minimum: check=tls minimum_interval=1s +49:33.391 [DEBUG] TestAgent: Check status updated: check=tls status=passing +49:33.391 [INFO] TestAgent: Requesting shutdown +49:33.392 [INFO] TestAgent.server: shutting down server +49:33.392 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:33.392 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:33.392 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:33.392 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:33.392 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:33.392 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:33.392 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:33.392 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:33.392 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:33.392 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:33.392 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:33.392Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:33.392Z [INFO] agent.router.manager: shutting down +49:33.392 [INFO] TestAgent: consul server down +49:33.392 [INFO] TestAgent: shutdown complete +49:33.392 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23785 network=tcp +49:33.392 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23785 network=udp +49:33.392 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23786 network=tcp +49:33.894 [INFO] TestAgent: Waiting for endpoints to shut down +49:33.894 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_HTTPCheck_TLSSkipVerify (1.57s) +=== CONT TestAgentConnectAuthorize_denyWildcard +2022-12-04T05:49:33.175Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.175Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2022-12-04T05:49:33.194Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.194Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +49:33.196 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9ca68016-e741-9cec-e47a-723e2379f1b0 Address:127.0.0.1:23874}]" +49:33.196 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23874 [Follower]" leader= +49:33.197 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9ca68016-e741-9cec-e47a-723e2379f1b0.dc1 127.0.0.1 +49:33.197 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9ca68016-e741-9cec-e47a-723e2379f1b0 127.0.0.1 +2022-12-04T05:49:33.197Z [INFO] agent.router: Initializing LAN area manager +49:33.197 [INFO] TestAgent.server: Adding LAN server: server="Node-9ca68016-e741-9cec-e47a-723e2379f1b0 (Addr: tcp/127.0.0.1:23874) (DC: dc1)" +49:33.197 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23883 network=udp +49:33.197 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9ca68016-e741-9cec-e47a-723e2379f1b0.dc1 area=wan +49:33.198 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23883 network=tcp +49:33.198 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23870 network=tcp +49:33.198 [INFO] TestAgent: started state syncer +49:33.198 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23812 network=tcp +49:33.237 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.237 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23874 [Candidate]" term=2 +49:33.238 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.238 [DEBUG] TestAgent.server.raft: vote granted: from=9ca68016-e741-9cec-e47a-723e2379f1b0 term=2 tally=1 +49:33.238 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.238 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23874 [Leader]" +49:33.238 [INFO] TestAgent.server: cluster leadership acquired +49:33.238 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23874 +49:33.239 [INFO] TestAgent.server: New leader elected: payload=Node-9ca68016-e741-9cec-e47a-723e2379f1b0 +49:33.239 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.242 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.242 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.242 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.242 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.242 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.242 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9ca68016-e741-9cec-e47a-723e2379f1b0 +49:33.242 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9ca68016-e741-9cec-e47a-723e2379f1b0 +49:33.244 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.305 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:33.307 [INFO] TestAgent: Synced node info +49:33.398 [INFO] TestAgent: Requesting shutdown +49:33.398 [INFO] TestAgent.server: shutting down server +49:33.398 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:33.398 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:33.398 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:33.398 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:33.398 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:33.398 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:33.398 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:33.398 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:33.398 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:33.398 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:33.398 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:33.398Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:33.398Z [INFO] agent.router.manager: shutting down +49:33.398 [INFO] TestAgent: consul server down +49:33.398 [INFO] TestAgent: shutdown complete +49:33.398 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23883 network=tcp +49:33.398 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23883 network=udp +49:33.398 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23870 network=tcp +49:33.899 [INFO] TestAgent: Waiting for endpoints to shut down +49:33.899 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_AddCheck_StartPassing (0.75s) +=== CONT TestAgentConnectAuthorize_allowTrustDomain +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=255)_compressed=false#01 +2022-12-04T05:49:33.394Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.407Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:33.409 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d6eede27-3ab8-7bd3-5b42-ce19f0c3aae4 Address:127.0.0.1:23881}]" +49:33.409 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23881 [Follower]" leader= +49:33.410 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d6eede27-3ab8-7bd3-5b42-ce19f0c3aae4.dc1 127.0.0.1 +49:33.410 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d6eede27-3ab8-7bd3-5b42-ce19f0c3aae4 127.0.0.1 +2022-12-04T05:49:33.410Z [INFO] agent.router: Initializing LAN area manager +49:33.411 [INFO] TestAgent.server: Adding LAN server: server="Node-d6eede27-3ab8-7bd3-5b42-ce19f0c3aae4 (Addr: tcp/127.0.0.1:23881) (DC: dc1)" +49:33.411 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23890 network=udp +49:33.411 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d6eede27-3ab8-7bd3-5b42-ce19f0c3aae4.dc1 area=wan +49:33.411 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23890 network=tcp +49:33.411 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23877 network=tcp +49:33.411 [INFO] TestAgent: started state syncer +49:33.412 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23966 network=tcp +49:33.477 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.477 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23881 [Candidate]" term=2 +49:33.477 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.477 [DEBUG] TestAgent.server.raft: vote granted: from=d6eede27-3ab8-7bd3-5b42-ce19f0c3aae4 term=2 tally=1 +49:33.477 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.477 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23881 [Leader]" +49:33.477 [INFO] TestAgent.server: cluster leadership acquired +49:33.477 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23881 +49:33.478 [INFO] TestAgent.server: New leader elected: payload=Node-d6eede27-3ab8-7bd3-5b42-ce19f0c3aae4 +49:33.478 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.480 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.480 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.480 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.480 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.480 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.480 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d6eede27-3ab8-7bd3-5b42-ce19f0c3aae4 +49:33.480 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d6eede27-3ab8-7bd3-5b42-ce19f0c3aae4 +49:33.480 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.515 [INFO] TestAgent: Requesting shutdown +49:33.515 [INFO] TestAgent.server: shutting down server +49:33.515 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:33.515 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:33.515 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:33.515 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:33.515 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:33.515 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:33.515 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:33.515 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:33.515 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:33.515 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:33.515 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:33.515 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:33.515Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:33.515Z [INFO] agent.router.manager: shutting down +49:33.516 [INFO] TestAgent: consul server down +49:33.516 [INFO] TestAgent: shutdown complete +49:33.516 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23890 network=tcp +49:33.516 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23890 network=udp +49:33.516 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23877 network=tcp +49:34.016 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.016 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_TokenStore (0.64s) +=== CONT TestAgentConnectAuthorize_deny +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:33.330Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.341Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:33.343 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a32e581d-1692-b757-8338-81dd04c5c85c Address:127.0.0.1:23937}]" +49:33.343 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23937 [Follower]" leader= +49:33.344 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a32e581d-1692-b757-8338-81dd04c5c85c 127.0.0.1 +2022-12-04T05:49:33.344Z [INFO] agent.router: Initializing LAN area manager +49:33.344 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23897 network=udp +49:33.344 [INFO] TestAgent.server: Adding LAN server: server="Node-a32e581d-1692-b757-8338-81dd04c5c85c (Addr: tcp/127.0.0.1:23937) (DC: dc1)" +49:33.344 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23897 network=tcp +49:33.345 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23933 network=tcp +49:33.345 [INFO] TestAgent: started state syncer +49:33.345 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23938 network=tcp +49:33.398 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.398 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23937 [Candidate]" term=2 +49:33.399 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.399 [DEBUG] TestAgent.server.raft: vote granted: from=a32e581d-1692-b757-8338-81dd04c5c85c term=2 tally=1 +49:33.399 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.399 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23937 [Leader]" +49:33.399 [INFO] TestAgent.server: cluster leadership acquired +49:33.399 [INFO] TestAgent.server: New leader elected: payload=Node-a32e581d-1692-b757-8338-81dd04c5c85c +49:33.399 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23937 +49:33.400 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.401 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.401 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.401 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.401 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.401 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.401 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a32e581d-1692-b757-8338-81dd04c5c85c +49:33.401 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a32e581d-1692-b757-8338-81dd04c5c85c +49:33.402 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.488 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:33.489 [INFO] TestAgent: Synced node info +49:33.489 [DEBUG] TestAgent: Node info in sync +49:33.584 [INFO] TestAgent: Requesting shutdown +49:33.584 [INFO] TestAgent.server: shutting down server +49:33.584 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:33.584 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:33.584 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:33.584 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:33.584 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:33.584 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:33.584 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:33.584 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:33.584 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:33.584 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:33.584Z [INFO] agent.router.manager: shutting down +49:33.585 [INFO] TestAgent: consul server down +49:33.585 [INFO] TestAgent: shutdown complete +49:33.585 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23897 network=tcp +49:33.585 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23897 network=udp +49:33.585 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23933 network=tcp +49:34.085 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.085 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ReconnectConfigWanDisabled (0.77s) +=== CONT TestAgentConnectAuthorize_allow +2022-12-04T05:49:33.638Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:33.638Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.652Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:33.652Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:33.658 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c38ce1df-48da-60f6-f30d-495fc84ed2e7 Address:127.0.0.1:23902}]" +49:33.659 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23902 [Follower]" leader= +49:33.660 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c38ce1df-48da-60f6-f30d-495fc84ed2e7.dc1 127.0.0.1 +49:33.660 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c38ce1df-48da-60f6-f30d-495fc84ed2e7 127.0.0.1 +2022-12-04T05:49:33.660Z [INFO] agent.router: Initializing LAN area manager +49:33.661 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23911 network=udp +49:33.661 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c38ce1df-48da-60f6-f30d-495fc84ed2e7.dc1 area=wan +49:33.661 [INFO] TestAgent.server: Adding LAN server: server="Node-c38ce1df-48da-60f6-f30d-495fc84ed2e7 (Addr: tcp/127.0.0.1:23902) (DC: dc1)" +49:33.662 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23911 network=tcp +49:33.663 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23898 network=tcp +49:33.663 [INFO] TestAgent: started state syncer +49:33.663 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23924 network=tcp +49:33.713 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:33.724 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.724 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23902 [Candidate]" term=2 +49:33.725 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.725 [DEBUG] TestAgent.server.raft: vote granted: from=c38ce1df-48da-60f6-f30d-495fc84ed2e7 term=2 tally=1 +49:33.725 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.725 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23902 [Leader]" +49:33.725 [INFO] TestAgent.server: cluster leadership acquired +49:33.725 [INFO] TestAgent.server: New leader elected: payload=Node-c38ce1df-48da-60f6-f30d-495fc84ed2e7 +49:33.725 [INFO] TestAgent.server: initializing acls +49:33.726 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:33.726 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:33.726 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:33.727 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:33.727 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:33.727 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:33.727 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c38ce1df-48da-60f6-f30d-495fc84ed2e7 +49:33.727 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c38ce1df-48da-60f6-f30d-495fc84ed2e7.dc1 +49:33.727 [INFO] TestAgent.server: Updating LAN server: server="Node-c38ce1df-48da-60f6-f30d-495fc84ed2e7 (Addr: tcp/127.0.0.1:23902) (DC: dc1)" +49:33.728 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c38ce1df-48da-60f6-f30d-495fc84ed2e7.dc1 area=wan +49:33.729 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.731 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.731 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.731 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.731 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.731 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.732 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.733 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c38ce1df-48da-60f6-f30d-495fc84ed2e7 +49:33.733 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c38ce1df-48da-60f6-f30d-495fc84ed2e7 +49:33.735 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c38ce1df-48da-60f6-f30d-495fc84ed2e7 +49:33.735 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-c38ce1df-48da-60f6-f30d-495fc84ed2e7 +49:33.735 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-c38ce1df-48da-60f6-f30d-495fc84ed2e7 +49:33.735 [INFO] TestAgent: Requesting shutdown +49:33.735 [INFO] TestAgent.server: shutting down server +49:33.735 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:33.735 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:33.735 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:33.735 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:33.735 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:33.735 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:33.735 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:33.736 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:33.736 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:33.736 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:33.736 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:33.736 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:33.736 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:33.736 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:33.736 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:33.736 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:33.736Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:33.736Z [INFO] agent.router.manager: shutting down +49:33.736 [INFO] TestAgent: consul server down +49:33.736 [INFO] TestAgent: shutdown complete +49:33.736 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23911 network=tcp +49:33.736 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23911 network=udp +49:33.736 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23898 network=tcp +49:34.237 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.237 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_HostBadACL (0.62s) +=== CONT TestAgentConnectAuthorize_idNotService +2022-12-04T05:49:33.686Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:33.686Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.699Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:33.699Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:33.701 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6c649931-2a51-f90f-8233-af9b906cd58a Address:127.0.0.1:23909}]" +49:33.702 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6c649931-2a51-f90f-8233-af9b906cd58a.dc1 127.0.0.1 +49:33.703 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6c649931-2a51-f90f-8233-af9b906cd58a 127.0.0.1 +2022-12-04T05:49:33.703Z [INFO] agent.router: Initializing LAN area manager +49:33.703 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23909 [Follower]" leader= +49:33.703 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23946 network=udp +49:33.703 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6c649931-2a51-f90f-8233-af9b906cd58a.dc1 area=wan +49:33.703 [INFO] TestAgent.server: Adding LAN server: server="Node-6c649931-2a51-f90f-8233-af9b906cd58a (Addr: tcp/127.0.0.1:23909) (DC: dc1)" +49:33.703 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23946 network=tcp +49:33.704 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23905 network=tcp +49:33.705 [INFO] TestAgent: started state syncer +49:33.705 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23945 network=tcp +49:33.746 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.747 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23909 [Candidate]" term=2 +49:33.747 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.747 [DEBUG] TestAgent.server.raft: vote granted: from=6c649931-2a51-f90f-8233-af9b906cd58a term=2 tally=1 +49:33.747 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.747 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23909 [Leader]" +49:33.747 [INFO] TestAgent.server: cluster leadership acquired +49:33.747 [INFO] TestAgent.server: New leader elected: payload=Node-6c649931-2a51-f90f-8233-af9b906cd58a +49:33.747 [INFO] TestAgent.server: initializing acls +49:33.748 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:33.748 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:33.748 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:33.748 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:33.748 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:33.748 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:33.749 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-6c649931-2a51-f90f-8233-af9b906cd58a +49:33.749 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-6c649931-2a51-f90f-8233-af9b906cd58a.dc1 +49:33.749 [INFO] TestAgent.server: Updating LAN server: server="Node-6c649931-2a51-f90f-8233-af9b906cd58a (Addr: tcp/127.0.0.1:23909) (DC: dc1)" +49:33.749 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-6c649931-2a51-f90f-8233-af9b906cd58a.dc1 area=wan +49:33.749 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.751 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.751 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.751 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.751 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.751 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.751 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6c649931-2a51-f90f-8233-af9b906cd58a +49:33.751 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6c649931-2a51-f90f-8233-af9b906cd58a +49:33.751 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.751 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6c649931-2a51-f90f-8233-af9b906cd58a +49:33.781 [INFO] TestAgent: Requesting shutdown +49:33.781 [INFO] TestAgent.server: shutting down server +49:33.781 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:33.781 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:33.781 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:33.781 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:33.781 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:33.781 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:33.781 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:33.781 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:33.781 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:33.781 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:33.781 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:33.781 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:33.781 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:33.781 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:33.781 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:33.781 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:33.781Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:33.782Z [INFO] agent.router.manager: shutting down +49:33.782 [INFO] TestAgent: consul server down +49:33.782 [INFO] TestAgent: shutdown complete +49:33.782 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23946 network=tcp +49:33.782 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23946 network=udp +49:33.782 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23905 network=tcp +49:34.282 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.282 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Host (0.62s) +=== CONT TestAgentConnectAuthorize_idInvalidFormat +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:33.456Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.474Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:33.475 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:de829842-59af-8241-5c90-827920407359 Address:127.0.0.1:23406}]" +49:33.476 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-de829842-59af-8241-5c90-827920407359.dc1 127.0.0.1 +49:33.476 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23406 [Follower]" leader= +49:33.476 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-de829842-59af-8241-5c90-827920407359 127.0.0.1 +2022-12-04T05:49:33.476Z [INFO] agent.router: Initializing LAN area manager +49:33.477 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23967 network=udp +49:33.477 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-de829842-59af-8241-5c90-827920407359.dc1 area=wan +49:33.477 [INFO] TestAgent.server: Adding LAN server: server="Node-de829842-59af-8241-5c90-827920407359 (Addr: tcp/127.0.0.1:23406) (DC: dc1)" +49:33.477 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23967 network=tcp +49:33.478 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23402 network=tcp +49:33.478 [INFO] TestAgent: started state syncer +49:33.478 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23903 network=tcp +49:33.538 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.538 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23406 [Candidate]" term=2 +49:33.538 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.538 [DEBUG] TestAgent.server.raft: vote granted: from=de829842-59af-8241-5c90-827920407359 term=2 tally=1 +49:33.538 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.538 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23406 [Leader]" +49:33.538 [INFO] TestAgent.server: cluster leadership acquired +49:33.538 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23406 +49:33.539 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.539 [INFO] TestAgent.server: New leader elected: payload=Node-de829842-59af-8241-5c90-827920407359 +49:33.541 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.541 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.541 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.541 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.541 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.541 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-de829842-59af-8241-5c90-827920407359 +49:33.541 [INFO] TestAgent.server: member joined, marking health alive: member=Node-de829842-59af-8241-5c90-827920407359 +49:33.541 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.812 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:33.813 [INFO] TestAgent: Synced node info +49:33.813 [DEBUG] TestAgent: Node info in sync +49:33.849 [INFO] TestAgent: Requesting shutdown +49:33.849 [INFO] TestAgent.server: shutting down server +49:33.849 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:33.849 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:33.849 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:33.849 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:33.849 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:33.849 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:33.849 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:33.849 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:33.849 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:33.849 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:33.849 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:33.849Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:33.849Z [INFO] agent.router.manager: shutting down +49:33.850 [INFO] TestAgent: consul server down +49:33.850 [INFO] TestAgent: shutdown complete +49:33.850 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23967 network=tcp +49:33.850 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23967 network=udp +49:33.850 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23402 network=tcp +49:34.350 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.350 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RPCPing (0.91s) +=== CONT TestAgentConnectAuthorize_noTarget +2022-12-04T05:49:33.590Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.630Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:33.632 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5a71addd-50e5-b47a-b7aa-39c4b84a0941 Address:127.0.0.1:23930}]" +49:33.633 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5a71addd-50e5-b47a-b7aa-39c4b84a0941.dc1 127.0.0.1 +49:33.634 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5a71addd-50e5-b47a-b7aa-39c4b84a0941 127.0.0.1 +2022-12-04T05:49:33.634Z [INFO] agent.router: Initializing LAN area manager +49:33.634 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23932 network=udp +49:33.634 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23930 [Follower]" leader= +49:33.634 [INFO] TestAgent.server: Adding LAN server: server="Node-5a71addd-50e5-b47a-b7aa-39c4b84a0941 (Addr: tcp/127.0.0.1:23930) (DC: dc1)" +49:33.634 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5a71addd-50e5-b47a-b7aa-39c4b84a0941.dc1 area=wan +49:33.634 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23932 network=tcp +49:33.635 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23926 network=tcp +49:33.635 [INFO] TestAgent: started state syncer +49:33.635 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23931 network=tcp +49:33.684 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.684 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23930 [Candidate]" term=2 +49:33.684 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.684 [DEBUG] TestAgent.server.raft: vote granted: from=5a71addd-50e5-b47a-b7aa-39c4b84a0941 term=2 tally=1 +49:33.684 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.684 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23930 [Leader]" +49:33.685 [INFO] TestAgent.server: cluster leadership acquired +49:33.685 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23930 +49:33.686 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.686 [INFO] TestAgent.server: New leader elected: payload=Node-5a71addd-50e5-b47a-b7aa-39c4b84a0941 +49:33.687 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.687 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.687 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.687 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.687 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.687 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a71addd-50e5-b47a-b7aa-39c4b84a0941 +49:33.687 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5a71addd-50e5-b47a-b7aa-39c4b84a0941 +49:33.688 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.724 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:33.726 [INFO] TestAgent: Synced node info +49:33.726 [DEBUG] TestAgent: Node info in sync +49:33.888 [INFO] TestAgent: Requesting shutdown +49:33.888 [INFO] TestAgent.server: shutting down server +49:33.888 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:33.888 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:33.888 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:33.888 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:33.888 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:33.888 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:33.888 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:33.888 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:33.888 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:33.888Z [INFO] agent.router.manager: shutting down +49:33.888 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +2022-12-04T05:49:33.888Z [INFO] agent.router.manager: shutting down +49:33.888 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:33.888 [INFO] TestAgent: consul server down +49:33.888 [INFO] TestAgent: shutdown complete +49:33.888 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23932 network=tcp +49:33.888 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23932 network=udp +49:33.888 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23926 network=tcp +49:34.395 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.395 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Services_ExposeConfig (0.82s) +=== CONT TestAgentConnectAuthorize_badBody +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=255)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=255)_compressed=false#01 +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:33.329Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.341Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:33.343 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ed5aed25-2090-05a7-7ff3-cca31d88ece1 Address:127.0.0.1:23811}]" +49:33.343 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23811 [Follower]" leader= +49:33.343 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ed5aed25-2090-05a7-7ff3-cca31d88ece1.dc1 127.0.0.1 +49:33.344 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ed5aed25-2090-05a7-7ff3-cca31d88ece1 127.0.0.1 +2022-12-04T05:49:33.344Z [INFO] agent.router: Initializing LAN area manager +49:33.344 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ed5aed25-2090-05a7-7ff3-cca31d88ece1.dc1 area=wan +49:33.344 [INFO] TestAgent.server: Adding LAN server: server="Node-ed5aed25-2090-05a7-7ff3-cca31d88ece1 (Addr: tcp/127.0.0.1:23811) (DC: dc1)" +49:33.344 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23758 network=udp +49:33.344 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23758 network=tcp +49:33.345 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23807 network=tcp +49:33.345 [INFO] TestAgent: started state syncer +49:33.345 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23889 network=tcp +49:33.407 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.407 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23811 [Candidate]" term=2 +49:33.407 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.407 [DEBUG] TestAgent.server.raft: vote granted: from=ed5aed25-2090-05a7-7ff3-cca31d88ece1 term=2 tally=1 +49:33.407 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.407 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23811 [Leader]" +49:33.407 [INFO] TestAgent.server: cluster leadership acquired +49:33.407 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23811 +49:33.408 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.408 [INFO] TestAgent.server: New leader elected: payload=Node-ed5aed25-2090-05a7-7ff3-cca31d88ece1 +49:33.410 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.410 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.410 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.410 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.410 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.410 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ed5aed25-2090-05a7-7ff3-cca31d88ece1 +49:33.410 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ed5aed25-2090-05a7-7ff3-cca31d88ece1 +49:33.410 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.534 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:33.535 [INFO] TestAgent: Synced node info +49:33.535 [DEBUG] TestAgent: Node info in sync +49:33.563 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:33.563 [DEBUG] TestAgent: Node info in sync +49:33.564 [INFO] TestAgent: Synced service: service=redis +49:33.564 [DEBUG] TestAgent: Check in sync: check=redis-check +49:33.564 [INFO] TestAgent: Synced check: check=node-check +49:34.066 [DEBUG] TestAgent: Registered node: node="Node-level check" +49:34.066 [DEBUG] TestAgent: Registered node: node="Serf Health Status" +49:34.066 [DEBUG] TestAgent: Registered node: node="Service-level check" +49:34.066 [INFO] TestAgent: Sync in progress: iteration=1 +49:34.066 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.066 [DEBUG] TestAgent: Node info in sync +49:34.066 [DEBUG] TestAgent: Service in sync: service=redis +49:34.066 [DEBUG] TestAgent: Check in sync: check=redis-check +49:34.066 [DEBUG] TestAgent: Check in sync: check=node-check +49:34.066 [INFO] TestAgent: Sync in progress: iteration=2 +49:34.066 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.066 [DEBUG] TestAgent: Node info in sync +49:34.066 [DEBUG] TestAgent: Service in sync: service=redis +49:34.066 [DEBUG] TestAgent: Check in sync: check=redis-check +49:34.066 [DEBUG] TestAgent: Check in sync: check=node-check +49:34.066 [INFO] TestAgent: Sync in progress: iteration=3 +49:34.066 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.066 [DEBUG] TestAgent: Node info in sync +49:34.066 [DEBUG] TestAgent: Service in sync: service=redis +49:34.066 [DEBUG] TestAgent: Check in sync: check=node-check +49:34.066 [DEBUG] TestAgent: Check in sync: check=redis-check +49:34.066 [INFO] TestAgent: Sync in progress: iteration=4 +49:34.066 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.066 [DEBUG] TestAgent: Node info in sync +49:34.066 [DEBUG] TestAgent: Service in sync: service=redis +49:34.066 [DEBUG] TestAgent: Check in sync: check=redis-check +49:34.066 [DEBUG] TestAgent: Check in sync: check=node-check +49:34.066 [INFO] TestAgent: Sync in progress: iteration=5 +49:34.066 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.066 [DEBUG] TestAgent: Node info in sync +49:34.066 [DEBUG] TestAgent: Service in sync: service=redis +49:34.066 [DEBUG] TestAgent: Check in sync: check=redis-check +49:34.066 [DEBUG] TestAgent: Check in sync: check=node-check +49:34.066 [INFO] TestAgent: Sync in progress: iteration=6 +49:34.066 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.066 [DEBUG] TestAgent: Node info in sync +49:34.066 [DEBUG] TestAgent: Service in sync: service=redis +49:34.066 [DEBUG] TestAgent: Check in sync: check=redis-check +49:34.066 [DEBUG] TestAgent: Check in sync: check=node-check +49:34.066 [INFO] TestAgent: Sync in progress: iteration=7 +49:34.066 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.066 [DEBUG] TestAgent: Node info in sync +49:34.066 [DEBUG] TestAgent: Service in sync: service=redis +49:34.066 [DEBUG] TestAgent: Check in sync: check=redis-check +49:34.066 [DEBUG] TestAgent: Check in sync: check=node-check +49:34.066 [INFO] TestAgent: Sync in progress: iteration=8 +49:34.066 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.066 [DEBUG] TestAgent: Node info in sync +49:34.066 [DEBUG] TestAgent: Service in sync: service=redis +49:34.066 [DEBUG] TestAgent: Check in sync: check=redis-check +49:34.066 [DEBUG] TestAgent: Check in sync: check=node-check +49:34.066 [INFO] TestAgent: Sync in progress: iteration=9 +49:34.067 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.067 [DEBUG] TestAgent: Node info in sync +49:34.067 [DEBUG] TestAgent: Service in sync: service=redis +49:34.067 [DEBUG] TestAgent: Check in sync: check=redis-check +49:34.067 [DEBUG] TestAgent: Check in sync: check=node-check +49:34.067 [INFO] TestAgent: Sync in progress: iteration=10 +49:34.067 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.067 [DEBUG] TestAgent: Node info in sync +49:34.067 [DEBUG] TestAgent: Service in sync: service=redis +49:34.067 [DEBUG] TestAgent: Check in sync: check=redis-check +49:34.067 [DEBUG] TestAgent: Check in sync: check=node-check +49:34.067 [INFO] TestAgent: Requesting shutdown +49:34.067 [INFO] TestAgent.server: shutting down server +49:34.067 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:34.067 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:34.067 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:34.067 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:34.067 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:34.067 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:34.067 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:34.067 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:34.067 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:34.067 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:34.067 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:49:34.067Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:34.067Z [INFO] agent.router.manager: shutting down +49:34.067 [INFO] TestAgent: consul server down +49:34.067 [INFO] TestAgent: shutdown complete +49:34.067 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23758 network=tcp +49:34.067 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23758 network=udp +49:34.067 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23807 network=tcp +49:34.568 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.568 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_IndexChurn/with_tags +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=true#01 +2022-12-04T05:49:33.816Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:33.816Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.828Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:33.828Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:33.830 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6d11d374-6dc7-cb4b-5217-304dd28bdd9e Address:127.0.0.1:23972}]" +49:33.830 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23972 [Follower]" leader= +49:33.830 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6d11d374-6dc7-cb4b-5217-304dd28bdd9e.dc1 127.0.0.1 +49:33.831 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6d11d374-6dc7-cb4b-5217-304dd28bdd9e 127.0.0.1 +2022-12-04T05:49:33.831Z [INFO] agent.router: Initializing LAN area manager +49:33.831 [INFO] TestAgent.server: Adding LAN server: server="Node-6d11d374-6dc7-cb4b-5217-304dd28bdd9e (Addr: tcp/127.0.0.1:23972) (DC: dc1)" +49:33.831 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6d11d374-6dc7-cb4b-5217-304dd28bdd9e.dc1 area=wan +49:33.831 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23953 network=udp +49:33.831 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23953 network=tcp +49:33.832 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23968 network=tcp +49:33.832 [INFO] TestAgent: started state syncer +49:33.832 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23952 network=tcp +49:33.876 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.876 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23972 [Candidate]" term=2 +49:33.877 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.877 [DEBUG] TestAgent.server.raft: vote granted: from=6d11d374-6dc7-cb4b-5217-304dd28bdd9e term=2 tally=1 +49:33.877 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.877 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23972 [Leader]" +49:33.877 [INFO] TestAgent.server: cluster leadership acquired +49:33.877 [INFO] TestAgent.server: New leader elected: payload=Node-6d11d374-6dc7-cb4b-5217-304dd28bdd9e +49:33.877 [INFO] TestAgent.server: initializing acls +49:33.877 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:33.877 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:33.878 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:33.878 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:33.878 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:33.878 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:33.879 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-6d11d374-6dc7-cb4b-5217-304dd28bdd9e +49:33.879 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-6d11d374-6dc7-cb4b-5217-304dd28bdd9e.dc1 +49:33.879 [INFO] TestAgent.server: Updating LAN server: server="Node-6d11d374-6dc7-cb4b-5217-304dd28bdd9e (Addr: tcp/127.0.0.1:23972) (DC: dc1)" +49:33.879 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-6d11d374-6dc7-cb4b-5217-304dd28bdd9e.dc1 area=wan +49:33.880 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.881 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.881 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.881 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.881 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.881 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.881 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6d11d374-6dc7-cb4b-5217-304dd28bdd9e +49:33.881 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6d11d374-6dc7-cb4b-5217-304dd28bdd9e +49:33.882 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6d11d374-6dc7-cb4b-5217-304dd28bdd9e +49:33.882 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.125 [INFO] TestAgent: Requesting shutdown +49:34.125 [INFO] TestAgent.server: shutting down server +49:34.125 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:34.125 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:34.125 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:34.125 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:34.125 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:34.125 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:34.125 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:34.125 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:34.125 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:34.125 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:34.125 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:34.125 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:34.125 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:34.125 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:34.125 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:34.125 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:34.126Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:34.126Z [INFO] agent.router.manager: shutting down +49:34.126 [INFO] TestAgent: consul server down +49:34.126 [INFO] TestAgent: shutdown complete +49:34.126 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23953 network=tcp +49:34.126 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23953 network=udp +49:34.126 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23968 network=tcp +49:34.628 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.628 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:33.941Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.958Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:33.961 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1275e531-f903-d5cc-ab08-152bf2f68c43 Address:127.0.0.1:23944}]" +49:33.961 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23944 [Follower]" leader= +49:33.962 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1275e531-f903-d5cc-ab08-152bf2f68c43.dc1 127.0.0.1 +49:33.963 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1275e531-f903-d5cc-ab08-152bf2f68c43 127.0.0.1 +2022-12-04T05:49:33.963Z [INFO] agent.router: Initializing LAN area manager +49:33.964 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23981 network=udp +49:33.964 [INFO] TestAgent.server: Adding LAN server: server="Node-1275e531-f903-d5cc-ab08-152bf2f68c43 (Addr: tcp/127.0.0.1:23944) (DC: dc1)" +49:33.964 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1275e531-f903-d5cc-ab08-152bf2f68c43.dc1 area=wan +49:33.964 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23981 network=tcp +49:33.965 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23940 network=tcp +49:33.965 [INFO] TestAgent: started state syncer +49:33.965 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23358 network=tcp +49:34.020 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.020 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23944 [Candidate]" term=2 +49:34.020 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.020 [DEBUG] TestAgent.server.raft: vote granted: from=1275e531-f903-d5cc-ab08-152bf2f68c43 term=2 tally=1 +49:34.020 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.020 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23944 [Leader]" +49:34.020 [INFO] TestAgent.server: cluster leadership acquired +49:34.020 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23944 +49:34.021 [INFO] TestAgent.server: New leader elected: payload=Node-1275e531-f903-d5cc-ab08-152bf2f68c43 +49:34.021 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.022 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.023 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.023 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.023 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.023 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.023 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1275e531-f903-d5cc-ab08-152bf2f68c43 +49:34.023 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1275e531-f903-d5cc-ab08-152bf2f68c43 +49:34.023 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.094 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.094 [INFO] TestAgent: Synced node info +49:34.123 [INFO] TestAgent: Requesting shutdown +49:34.123 [INFO] TestAgent.server: shutting down server +49:34.123 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:34.123 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:34.123 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:34.123 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:34.123 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:34.123 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:34.123 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:34.123 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:34.123 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:34.123 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:34.123 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:34.124Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:34.124Z [INFO] agent.router.manager: shutting down +49:34.124 [INFO] TestAgent: consul server down +49:34.124 [INFO] TestAgent: shutdown complete +49:34.124 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23981 network=tcp +49:34.124 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23981 network=udp +49:34.124 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23940 network=tcp +49:34.628 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.628 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_allowTrustDomain (0.73s) +=== CONT TestAgentConnectCALeafCert_secondaryDC_good +--- PASS: TestAgentConnectAuthorize_defaultAllow (0.86s) +=== CONT TestAgentConnectCALeafCert_aclServiceReadDeny +2022-12-04T05:49:33.867Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:33.867Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.883Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:33.883Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:33.886 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6b2a83bf-c1e0-9757-6f74-e1875cddef72 Address:127.0.0.1:23993}]" +49:33.886 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23993 [Follower]" leader= +49:33.886 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6b2a83bf-c1e0-9757-6f74-e1875cddef72.dc1 127.0.0.1 +49:33.887 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6b2a83bf-c1e0-9757-6f74-e1875cddef72 127.0.0.1 +2022-12-04T05:49:33.887Z [INFO] agent.router: Initializing LAN area manager +49:33.888 [INFO] TestAgent.server: Adding LAN server: server="Node-6b2a83bf-c1e0-9757-6f74-e1875cddef72 (Addr: tcp/127.0.0.1:23993) (DC: dc1)" +49:33.888 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6b2a83bf-c1e0-9757-6f74-e1875cddef72.dc1 area=wan +49:33.888 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23939 network=tcp +49:33.888 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23939 network=udp +49:33.889 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23989 network=tcp +49:33.889 [INFO] TestAgent: started state syncer +49:33.889 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23414 network=tcp +49:33.929 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.930 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23993 [Candidate]" term=2 +49:33.930 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.930 [DEBUG] TestAgent.server.raft: vote granted: from=6b2a83bf-c1e0-9757-6f74-e1875cddef72 term=2 tally=1 +49:33.930 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.930 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23993 [Leader]" +49:33.930 [INFO] TestAgent.server: cluster leadership acquired +49:33.930 [INFO] TestAgent.server: initializing acls +49:33.931 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:33.931 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:33.931 [INFO] TestAgent.server: New leader elected: payload=Node-6b2a83bf-c1e0-9757-6f74-e1875cddef72 +49:33.931 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:33.933 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:33.933 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:33.933 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:33.933 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-6b2a83bf-c1e0-9757-6f74-e1875cddef72 +49:33.933 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-6b2a83bf-c1e0-9757-6f74-e1875cddef72.dc1 +49:33.934 [INFO] TestAgent.server: Updating LAN server: server="Node-6b2a83bf-c1e0-9757-6f74-e1875cddef72 (Addr: tcp/127.0.0.1:23993) (DC: dc1)" +49:33.934 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-6b2a83bf-c1e0-9757-6f74-e1875cddef72.dc1 area=wan +49:33.935 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.945 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.945 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.945 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.945 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.945 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.945 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6b2a83bf-c1e0-9757-6f74-e1875cddef72 +49:33.945 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6b2a83bf-c1e0-9757-6f74-e1875cddef72 +49:33.946 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6b2a83bf-c1e0-9757-6f74-e1875cddef72 +49:33.946 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.151 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-6b2a83bf-c1e0-9757-6f74-e1875cddef72 +49:34.152 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-6b2a83bf-c1e0-9757-6f74-e1875cddef72 +49:34.153 [INFO] TestAgent: Requesting shutdown +49:34.153 [INFO] TestAgent.server: shutting down server +49:34.153 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:34.153 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:34.153 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:34.153 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:34.153 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:34.153 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:34.153 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:34.153 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:34.153 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:34.153 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:34.153 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:34.153 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:34.153 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:34.153 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:34.153 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:34.153 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:34.153Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:34.154Z [INFO] agent.router.manager: shutting down +49:34.154 [INFO] TestAgent: consul server down +49:34.154 [INFO] TestAgent: shutdown complete +49:34.154 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23939 network=tcp +49:34.154 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23939 network=udp +49:34.154 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23989 network=tcp +49:34.655 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.655 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_defaultDeny (0.82s) +=== CONT TestAgentConnectCALeafCert_aclServiceWrite +2022-12-04T05:49:34.027Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.038Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.040 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:61f851ae-960f-772a-efe7-117d67bdfec3 Address:127.0.0.1:23951}]" +49:34.040 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23951 [Follower]" leader= +49:34.041 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-61f851ae-960f-772a-efe7-117d67bdfec3.dc1 127.0.0.1 +49:34.041 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-61f851ae-960f-772a-efe7-117d67bdfec3 127.0.0.1 +2022-12-04T05:49:34.041Z [INFO] agent.router: Initializing LAN area manager +49:34.042 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23359 network=udp +49:34.042 [INFO] TestAgent.server: Adding LAN server: server="Node-61f851ae-960f-772a-efe7-117d67bdfec3 (Addr: tcp/127.0.0.1:23951) (DC: dc1)" +49:34.042 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-61f851ae-960f-772a-efe7-117d67bdfec3.dc1 area=wan +49:34.042 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23359 network=tcp +49:34.042 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23947 network=tcp +49:34.042 [INFO] TestAgent: started state syncer +49:34.043 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23910 network=tcp +49:34.092 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.092 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23951 [Candidate]" term=2 +49:34.092 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.092 [DEBUG] TestAgent.server.raft: vote granted: from=61f851ae-960f-772a-efe7-117d67bdfec3 term=2 tally=1 +49:34.092 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.092 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23951 [Leader]" +49:34.092 [INFO] TestAgent.server: cluster leadership acquired +49:34.093 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23951 +49:34.093 [INFO] TestAgent.server: New leader elected: payload=Node-61f851ae-960f-772a-efe7-117d67bdfec3 +49:34.093 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.095 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.095 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.095 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.095 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.095 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.095 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-61f851ae-960f-772a-efe7-117d67bdfec3 +49:34.095 [INFO] TestAgent.server: member joined, marking health alive: member=Node-61f851ae-960f-772a-efe7-117d67bdfec3 +49:34.096 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.171 [INFO] TestAgent: Requesting shutdown +49:34.171 [INFO] TestAgent.server: shutting down server +49:34.171 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:34.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:34.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:34.171 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:34.171 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:34.172 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:34.172 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:34.172 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:34.172 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:34.172 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:34.172 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:34.172 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:49:34.173Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:34.173Z [INFO] agent.router.manager: shutting down +49:34.173 [INFO] TestAgent: consul server down +49:34.173 [INFO] TestAgent: shutdown complete +49:34.173 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23359 network=tcp +49:34.173 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23359 network=udp +49:34.173 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23947 network=tcp +49:34.673 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.674 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_deny (0.66s) +=== CONT TestAgentConnectCALeafCert_aclDefaultDeny +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:34.099Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.116Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.119 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:59c33e31-7751-06df-cf19-4a7ac1302eec Address:127.0.0.1:23916}]" +49:34.120 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-59c33e31-7751-06df-cf19-4a7ac1302eec.dc1 127.0.0.1 +49:34.120 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23916 [Follower]" leader= +49:34.121 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-59c33e31-7751-06df-cf19-4a7ac1302eec 127.0.0.1 +2022-12-04T05:49:34.121Z [INFO] agent.router: Initializing LAN area manager +49:34.121 [INFO] TestAgent.server: Adding LAN server: server="Node-59c33e31-7751-06df-cf19-4a7ac1302eec (Addr: tcp/127.0.0.1:23916) (DC: dc1)" +49:34.121 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23925 network=tcp +49:34.121 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-59c33e31-7751-06df-cf19-4a7ac1302eec.dc1 area=wan +49:34.121 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23925 network=udp +49:34.122 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23912 network=tcp +49:34.123 [INFO] TestAgent: started state syncer +49:34.123 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23875 network=tcp +49:34.177 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.177 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23916 [Candidate]" term=2 +49:34.178 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.178 [DEBUG] TestAgent.server.raft: vote granted: from=59c33e31-7751-06df-cf19-4a7ac1302eec term=2 tally=1 +49:34.178 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.178 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23916 [Leader]" +49:34.178 [INFO] TestAgent.server: cluster leadership acquired +49:34.179 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23916 +49:34.180 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.181 [INFO] TestAgent.server: New leader elected: payload=Node-59c33e31-7751-06df-cf19-4a7ac1302eec +49:34.182 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.182 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.182 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.182 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.182 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.182 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-59c33e31-7751-06df-cf19-4a7ac1302eec +49:34.182 [INFO] TestAgent.server: member joined, marking health alive: member=Node-59c33e31-7751-06df-cf19-4a7ac1302eec +49:34.183 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.306 [INFO] TestAgent: Requesting shutdown +49:34.307 [INFO] TestAgent.server: shutting down server +49:34.307 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:34.307 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:34.307 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:34.307 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:34.307 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:34.307 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:34.307 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:34.307 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:34.307 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:34.307 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:34.307 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:34.307 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:34.307Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:34.307Z [INFO] agent.router.manager: shutting down +49:34.308 [INFO] TestAgent: consul server down +49:34.308 [INFO] TestAgent: shutdown complete +49:34.308 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23925 network=tcp +49:34.308 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23925 network=udp +49:34.308 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23912 network=tcp +49:34.808 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.808 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_allow (0.72s) +=== CONT TestAgentConnectCARoots_list +2022-12-04T05:49:33.919Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:33.919Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.946Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:33.946Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:33.947 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:687f61ba-bb7d-f9d2-86ce-1ff6690fa237 Address:127.0.0.1:23343}]" +49:33.947 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23343 [Follower]" leader= +49:33.948 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-687f61ba-bb7d-f9d2-86ce-1ff6690fa237.dc1 127.0.0.1 +49:33.949 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-687f61ba-bb7d-f9d2-86ce-1ff6690fa237 127.0.0.1 +2022-12-04T05:49:33.949Z [INFO] agent.router: Initializing LAN area manager +49:33.949 [INFO] TestAgent.server: Adding LAN server: server="Node-687f61ba-bb7d-f9d2-86ce-1ff6690fa237 (Addr: tcp/127.0.0.1:23343) (DC: dc1)" +49:33.949 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23415 network=tcp +49:33.949 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-687f61ba-bb7d-f9d2-86ce-1ff6690fa237.dc1 area=wan +49:33.949 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23415 network=udp +49:33.950 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23339 network=tcp +49:33.950 [INFO] TestAgent: started state syncer +49:33.950 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23861 network=tcp +49:33.990 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.990 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23343 [Candidate]" term=2 +49:33.991 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.991 [DEBUG] TestAgent.server.raft: vote granted: from=687f61ba-bb7d-f9d2-86ce-1ff6690fa237 term=2 tally=1 +49:33.991 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.991 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23343 [Leader]" +49:33.991 [INFO] TestAgent.server: cluster leadership acquired +49:33.991 [INFO] TestAgent.server: initializing acls +49:33.991 [INFO] TestAgent.server: New leader elected: payload=Node-687f61ba-bb7d-f9d2-86ce-1ff6690fa237 +49:33.992 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:33.992 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:33.993 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:33.993 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:33.993 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:33.993 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:33.994 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-687f61ba-bb7d-f9d2-86ce-1ff6690fa237 +49:33.994 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-687f61ba-bb7d-f9d2-86ce-1ff6690fa237.dc1 +49:33.995 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.995 [INFO] TestAgent.server: Updating LAN server: server="Node-687f61ba-bb7d-f9d2-86ce-1ff6690fa237 (Addr: tcp/127.0.0.1:23343) (DC: dc1)" +49:33.995 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-687f61ba-bb7d-f9d2-86ce-1ff6690fa237.dc1 area=wan +49:33.997 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.997 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.997 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.997 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.997 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.997 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-687f61ba-bb7d-f9d2-86ce-1ff6690fa237 +49:33.997 [INFO] TestAgent.server: member joined, marking health alive: member=Node-687f61ba-bb7d-f9d2-86ce-1ff6690fa237 +49:33.998 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.998 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-687f61ba-bb7d-f9d2-86ce-1ff6690fa237 +49:34.317 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-687f61ba-bb7d-f9d2-86ce-1ff6690fa237 +49:34.317 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-687f61ba-bb7d-f9d2-86ce-1ff6690fa237 +49:34.319 [INFO] TestAgent: Requesting shutdown +49:34.319 [INFO] TestAgent.server: shutting down server +49:34.319 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:34.319 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:34.319 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:34.319 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:34.319 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:34.319 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:34.319 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:34.323 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:34.323 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:34.323 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:34.323 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:34.323 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:34.323 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:34.323 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:34.323 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:34.324 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:34.324Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:34.324Z [INFO] agent.router.manager: shutting down +49:34.324 [INFO] TestAgent: consul server down +49:34.324 [INFO] TestAgent: shutdown complete +49:34.324 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23415 network=tcp +49:34.324 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23415 network=udp +49:34.324 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23339 network=tcp +49:34.825 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.825 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_serviceWrite (0.93s) === CONT TestAgentConnectCARoots_empty -2021-10-31T23:13:18.189Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:18.235Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:18.237 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:96995f2a-a019-7354-aaf2-d88223bee89d Address:127.0.0.1:16587}]" -13:18.237 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16587 [Follower]" leader= -13:18.254 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-96995f2a-a019-7354-aaf2-d88223bee89d.dc1 127.0.0.1 -13:18.255 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-96995f2a-a019-7354-aaf2-d88223bee89d 127.0.0.1 -2021-10-31T23:13:18.255Z [INFO] agent.router: Initializing LAN area manager -13:18.255 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16603 network=udp -13:18.255 [INFO] TestAgent.server: Adding LAN server: server="Node-96995f2a-a019-7354-aaf2-d88223bee89d (Addr: tcp/127.0.0.1:16587) (DC: dc1)" -13:18.255 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-96995f2a-a019-7354-aaf2-d88223bee89d.dc1 area=wan -13:18.255 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16603 network=tcp -13:18.256 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16604 network=tcp -13:18.256 [INFO] TestAgent: started state syncer -13:18.256 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16588 network=tcp -13:18.290 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:18.290 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16587 [Candidate]" term=2 -13:18.290 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:18.290 [DEBUG] TestAgent.server.raft: vote granted: from=96995f2a-a019-7354-aaf2-d88223bee89d term=2 tally=1 -13:18.290 [INFO] TestAgent.server.raft: election won: tally=1 -13:18.290 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16587 [Leader]" -13:18.290 [INFO] TestAgent.server: cluster leadership acquired -13:18.290 [INFO] TestAgent.server: New leader elected: payload=Node-96995f2a-a019-7354-aaf2-d88223bee89d -13:18.290 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16587 -13:18.291 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:18.297 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:18.297 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:18.297 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:18.297 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:18.297 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:18.297 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-96995f2a-a019-7354-aaf2-d88223bee89d -13:18.297 [INFO] TestAgent.server: member joined, marking health alive: member=Node-96995f2a-a019-7354-aaf2-d88223bee89d -13:18.298 [INFO] TestAgent.server: federation state anti-entropy synced -13:18.455 [INFO] TestAgent: Requesting shutdown -13:18.455 [INFO] TestAgent.server: shutting down server -13:18.455 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:18.455 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:18.455 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:18.455 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:18.455 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:18.455 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:18.455 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:18.455 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:18.455 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:18.455 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:18.455 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:18.455 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:18.456Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:18.456Z [INFO] agent.router.manager: shutting down -13:18.456 [INFO] TestAgent: consul server down -13:18.456 [INFO] TestAgent: shutdown complete -13:18.456 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16603 network=tcp -13:18.456 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16603 network=udp -13:18.456 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16604 network=tcp -13:18.957 [INFO] TestAgent: Waiting for endpoints to shut down -13:18.957 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalog_GatewayServices_Ingress (0.85s) -=== CONT TestCatalogNodeServices -2021-10-31T23:13:18.262Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:18.262Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:18.335Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:18.335Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:18.337 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a58d7748-1aaa-2b83-7369-1e8c788f0f94 Address:127.0.0.1:16538}]" -13:18.337 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16538 [Follower]" leader= -13:18.350 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a58d7748-1aaa-2b83-7369-1e8c788f0f94.dc1 127.0.0.1 -13:18.351 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a58d7748-1aaa-2b83-7369-1e8c788f0f94 127.0.0.1 -2021-10-31T23:13:18.351Z [INFO] agent.router: Initializing LAN area manager -13:18.351 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16589 network=udp -13:18.351 [INFO] TestAgent.server: Adding LAN server: server="Node-a58d7748-1aaa-2b83-7369-1e8c788f0f94 (Addr: tcp/127.0.0.1:16538) (DC: dc1)" -13:18.351 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a58d7748-1aaa-2b83-7369-1e8c788f0f94.dc1 area=wan -13:18.351 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16589 network=tcp -13:18.352 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16590 network=tcp -13:18.352 [INFO] TestAgent: started state syncer -13:18.352 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16539 network=tcp -13:18.401 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:18.406 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:18.406 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16538 [Candidate]" term=2 -13:18.406 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:18.406 [DEBUG] TestAgent.server.raft: vote granted: from=a58d7748-1aaa-2b83-7369-1e8c788f0f94 term=2 tally=1 -13:18.406 [INFO] TestAgent.server.raft: election won: tally=1 -13:18.406 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16538 [Leader]" -13:18.408 [INFO] TestAgent.server: cluster leadership acquired -13:18.408 [INFO] TestAgent.server: initializing acls -13:18.408 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:18.408 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:18.408 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:18.409 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:18.409 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:18.409 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:18.409 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a58d7748-1aaa-2b83-7369-1e8c788f0f94 -13:18.409 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a58d7748-1aaa-2b83-7369-1e8c788f0f94.dc1 -13:18.409 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a58d7748-1aaa-2b83-7369-1e8c788f0f94.dc1 area=wan -13:18.410 [INFO] TestAgent.server: New leader elected: payload=Node-a58d7748-1aaa-2b83-7369-1e8c788f0f94 -13:18.410 [INFO] TestAgent.server: Updating LAN server: server="Node-a58d7748-1aaa-2b83-7369-1e8c788f0f94 (Addr: tcp/127.0.0.1:16538) (DC: dc1)" -13:18.410 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:18.411 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:18.411 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:18.411 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:18.411 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:18.411 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:18.411 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a58d7748-1aaa-2b83-7369-1e8c788f0f94 -13:18.411 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a58d7748-1aaa-2b83-7369-1e8c788f0f94 -13:18.412 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a58d7748-1aaa-2b83-7369-1e8c788f0f94 -13:18.412 [INFO] TestAgent.server: federation state anti-entropy synced -13:18.439 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:18.440 [INFO] TestAgent: Synced node info -13:18.487 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a58d7748-1aaa-2b83-7369-1e8c788f0f94 -13:18.487 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a58d7748-1aaa-2b83-7369-1e8c788f0f94 -13:18.515 [DEBUG] TestAgent: Node info in sync -13:18.515 [INFO] TestAgent: Synced service: service=test-id -13:18.515 [DEBUG] TestAgent: Check in sync: check=service:test-id -13:18.523 [INFO] TestAgent: Requesting shutdown -13:18.523 [INFO] TestAgent.server: shutting down server -13:18.523 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:18.523 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:18.523 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:18.523 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:18.523 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:18.523 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:18.523 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:18.523 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:18.523 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:18.523 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:18.523 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:18.523 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:18.523 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:18.523 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:18.523 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:18.523Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:18.523Z [INFO] agent.router.manager: shutting down -13:18.524 [INFO] TestAgent: consul server down -13:18.524 [INFO] TestAgent: shutdown complete -13:18.524 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16589 network=tcp -13:18.524 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16589 network=udp -13:18.524 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16590 network=tcp -13:19.024 [INFO] TestAgent: Waiting for endpoints to shut down -13:19.024 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectCALeafCert_aclServiceWrite (0.87s) +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=false#01 +2022-12-04T05:49:33.917Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.945Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:33.947 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2ba1bd0a-cffc-346b-2444-7771f599375d Address:127.0.0.1:23986}]" +49:33.947 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23986 [Follower]" leader= +49:33.948 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2ba1bd0a-cffc-346b-2444-7771f599375d.dc1 127.0.0.1 +49:33.949 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2ba1bd0a-cffc-346b-2444-7771f599375d 127.0.0.1 +2022-12-04T05:49:33.949Z [INFO] agent.router: Initializing LAN area manager +49:33.949 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23876 network=udp +49:33.949 [INFO] TestAgent.server: Adding LAN server: server="Node-2ba1bd0a-cffc-346b-2444-7771f599375d (Addr: tcp/127.0.0.1:23986) (DC: dc1)" +49:33.949 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2ba1bd0a-cffc-346b-2444-7771f599375d.dc1 area=wan +49:33.949 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23876 network=tcp +49:33.950 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23982 network=tcp +49:33.950 [INFO] TestAgent: started state syncer +49:33.950 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23994 network=tcp +49:34.001 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.001 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23986 [Candidate]" term=2 +49:34.002 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.002 [DEBUG] TestAgent.server.raft: vote granted: from=2ba1bd0a-cffc-346b-2444-7771f599375d term=2 tally=1 +49:34.002 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.002 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23986 [Leader]" +49:34.002 [INFO] TestAgent.server: cluster leadership acquired +49:34.002 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23986 +49:34.003 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.004 [INFO] TestAgent.server: New leader elected: payload=Node-2ba1bd0a-cffc-346b-2444-7771f599375d +49:34.004 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.004 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.004 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.004 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.004 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.004 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2ba1bd0a-cffc-346b-2444-7771f599375d +49:34.004 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2ba1bd0a-cffc-346b-2444-7771f599375d +49:34.005 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.274 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.275 [INFO] TestAgent: Synced node info +49:34.275 [DEBUG] TestAgent: Node info in sync +49:34.332 [INFO] TestAgent: Requesting shutdown +49:34.333 [INFO] TestAgent.server: shutting down server +49:34.333 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:34.333 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:34.333 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:34.333 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:34.333 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:34.333 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:34.333 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:34.333 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:34.333 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:34.333 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:34.333 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:34.333Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:34.333Z [INFO] agent.router.manager: shutting down +49:34.336 [INFO] TestAgent: consul server down +49:34.336 [INFO] TestAgent: shutdown complete +49:34.336 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23876 network=tcp +49:34.346 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23876 network=udp +49:34.346 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23982 network=tcp +49:34.846 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.846 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_denyWildcard (0.95s) === CONT TestAgent_Token -2021-10-31T23:13:18.158Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:18.158Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:18.182Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:18.182Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:18.184 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6e328b40-490c-b34f-91de-dc089ca308e0 Address:127.0.0.1:16601}]" -13:18.184 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6e328b40-490c-b34f-91de-dc089ca308e0.dc1 127.0.0.1 -13:18.185 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16601 [Follower]" leader= -13:18.185 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6e328b40-490c-b34f-91de-dc089ca308e0 127.0.0.1 -2021-10-31T23:13:18.185Z [INFO] agent.router: Initializing LAN area manager -13:18.185 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6e328b40-490c-b34f-91de-dc089ca308e0.dc1 area=wan -13:18.186 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16596 network=tcp -13:18.198 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16596 network=udp -13:18.199 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16597 network=tcp -13:18.199 [INFO] TestAgent: started state syncer -13:18.199 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16602 network=tcp -13:18.207 [INFO] TestAgent.server: Adding LAN server: server="Node-6e328b40-490c-b34f-91de-dc089ca308e0 (Addr: tcp/127.0.0.1:16601) (DC: dc1)" -13:18.238 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:18.246 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:18.246 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16601 [Candidate]" term=2 -13:18.246 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:18.246 [DEBUG] TestAgent.server.raft: vote granted: from=6e328b40-490c-b34f-91de-dc089ca308e0 term=2 tally=1 -13:18.246 [INFO] TestAgent.server.raft: election won: tally=1 -13:18.246 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16601 [Leader]" -13:18.246 [INFO] TestAgent.server: cluster leadership acquired -13:18.247 [INFO] TestAgent.server: initializing acls -13:18.247 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:18.247 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:18.247 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:18.247 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:18.247 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:18.248 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:18.248 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-6e328b40-490c-b34f-91de-dc089ca308e0 -13:18.248 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-6e328b40-490c-b34f-91de-dc089ca308e0.dc1 -13:18.248 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:18.250 [INFO] TestAgent.server: New leader elected: payload=Node-6e328b40-490c-b34f-91de-dc089ca308e0 -13:18.250 [INFO] TestAgent.server: Updating LAN server: server="Node-6e328b40-490c-b34f-91de-dc089ca308e0 (Addr: tcp/127.0.0.1:16601) (DC: dc1)" -13:18.250 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-6e328b40-490c-b34f-91de-dc089ca308e0.dc1 area=wan -13:18.262 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:18.262 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:18.262 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:18.262 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:18.262 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:18.262 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6e328b40-490c-b34f-91de-dc089ca308e0 -13:18.262 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6e328b40-490c-b34f-91de-dc089ca308e0 -13:18.263 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6e328b40-490c-b34f-91de-dc089ca308e0 -13:18.263 [INFO] TestAgent.server: federation state anti-entropy synced -13:18.266 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:18.267 [INFO] TestAgent: Synced node info -13:18.590 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-6e328b40-490c-b34f-91de-dc089ca308e0 -13:18.590 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-6e328b40-490c-b34f-91de-dc089ca308e0 -13:18.591 [DEBUG] TestAgent: Node info in sync -13:18.591 [INFO] TestAgent: Synced service: service=test-id -13:18.591 [DEBUG] TestAgent: Check in sync: check=service:test-id -13:18.592 [INFO] TestAgent: Requesting shutdown -13:18.592 [INFO] TestAgent.server: shutting down server -13:18.592 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:18.592 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:18.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:18.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:18.592 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:18.592 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:18.592 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:18.593 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:18.593 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:18.593 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:18.593 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:18.593 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:18.593 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:18.593 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:18.593 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:18.593Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:18.593Z [INFO] agent.router.manager: shutting down -13:18.593 [INFO] TestAgent: consul server down -13:18.593 [INFO] TestAgent: shutdown complete -13:18.593 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16596 network=tcp -13:18.593 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16596 network=udp -13:18.593 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16597 network=tcp -13:19.102 [INFO] TestAgent: Waiting for endpoints to shut down -13:19.102 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectCALeafCert_aclServiceReadDeny (1.05s) -=== CONT TestCatalogConnectServiceNodes_good -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:17.680Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.786Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.787 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c5bf4c67-30a8-4768-0b81-dc35e10bfd23 Address:127.0.0.1:16510}]" -13:17.788 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c5bf4c67-30a8-4768-0b81-dc35e10bfd23.dc1 127.0.0.1 -13:17.789 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c5bf4c67-30a8-4768-0b81-dc35e10bfd23 127.0.0.1 -2021-10-31T23:13:17.789Z [INFO] agent.router: Initializing LAN area manager -13:17.789 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16519 network=udp -13:17.789 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16510 [Follower]" leader= -13:17.789 [INFO] TestAgent.server: Adding LAN server: server="Node-c5bf4c67-30a8-4768-0b81-dc35e10bfd23 (Addr: tcp/127.0.0.1:16510) (DC: dc1)" -13:17.789 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c5bf4c67-30a8-4768-0b81-dc35e10bfd23.dc1 area=wan -13:17.790 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16519 network=tcp -13:17.790 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16520 network=tcp -13:17.790 [INFO] TestAgent: started state syncer -13:17.790 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16511 network=tcp -13:17.854 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:17.854 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16510 [Candidate]" term=2 -13:17.854 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:17.854 [DEBUG] TestAgent.server.raft: vote granted: from=c5bf4c67-30a8-4768-0b81-dc35e10bfd23 term=2 tally=1 -13:17.854 [INFO] TestAgent.server.raft: election won: tally=1 -13:17.854 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16510 [Leader]" -13:17.854 [INFO] TestAgent.server: cluster leadership acquired -13:17.854 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16510 -13:17.855 [INFO] TestAgent.server: New leader elected: payload=Node-c5bf4c67-30a8-4768-0b81-dc35e10bfd23 -13:17.855 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:17.857 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:17.857 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:17.857 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:17.857 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:17.857 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:17.857 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c5bf4c67-30a8-4768-0b81-dc35e10bfd23 -13:17.857 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c5bf4c67-30a8-4768-0b81-dc35e10bfd23 -13:17.858 [INFO] TestAgent.server: federation state anti-entropy synced -13:18.156 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:18.157 [INFO] TestAgent: Synced node info -13:18.157 [DEBUG] TestAgent: Node info in sync -13:18.190 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:38826 -13:18.191 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8 127.0.0.1 -13:18.191 [INFO] TestAgent.server: Adding LAN server: server="Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8 (Addr: tcp/127.0.0.1:16566) (DC: dc1)" -13:18.191 [INFO] TestAgent.server: New leader elected: payload=Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8 -13:18.191 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8 other=Node-c5bf4c67-30a8-4768-0b81-dc35e10bfd23 -13:18.191 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8 -13:18.192 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8.dc1 127.0.0.1:16565 -13:18.192 [INFO] TestAgent: Requesting shutdown -13:18.192 [INFO] TestAgent.server: shutting down server -13:18.192 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:18.192 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:18.192 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:18.192 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:18.192 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:18.193 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8.dc1 127.0.0.1 -13:18.193 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:18.193 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:18.193 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8.dc1 address=127.0.0.1:16565 -13:18.193 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:18.193 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:18.193 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8.dc1 area=wan -13:18.193 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:18.193 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:18.193Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:18.193Z [INFO] agent.router.manager: shutting down -13:18.206 [INFO] TestAgent: consul server down -13:18.206 [INFO] TestAgent: shutdown complete -13:18.206 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16519 network=tcp -13:18.206 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16519 network=udp -13:18.206 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16520 network=tcp -13:18.706 [INFO] TestAgent: Waiting for endpoints to shut down -13:18.706 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:17.470Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.504Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.511 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3e113312-fe7d-33ea-3541-9b0dff7de7a8 Address:127.0.0.1:16566}]" -13:17.511 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8.dc1 127.0.0.1 -13:17.512 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8 127.0.0.1 -2021-10-31T23:13:17.512Z [INFO] agent.router: Initializing LAN area manager -13:17.512 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16540 network=udp -13:17.512 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16566 [Follower]" leader= -13:17.512 [INFO] TestAgent.server: Adding LAN server: server="Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8 (Addr: tcp/127.0.0.1:16566) (DC: dc1)" -13:17.512 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8.dc1 area=wan -13:17.512 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16540 network=tcp -13:17.513 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16541 network=tcp -13:17.513 [INFO] TestAgent: started state syncer -13:17.513 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16567 network=tcp -13:17.577 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:17.577 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16566 [Candidate]" term=2 -13:17.578 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:17.578 [DEBUG] TestAgent.server.raft: vote granted: from=3e113312-fe7d-33ea-3541-9b0dff7de7a8 term=2 tally=1 -13:17.578 [INFO] TestAgent.server.raft: election won: tally=1 -13:17.578 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16566 [Leader]" -13:17.578 [INFO] TestAgent.server: cluster leadership acquired -13:17.578 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16566 -13:17.578 [INFO] TestAgent.server: New leader elected: payload=Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8 -13:17.579 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:17.581 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:17.581 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:17.581 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:17.581 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:17.581 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:17.581 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8 -13:17.581 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8 -13:17.585 [INFO] TestAgent.server: federation state anti-entropy synced -13:17.698 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:17.699 [INFO] TestAgent: Synced node info -13:17.699 [DEBUG] TestAgent: Node info in sync -13:18.154 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:18.154 [DEBUG] TestAgent: Node info in sync -13:18.190 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:16508] -13:18.190 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:16508 -13:18.192 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c5bf4c67-30a8-4768-0b81-dc35e10bfd23 127.0.0.1 -13:18.192 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -13:18.192 [DEBUG] TestAgent: systemd notify failed: error="No socket" -13:18.192 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:41070 -13:18.192 [INFO] TestAgent.server: Adding LAN server: server="Node-c5bf4c67-30a8-4768-0b81-dc35e10bfd23 (Addr: tcp/127.0.0.1:16510) (DC: dc1)" -13:18.192 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-c5bf4c67-30a8-4768-0b81-dc35e10bfd23 other=Node-3e113312-fe7d-33ea-3541-9b0dff7de7a8 -13:18.192 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c5bf4c67-30a8-4768-0b81-dc35e10bfd23 -13:18.193 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c5bf4c67-30a8-4768-0b81-dc35e10bfd23.dc1 127.0.0.1 -13:18.193 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c5bf4c67-30a8-4768-0b81-dc35e10bfd23.dc1 area=wan -13:18.706 [INFO] TestAgent: Requesting shutdown -13:18.714 [INFO] TestAgent.server: shutting down server -13:18.714 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:18.714 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:18.714 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:18.714 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:18.714 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:18.714 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:18.714 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:18.714 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:18.714 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:18.714 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:18.714 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:18.714Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:18.714Z [INFO] agent.router.manager: shutting down -13:18.714 [INFO] TestAgent: consul server down -13:18.714 [INFO] TestAgent: shutdown complete -13:18.714 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16540 network=tcp -13:18.714 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16540 network=udp -13:18.714 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16541 network=tcp -13:19.022 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-c5bf4c67-30a8-4768-0b81-dc35e10bfd23 (timeout reached) -13:19.217 [INFO] TestAgent: Waiting for endpoints to shut down -13:19.218 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Join (1.81s) +2022-12-04T05:49:34.263Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.285Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.287 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:30a7bae5-17a0-5283-0f03-885fd0ad314c Address:127.0.0.1:23867}]" +49:34.287 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-30a7bae5-17a0-5283-0f03-885fd0ad314c.dc1 127.0.0.1 +49:34.288 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23867 [Follower]" leader= +49:34.288 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-30a7bae5-17a0-5283-0f03-885fd0ad314c 127.0.0.1 +2022-12-04T05:49:34.288Z [INFO] agent.router: Initializing LAN area manager +49:34.289 [INFO] TestAgent.server: Adding LAN server: server="Node-30a7bae5-17a0-5283-0f03-885fd0ad314c (Addr: tcp/127.0.0.1:23867) (DC: dc1)" +49:34.289 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23855 network=udp +49:34.289 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-30a7bae5-17a0-5283-0f03-885fd0ad314c.dc1 area=wan +49:34.289 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23855 network=tcp +49:34.290 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23863 network=tcp +49:34.290 [INFO] TestAgent: started state syncer +49:34.290 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23987 network=tcp +49:34.333 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.333 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23867 [Candidate]" term=2 +49:34.333 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.333 [DEBUG] TestAgent.server.raft: vote granted: from=30a7bae5-17a0-5283-0f03-885fd0ad314c term=2 tally=1 +49:34.333 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.333 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23867 [Leader]" +49:34.333 [INFO] TestAgent.server: cluster leadership acquired +49:34.333 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23867 +49:34.333 [INFO] TestAgent.server: New leader elected: payload=Node-30a7bae5-17a0-5283-0f03-885fd0ad314c +49:34.337 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.350 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.350 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.350 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.350 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.350 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.350 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-30a7bae5-17a0-5283-0f03-885fd0ad314c +49:34.350 [INFO] TestAgent.server: member joined, marking health alive: member=Node-30a7bae5-17a0-5283-0f03-885fd0ad314c +49:34.350 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.360 [INFO] TestAgent: Requesting shutdown +49:34.360 [INFO] TestAgent.server: shutting down server +49:34.360 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:34.360 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:34.360 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:34.360 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:34.360 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:34.360 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:34.360 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:34.360 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:34.360 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:34.360 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:34.360 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:34.360 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:34.361Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:34.361Z [INFO] agent.router.manager: shutting down +49:34.361 [INFO] TestAgent: consul server down +49:34.361 [INFO] TestAgent: shutdown complete +49:34.361 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23855 network=tcp +49:34.361 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23855 network=udp +49:34.361 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23863 network=tcp +49:34.861 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.861 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_idNotService (0.62s) === CONT TestAgent_TokenTriggersFullSync === RUN TestAgent_TokenTriggersFullSync/acl_agent_token -2021-10-31T23:13:18.272Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:18.301Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:18.307 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1debc4fc-74a4-4917-946a-c11446e38787 Address:127.0.0.1:16559}]" -13:18.307 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16559 [Follower]" leader= -13:18.308 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1debc4fc-74a4-4917-946a-c11446e38787.dc1 127.0.0.1 -13:18.308 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1debc4fc-74a4-4917-946a-c11446e38787 127.0.0.1 -2021-10-31T23:13:18.308Z [INFO] agent.router: Initializing LAN area manager -13:18.308 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16526 network=udp -13:18.308 [INFO] TestAgent.server: Adding LAN server: server="Node-1debc4fc-74a4-4917-946a-c11446e38787 (Addr: tcp/127.0.0.1:16559) (DC: dc1)" -13:18.309 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1debc4fc-74a4-4917-946a-c11446e38787.dc1 area=wan -13:18.309 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16526 network=tcp -13:18.310 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16527 network=tcp -13:18.310 [INFO] TestAgent: started state syncer -13:18.310 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16560 network=tcp -13:18.349 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:18.349 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16559 [Candidate]" term=2 -13:18.350 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:18.350 [DEBUG] TestAgent.server.raft: vote granted: from=1debc4fc-74a4-4917-946a-c11446e38787 term=2 tally=1 -13:18.350 [INFO] TestAgent.server.raft: election won: tally=1 -13:18.350 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16559 [Leader]" -13:18.350 [INFO] TestAgent.server: cluster leadership acquired -13:18.350 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16559 -13:18.351 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:18.352 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:18.352 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:18.352 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:18.352 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:18.352 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:18.352 [INFO] TestAgent.server: New leader elected: payload=Node-1debc4fc-74a4-4917-946a-c11446e38787 -13:18.352 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1debc4fc-74a4-4917-946a-c11446e38787 -13:18.352 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1debc4fc-74a4-4917-946a-c11446e38787 -13:18.353 [INFO] TestAgent.server: federation state anti-entropy synced -13:18.527 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:18.527 [INFO] TestAgent: Synced node info -13:18.527 [DEBUG] TestAgent: Node info in sync -13:18.756 [INFO] TestAgent: Requesting shutdown -13:18.756 [INFO] TestAgent.server: shutting down server -13:18.756 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:18.756 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:18.756 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:18.756 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:18.756 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:18.756 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:18.756 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:18.756 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:18.756 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:18.756 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:18.757 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:18.757Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:18.757Z [INFO] agent.router.manager: shutting down -13:18.757 [INFO] TestAgent: consul server down -13:18.757 [INFO] TestAgent: shutdown complete -13:18.757 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16526 network=tcp -13:18.757 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16526 network=udp -13:18.757 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16527 network=tcp -13:19.257 [INFO] TestAgent: Waiting for endpoints to shut down -13:19.257 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalog_GatewayServices_Terminating (1.05s) -=== CONT TestCatalogServiceNodes_ConnectProxy -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:18.522Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:18.578Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:18.581 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ac2c657d-9c21-2482-1423-6074ca8e7f43 Address:127.0.0.1:16635}]" -13:18.581 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ac2c657d-9c21-2482-1423-6074ca8e7f43.dc1 127.0.0.1 -13:18.581 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16635 [Follower]" leader= -13:18.606 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ac2c657d-9c21-2482-1423-6074ca8e7f43 127.0.0.1 -2021-10-31T23:13:18.606Z [INFO] agent.router: Initializing LAN area manager -13:18.606 [INFO] TestAgent.server: Adding LAN server: server="Node-ac2c657d-9c21-2482-1423-6074ca8e7f43 (Addr: tcp/127.0.0.1:16635) (DC: dc1)" -13:18.606 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ac2c657d-9c21-2482-1423-6074ca8e7f43.dc1 area=wan -13:18.606 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16610 network=udp -13:18.606 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16610 network=tcp -13:18.608 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16611 network=tcp -13:18.608 [INFO] TestAgent: started state syncer -13:18.608 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16636 network=tcp -13:18.645 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:18.645 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16635 [Candidate]" term=2 -13:18.646 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:18.646 [DEBUG] TestAgent.server.raft: vote granted: from=ac2c657d-9c21-2482-1423-6074ca8e7f43 term=2 tally=1 -13:18.646 [INFO] TestAgent.server.raft: election won: tally=1 -13:18.646 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16635 [Leader]" -13:18.646 [INFO] TestAgent.server: cluster leadership acquired -13:18.646 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16635 -13:18.647 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:18.647 [INFO] TestAgent.server: New leader elected: payload=Node-ac2c657d-9c21-2482-1423-6074ca8e7f43 -13:18.648 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:18.648 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:18.648 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:18.648 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:18.648 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:18.649 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ac2c657d-9c21-2482-1423-6074ca8e7f43 -13:18.649 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ac2c657d-9c21-2482-1423-6074ca8e7f43 -13:18.649 [INFO] TestAgent.server: federation state anti-entropy synced -13:18.762 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:18.762 [INFO] TestAgent: Synced node info -13:18.884 [INFO] TestAgent: Requesting shutdown -13:18.884 [INFO] TestAgent.server: shutting down server -13:18.884 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:18.884 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:18.884 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:18.884 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:18.884 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:18.884 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:18.884 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:18.884 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:18.884 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:18.884 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:18.884 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:18.884Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:18.884Z [INFO] agent.router.manager: shutting down -13:18.884 [INFO] TestAgent: consul server down -13:18.884 [INFO] TestAgent: shutdown complete -13:18.884 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16610 network=tcp -13:18.884 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16610 network=udp -13:18.884 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16611 network=tcp -13:19.403 [INFO] TestAgent: Waiting for endpoints to shut down -13:19.404 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodeServices_ConnectProxy (0.99s) -=== CONT TestCatalogServiceNodes_DistanceSort -2021-10-31T23:13:18.402Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:18.402Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:18.483Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:18.483Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:18.526 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7cac3036-fe42-2e42-83c4-e24f3e81020e Address:127.0.0.1:16608}]" -13:18.527 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7cac3036-fe42-2e42-83c4-e24f3e81020e.dc1 127.0.0.1 -13:18.527 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7cac3036-fe42-2e42-83c4-e24f3e81020e 127.0.0.1 -2021-10-31T23:13:18.527Z [INFO] agent.router: Initializing LAN area manager -13:18.528 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16608 [Follower]" leader= -13:18.528 [INFO] TestAgent.server: Adding LAN server: server="Node-7cac3036-fe42-2e42-83c4-e24f3e81020e (Addr: tcp/127.0.0.1:16608) (DC: dc1)" -13:18.528 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7cac3036-fe42-2e42-83c4-e24f3e81020e.dc1 area=wan -13:18.528 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16575 network=tcp -13:18.528 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16575 network=udp -13:18.529 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16576 network=tcp -13:18.529 [INFO] TestAgent: started state syncer -13:18.529 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16609 network=tcp -13:18.581 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:18.581 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16608 [Candidate]" term=2 -13:18.582 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:18.582 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:18.582 [DEBUG] TestAgent.server.raft: vote granted: from=7cac3036-fe42-2e42-83c4-e24f3e81020e term=2 tally=1 -13:18.582 [INFO] TestAgent.server.raft: election won: tally=1 -13:18.582 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16608 [Leader]" -13:18.582 [INFO] TestAgent.server: cluster leadership acquired -13:18.582 [INFO] TestAgent.server: initializing acls -13:18.582 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:18.582 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:18.583 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:18.583 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:18.583 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:18.583 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:18.583 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7cac3036-fe42-2e42-83c4-e24f3e81020e -13:18.583 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7cac3036-fe42-2e42-83c4-e24f3e81020e.dc1 -13:18.584 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:18.585 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:18.585 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:18.585 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:18.585 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:18.585 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:18.585 [INFO] TestAgent.server: New leader elected: payload=Node-7cac3036-fe42-2e42-83c4-e24f3e81020e -13:18.585 [INFO] TestAgent.server: Updating LAN server: server="Node-7cac3036-fe42-2e42-83c4-e24f3e81020e (Addr: tcp/127.0.0.1:16608) (DC: dc1)" -13:18.585 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7cac3036-fe42-2e42-83c4-e24f3e81020e.dc1 area=wan -13:18.586 [INFO] TestAgent.server: federation state anti-entropy synced -13:18.605 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7cac3036-fe42-2e42-83c4-e24f3e81020e -13:18.606 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7cac3036-fe42-2e42-83c4-e24f3e81020e -13:18.606 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7cac3036-fe42-2e42-83c4-e24f3e81020e -13:18.763 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:18.764 [INFO] TestAgent: Synced node info -13:18.883 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7cac3036-fe42-2e42-83c4-e24f3e81020e -13:18.883 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7cac3036-fe42-2e42-83c4-e24f3e81020e -13:18.884 [DEBUG] TestAgent: Node info in sync -13:18.885 [INFO] TestAgent: Synced service: service=test-id -13:18.885 [DEBUG] TestAgent: Check in sync: check=service:test-id -13:18.887 [INFO] TestAgent: Requesting shutdown -13:18.887 [INFO] TestAgent.server: shutting down server -13:18.887 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:18.887 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:18.887 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:18.887 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:18.887 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:18.887 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:18.887 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:18.887 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:18.887 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:18.887 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:18.887 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:18.887 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:18.887 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:18.887 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:18.887 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:18.887Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:18.887Z [INFO] agent.router.manager: shutting down -13:18.887 [INFO] TestAgent: consul server down -13:18.887 [INFO] TestAgent: shutdown complete -13:18.887 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16575 network=tcp -13:18.887 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16575 network=udp -13:18.887 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16576 network=tcp -13:19.409 [INFO] TestAgent: Waiting for endpoints to shut down -13:19.410 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectCALeafCert_aclDefaultDeny (1.05s) +2022-12-04T05:49:34.343Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.357Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.359 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:26deb142-450c-cea4-3779-6d41f660ad8e Address:127.0.0.1:23860}]" +49:34.359 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-26deb142-450c-cea4-3779-6d41f660ad8e.dc1 127.0.0.1 +49:34.360 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23860 [Follower]" leader= +49:34.360 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-26deb142-450c-cea4-3779-6d41f660ad8e 127.0.0.1 +2022-12-04T05:49:34.360Z [INFO] agent.router: Initializing LAN area manager +49:34.360 [INFO] TestAgent.server: Adding LAN server: server="Node-26deb142-450c-cea4-3779-6d41f660ad8e (Addr: tcp/127.0.0.1:23860) (DC: dc1)" +49:34.360 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23974 network=udp +49:34.360 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-26deb142-450c-cea4-3779-6d41f660ad8e.dc1 area=wan +49:34.361 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23974 network=tcp +49:34.361 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23856 network=tcp +49:34.361 [INFO] TestAgent: started state syncer +49:34.361 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23980 network=tcp +49:34.404 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.404 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23860 [Candidate]" term=2 +49:34.404 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.404 [DEBUG] TestAgent.server.raft: vote granted: from=26deb142-450c-cea4-3779-6d41f660ad8e term=2 tally=1 +49:34.404 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.404 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23860 [Leader]" +49:34.404 [INFO] TestAgent.server: cluster leadership acquired +49:34.405 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23860 +49:34.406 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.405 [INFO] TestAgent.server: New leader elected: payload=Node-26deb142-450c-cea4-3779-6d41f660ad8e +49:34.407 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.407 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.407 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.407 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.407 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.407 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-26deb142-450c-cea4-3779-6d41f660ad8e +49:34.407 [INFO] TestAgent.server: member joined, marking health alive: member=Node-26deb142-450c-cea4-3779-6d41f660ad8e +49:34.408 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.416 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.416 [INFO] TestAgent: Synced node info +49:34.416 [DEBUG] TestAgent: Node info in sync +49:34.450 [INFO] TestAgent: Requesting shutdown +49:34.450 [INFO] TestAgent.server: shutting down server +49:34.450 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:34.450 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:34.450 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:34.450 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:34.450 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:34.450 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:34.450 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:34.450 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:34.450 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:34.450 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:34.450 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:34.452Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:34.452Z [INFO] agent.router.manager: shutting down +49:34.452 [INFO] TestAgent: consul server down +49:34.452 [INFO] TestAgent: shutdown complete +49:34.452 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23974 network=tcp +49:34.452 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23974 network=udp +49:34.452 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23856 network=tcp +49:34.952 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.952 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:33.350Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.363Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:33.365 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aa45f2fb-8a41-2009-1b1d-b43c76e8835a Address:127.0.0.1:23923}]" +49:33.366 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23923 [Follower]" leader= +49:33.366 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aa45f2fb-8a41-2009-1b1d-b43c76e8835a.dc1 127.0.0.1 +49:33.367 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aa45f2fb-8a41-2009-1b1d-b43c76e8835a 127.0.0.1 +2022-12-04T05:49:33.367Z [INFO] agent.router: Initializing LAN area manager +49:33.367 [INFO] TestAgent.server: Adding LAN server: server="Node-aa45f2fb-8a41-2009-1b1d-b43c76e8835a (Addr: tcp/127.0.0.1:23923) (DC: dc1)" +49:33.367 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aa45f2fb-8a41-2009-1b1d-b43c76e8835a.dc1 area=wan +49:33.367 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23918 network=udp +49:33.367 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23918 network=tcp +49:33.368 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23919 network=tcp +49:33.368 [INFO] TestAgent: started state syncer +49:33.368 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23868 network=tcp +49:33.412 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.412 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23923 [Candidate]" term=2 +49:33.412 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.412 [DEBUG] TestAgent.server.raft: vote granted: from=aa45f2fb-8a41-2009-1b1d-b43c76e8835a term=2 tally=1 +49:33.412 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.412 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23923 [Leader]" +49:33.412 [INFO] TestAgent.server: cluster leadership acquired +49:33.413 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23923 +49:33.413 [INFO] TestAgent.server: New leader elected: payload=Node-aa45f2fb-8a41-2009-1b1d-b43c76e8835a +49:33.414 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.415 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.415 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.415 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.415 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.415 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.415 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aa45f2fb-8a41-2009-1b1d-b43c76e8835a +49:33.415 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aa45f2fb-8a41-2009-1b1d-b43c76e8835a +49:33.416 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.706 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:33.713 [INFO] TestAgent: Synced node info +49:33.798 [INFO] TestAgent: Requesting shutdown +49:33.798 [INFO] TestAgent.server: shutting down server +49:33.798 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:33.798 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:33.798 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:33.798 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:33.798 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:33.798 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:33.798 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:33.798 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:33.798 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:33.798 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:33.798 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:33.799Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:33.799Z [INFO] agent.router.manager: shutting down +49:33.799 [INFO] TestAgent: consul server down +49:33.799 [INFO] TestAgent: shutdown complete +49:33.799 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23918 network=tcp +49:33.799 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23918 network=udp +49:33.799 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23919 network=tcp +49:34.300 [INFO] TestAgent: Waiting for endpoints to shut down +49:34.300 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ReconnectConfigSettings (1.62s) === CONT TestAgent_Monitor_ACLDeny +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:34.415Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.431Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.434 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e14ec8d6-6707-a323-e474-7c05359954a9 Address:127.0.0.1:24049}]" +49:34.435 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24049 [Follower]" leader= +49:34.435 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e14ec8d6-6707-a323-e474-7c05359954a9.dc1 127.0.0.1 +49:34.436 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e14ec8d6-6707-a323-e474-7c05359954a9 127.0.0.1 +2022-12-04T05:49:34.436Z [INFO] agent.router: Initializing LAN area manager +49:34.437 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24009 network=udp +49:34.437 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e14ec8d6-6707-a323-e474-7c05359954a9.dc1 area=wan +49:34.438 [INFO] TestAgent.server: Adding LAN server: server="Node-e14ec8d6-6707-a323-e474-7c05359954a9 (Addr: tcp/127.0.0.1:24049) (DC: dc1)" +49:34.438 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24009 network=tcp +49:34.439 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24045 network=tcp +49:34.439 [INFO] TestAgent: started state syncer +49:34.439 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24036 network=tcp +49:34.488 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.489 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24049 [Candidate]" term=2 +49:34.489 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.489 [DEBUG] TestAgent.server.raft: vote granted: from=e14ec8d6-6707-a323-e474-7c05359954a9 term=2 tally=1 +49:34.489 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.489 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24049 [Leader]" +49:34.489 [INFO] TestAgent.server: cluster leadership acquired +49:34.489 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24049 +49:34.489 [INFO] TestAgent.server: New leader elected: payload=Node-e14ec8d6-6707-a323-e474-7c05359954a9 +49:34.490 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.491 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.491 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.491 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.491 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.491 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.491 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e14ec8d6-6707-a323-e474-7c05359954a9 +49:34.491 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e14ec8d6-6707-a323-e474-7c05359954a9 +49:34.492 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.508 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.508 [INFO] TestAgent: Synced node info +49:34.568 [INFO] TestAgent: Requesting shutdown +49:34.568 [INFO] TestAgent.server: shutting down server +49:34.568 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:34.568 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:34.568 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:34.568 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:34.568 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:34.568 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:34.568 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:34.568 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:34.568 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:34.568 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:34.568 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:34.568Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:34.569Z [INFO] agent.router.manager: shutting down +49:34.569 [INFO] TestAgent: consul server down +49:34.569 [INFO] TestAgent: shutdown complete +49:34.569 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24009 network=tcp +49:34.569 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24009 network=udp +49:34.569 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24045 network=tcp +49:35.072 [INFO] TestAgent: Waiting for endpoints to shut down +49:35.072 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_badBody (0.68s) +=== CONT TestAgent_Monitor +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=true#01 +2022-12-04T05:49:34.363Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.383Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.387 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7f4a568b-8118-d4cd-90bf-cca0533e943f Address:127.0.0.1:24000}]" +49:34.388 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24000 [Follower]" leader= +49:34.388 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7f4a568b-8118-d4cd-90bf-cca0533e943f.dc1 127.0.0.1 +49:34.389 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7f4a568b-8118-d4cd-90bf-cca0533e943f 127.0.0.1 +2022-12-04T05:49:34.389Z [INFO] agent.router: Initializing LAN area manager +49:34.390 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23988 network=udp +49:34.390 [INFO] TestAgent.server: Adding LAN server: server="Node-7f4a568b-8118-d4cd-90bf-cca0533e943f (Addr: tcp/127.0.0.1:24000) (DC: dc1)" +49:34.390 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7f4a568b-8118-d4cd-90bf-cca0533e943f.dc1 area=wan +49:34.390 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23988 network=tcp +49:34.395 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23996 network=tcp +49:34.395 [INFO] TestAgent: started state syncer +49:34.395 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24022 network=tcp +49:34.432 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.432 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24000 [Candidate]" term=2 +49:34.433 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.433 [DEBUG] TestAgent.server.raft: vote granted: from=7f4a568b-8118-d4cd-90bf-cca0533e943f term=2 tally=1 +49:34.433 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.433 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24000 [Leader]" +49:34.433 [INFO] TestAgent.server: cluster leadership acquired +49:34.433 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24000 +49:34.434 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.435 [INFO] TestAgent.server: New leader elected: payload=Node-7f4a568b-8118-d4cd-90bf-cca0533e943f +49:34.437 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.437 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.437 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.437 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.437 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.437 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7f4a568b-8118-d4cd-90bf-cca0533e943f +49:34.437 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7f4a568b-8118-d4cd-90bf-cca0533e943f +49:34.438 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.607 [INFO] TestAgent: Requesting shutdown +49:34.607 [INFO] TestAgent.server: shutting down server +49:34.607 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:34.607 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:34.607 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:34.607 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:34.607 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:34.607 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:34.608 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:34.608 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:34.608 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:34.608 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:34.608 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:34.608 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:34.608Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:34.608Z [INFO] agent.router.manager: shutting down +49:34.608 [INFO] TestAgent: consul server down +49:34.608 [INFO] TestAgent: shutdown complete +49:34.608 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23988 network=tcp +49:34.608 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23988 network=udp +49:34.608 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23996 network=tcp +49:35.109 [INFO] TestAgent: Waiting for endpoints to shut down +49:35.109 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_noTarget (0.76s) +=== CONT TestAgent_RegisterCheck_Service +2022-12-04T05:49:34.295Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.324Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.347 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:86099ff5-d68c-3b49-523c-0bbf8b84045e Address:127.0.0.1:24007}]" +49:34.348 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-86099ff5-d68c-3b49-523c-0bbf8b84045e.dc1 127.0.0.1 +49:34.348 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24007 [Follower]" leader= +49:34.348 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-86099ff5-d68c-3b49-523c-0bbf8b84045e 127.0.0.1 +2022-12-04T05:49:34.348Z [INFO] agent.router: Initializing LAN area manager +49:34.349 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24002 network=udp +49:34.349 [INFO] TestAgent.server: Adding LAN server: server="Node-86099ff5-d68c-3b49-523c-0bbf8b84045e (Addr: tcp/127.0.0.1:24007) (DC: dc1)" +49:34.349 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24002 network=tcp +49:34.349 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-86099ff5-d68c-3b49-523c-0bbf8b84045e.dc1 area=wan +49:34.349 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24003 network=tcp +49:34.349 [INFO] TestAgent: started state syncer +49:34.350 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23973 network=tcp +49:34.400 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.400 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24007 [Candidate]" term=2 +49:34.400 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.400 [DEBUG] TestAgent.server.raft: vote granted: from=86099ff5-d68c-3b49-523c-0bbf8b84045e term=2 tally=1 +49:34.400 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.400 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24007 [Leader]" +49:34.400 [INFO] TestAgent.server: cluster leadership acquired +49:34.401 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24007 +49:34.401 [INFO] TestAgent.server: New leader elected: payload=Node-86099ff5-d68c-3b49-523c-0bbf8b84045e +49:34.402 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.406 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.407 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.407 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.407 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.407 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.407 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-86099ff5-d68c-3b49-523c-0bbf8b84045e +49:34.407 [INFO] TestAgent.server: member joined, marking health alive: member=Node-86099ff5-d68c-3b49-523c-0bbf8b84045e +49:34.408 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.498 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.499 [INFO] TestAgent: Synced node info +49:34.499 [DEBUG] TestAgent: Node info in sync +49:34.617 [INFO] TestAgent: Requesting shutdown +49:34.618 [INFO] TestAgent.server: shutting down server +49:34.618 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:34.618 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:34.618 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:34.618 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:34.618 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:34.618 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:34.618 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:34.618 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:34.618 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:34.618 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:34.618 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:34.618Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:34.618Z [INFO] agent.router.manager: shutting down +49:34.624 [INFO] TestAgent: consul server down +49:34.624 [INFO] TestAgent: shutdown complete +49:34.624 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24002 network=tcp +49:34.624 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24002 network=udp +49:34.624 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24003 network=tcp +49:35.125 [INFO] TestAgent: Waiting for endpoints to shut down +49:35.125 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectAuthorize_idInvalidFormat (0.84s) +=== CONT TestAgent_NodeMaintenance_ACLDeny +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=false#01 +=== RUN TestAgent_Monitor/unknown_log_level +=== RUN TestAgent_Monitor/stream_unstructured_logs +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=1)_compressed=true#01 === RUN TestAgent_Token/bad_token_name === RUN TestAgent_Token/bad_JSON === RUN TestAgent_Token/set_user_legacy @@ -34957,64 +46066,6 @@ === RUN TestAgent_Token/set_repl === RUN TestAgent_Token/clear_user_legacy === RUN TestAgent_Token/clear_default -2021-10-31T23:13:18.564Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:18.615Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:18.626 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fb5c8bf4-2db3-543d-5c15-efa06ca4dc23 Address:127.0.0.1:16649}]" -13:18.626 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fb5c8bf4-2db3-543d-5c15-efa06ca4dc23.dc1 127.0.0.1 -13:18.627 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fb5c8bf4-2db3-543d-5c15-efa06ca4dc23 127.0.0.1 -2021-10-31T23:13:18.627Z [INFO] agent.router: Initializing LAN area manager -13:18.627 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16658 network=udp -13:18.627 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16649 [Follower]" leader= -13:18.627 [INFO] TestAgent.server: Adding LAN server: server="Node-fb5c8bf4-2db3-543d-5c15-efa06ca4dc23 (Addr: tcp/127.0.0.1:16649) (DC: dc1)" -13:18.627 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fb5c8bf4-2db3-543d-5c15-efa06ca4dc23.dc1 area=wan -13:18.627 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16658 network=tcp -13:18.628 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16659 network=tcp -13:18.628 [INFO] TestAgent: started state syncer -13:18.628 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16650 network=tcp -13:18.673 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:18.673 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16649 [Candidate]" term=2 -13:18.674 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:18.674 [DEBUG] TestAgent.server.raft: vote granted: from=fb5c8bf4-2db3-543d-5c15-efa06ca4dc23 term=2 tally=1 -13:18.674 [INFO] TestAgent.server.raft: election won: tally=1 -13:18.674 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16649 [Leader]" -13:18.674 [INFO] TestAgent.server: cluster leadership acquired -13:18.674 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16649 -13:18.674 [INFO] TestAgent.server: New leader elected: payload=Node-fb5c8bf4-2db3-543d-5c15-efa06ca4dc23 -13:18.675 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:18.676 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:18.676 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:18.676 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:18.676 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:18.676 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:18.677 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fb5c8bf4-2db3-543d-5c15-efa06ca4dc23 -13:18.677 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fb5c8bf4-2db3-543d-5c15-efa06ca4dc23 -13:18.678 [INFO] TestAgent.server: federation state anti-entropy synced -13:18.746 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:18.747 [INFO] TestAgent: Synced node info -13:18.923 [INFO] TestAgent: Requesting shutdown -13:18.924 [INFO] TestAgent.server: shutting down server -13:18.924 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:18.924 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:18.924 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:18.924 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:18.924 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:18.924 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:18.924 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:18.924 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:18.924 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:18.924 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:18.924 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:18.924Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:18.924Z [INFO] agent.router.manager: shutting down -13:18.924 [INFO] TestAgent: consul server down -13:18.924 [INFO] TestAgent: shutdown complete -13:18.924 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16658 network=tcp -13:18.924 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16658 network=udp -13:18.924 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16659 network=tcp -13:19.427 [INFO] TestAgent: Waiting for endpoints to shut down -13:19.427 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodeServices_Filter (0.90s) -=== CONT TestCatalogServiceNodes_WanTranslation === RUN TestAgent_Token/clear_agent_legacy === RUN TestAgent_Token/clear_agent === RUN TestAgent_Token/clear_master_legacy @@ -35022,540 +46073,455 @@ === RUN TestAgent_Token/clear_repl_legacy === RUN TestAgent_Token/clear_repl === RUN TestAgent_Token/permission_denied -2021-10-31T23:13:18.544Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:18.630Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:18.632 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e26b8b7e-a089-7970-3cb8-004fb73711ac Address:127.0.0.1:16656}]" -13:18.633 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e26b8b7e-a089-7970-3cb8-004fb73711ac.dc1 127.0.0.1 -13:18.633 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e26b8b7e-a089-7970-3cb8-004fb73711ac 127.0.0.1 -2021-10-31T23:13:18.633Z [INFO] agent.router: Initializing LAN area manager -13:18.633 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16656 [Follower]" leader= -13:18.634 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e26b8b7e-a089-7970-3cb8-004fb73711ac.dc1 area=wan -13:18.634 [INFO] TestAgent.server: Adding LAN server: server="Node-e26b8b7e-a089-7970-3cb8-004fb73711ac (Addr: tcp/127.0.0.1:16656) (DC: dc1)" -13:18.634 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16637 network=udp -13:18.634 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16637 network=tcp -13:18.634 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16624 network=tcp -13:18.635 [INFO] TestAgent: started state syncer -13:18.635 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16657 network=tcp -13:18.701 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:18.701 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16656 [Candidate]" term=2 -13:18.702 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:18.702 [DEBUG] TestAgent.server.raft: vote granted: from=e26b8b7e-a089-7970-3cb8-004fb73711ac term=2 tally=1 -13:18.702 [INFO] TestAgent.server.raft: election won: tally=1 -13:18.702 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16656 [Leader]" -13:18.702 [INFO] TestAgent.server: cluster leadership acquired -13:18.702 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16656 -13:18.704 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:18.705 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:18.705 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:18.705 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:18.705 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:18.705 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:18.705 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e26b8b7e-a089-7970-3cb8-004fb73711ac -13:18.705 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e26b8b7e-a089-7970-3cb8-004fb73711ac -13:18.705 [INFO] TestAgent.server: New leader elected: payload=Node-e26b8b7e-a089-7970-3cb8-004fb73711ac -13:18.709 [INFO] TestAgent.server: federation state anti-entropy synced -13:18.916 [DEBUG] connect.ca.consul: consul CA provider configured: id=a4:c1:90:58:a5:55:ce:48:1e:c0:3d:12:c2:95:24:e9:ec:5a:fc:66:0c:44:97:c0:9e:33:93:85:b4:e6:8f:14 is_primary=true -13:18.918 [INFO] TestAgent.server.connect: CA rotated to new root under provider: provider=consul -13:18.919 [DEBUG] connect.ca.consul: consul CA provider configured: id=22:7b:7e:05:b2:f2:20:ab:23:b3:56:35:ee:6a:c6:73:a4:ce:65:6e:01:df:92:b3:24:1f:7c:36:d4:a1:93:85 is_primary=true -13:18.921 [INFO] TestAgent.server.connect: CA rotated to new root under provider: provider=consul -13:18.947 [INFO] TestAgent: Requesting shutdown -13:18.947 [INFO] TestAgent.server: shutting down server -13:18.947 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:18.947 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:18.947 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:18.947 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:18.947 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:18.947 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:18.947 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:18.947 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:18.947 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:18.948 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:18.948 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:18.948 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:18.948Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:18.948Z [INFO] agent.router.manager: shutting down -13:18.948 [INFO] TestAgent: consul server down -13:18.948 [INFO] TestAgent: shutdown complete -13:18.948 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16637 network=tcp -13:18.948 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16637 network=udp -13:18.948 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16624 network=tcp -13:19.449 [INFO] TestAgent: Waiting for endpoints to shut down -13:19.449 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectCARoots_list (1.00s) -=== CONT TestAgent_Monitor -2021-10-31T23:13:18.639Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:18.663Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:18.665 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e9a14bbc-4ef4-a51c-9c01-f8c8f652c5e8 Address:127.0.0.1:16642}]" -13:18.665 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16642 [Follower]" leader= -13:18.678 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e9a14bbc-4ef4-a51c-9c01-f8c8f652c5e8.dc1 127.0.0.1 -13:18.679 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e9a14bbc-4ef4-a51c-9c01-f8c8f652c5e8 127.0.0.1 -2021-10-31T23:13:18.679Z [INFO] agent.router: Initializing LAN area manager -13:18.679 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16630 network=udp -13:18.679 [INFO] TestAgent.server: Adding LAN server: server="Node-e9a14bbc-4ef4-a51c-9c01-f8c8f652c5e8 (Addr: tcp/127.0.0.1:16642) (DC: dc1)" -13:18.679 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e9a14bbc-4ef4-a51c-9c01-f8c8f652c5e8.dc1 area=wan -13:18.679 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16630 network=tcp -13:18.680 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16645 network=tcp -13:18.680 [INFO] TestAgent: started state syncer -13:18.680 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16643 network=tcp -13:18.745 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:18.745 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16642 [Candidate]" term=2 -13:18.746 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:18.746 [DEBUG] TestAgent.server.raft: vote granted: from=e9a14bbc-4ef4-a51c-9c01-f8c8f652c5e8 term=2 tally=1 -13:18.746 [INFO] TestAgent.server.raft: election won: tally=1 -13:18.746 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16642 [Leader]" -13:18.746 [INFO] TestAgent.server: cluster leadership acquired -13:18.746 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16642 -13:18.749 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:18.757 [INFO] TestAgent.server: New leader elected: payload=Node-e9a14bbc-4ef4-a51c-9c01-f8c8f652c5e8 -13:18.766 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:18.766 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:18.766 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:18.766 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:18.766 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:18.766 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e9a14bbc-4ef4-a51c-9c01-f8c8f652c5e8 -13:18.766 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e9a14bbc-4ef4-a51c-9c01-f8c8f652c5e8 -13:18.767 [INFO] TestAgent.server: federation state anti-entropy synced -13:18.846 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:18.846 [INFO] TestAgent: Synced node info -13:19.007 [INFO] TestAgent: Requesting shutdown -13:19.007 [INFO] TestAgent.server: shutting down server -13:19.007 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:19.007 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:19.007 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:19.007 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:19.007 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:19.007 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:19.007 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:19.007 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:19.007 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:19.007 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:19.007 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:19.007Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:19.007Z [INFO] agent.router.manager: shutting down -13:19.007 [INFO] TestAgent: consul server down -13:19.007 [INFO] TestAgent: shutdown complete -13:19.007 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16630 network=tcp -13:19.007 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16630 network=udp -13:19.007 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16645 network=tcp -13:19.509 [INFO] TestAgent: Waiting for endpoints to shut down -13:19.509 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodeServiceList (0.91s) -=== CONT TestCatalogServiceNodes_Filter -[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:18.857Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:18.900Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:18.903 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:80657ce3-0e73-9da0-8a3b-49a7c3a2e2f7 Address:127.0.0.1:16615}]" -13:18.904 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16615 [Follower]" leader= -13:18.904 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-80657ce3-0e73-9da0-8a3b-49a7c3a2e2f7.dc1 127.0.0.1 -13:18.905 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-80657ce3-0e73-9da0-8a3b-49a7c3a2e2f7 127.0.0.1 -2021-10-31T23:13:18.905Z [INFO] agent.router: Initializing LAN area manager -13:18.905 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16644 network=udp -13:18.905 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-80657ce3-0e73-9da0-8a3b-49a7c3a2e2f7.dc1 area=wan -13:18.905 [INFO] TestAgent.server: Adding LAN server: server="Node-80657ce3-0e73-9da0-8a3b-49a7c3a2e2f7 (Addr: tcp/127.0.0.1:16615) (DC: dc1)" -13:18.906 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16644 network=tcp -13:18.906 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16652 network=tcp -13:18.906 [INFO] TestAgent: started state syncer -13:18.906 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16616 network=tcp -13:18.961 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:18.961 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16615 [Candidate]" term=2 -13:18.962 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:18.962 [DEBUG] TestAgent.server.raft: vote granted: from=80657ce3-0e73-9da0-8a3b-49a7c3a2e2f7 term=2 tally=1 -13:18.962 [INFO] TestAgent.server.raft: election won: tally=1 -13:18.962 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16615 [Leader]" -13:18.962 [INFO] TestAgent.server: cluster leadership acquired -13:18.962 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16615 -13:18.963 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:18.964 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:18.964 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:18.964 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:18.964 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:18.964 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:18.964 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-80657ce3-0e73-9da0-8a3b-49a7c3a2e2f7 -13:18.964 [INFO] TestAgent.server: member joined, marking health alive: member=Node-80657ce3-0e73-9da0-8a3b-49a7c3a2e2f7 -13:18.965 [INFO] TestAgent.server: New leader elected: payload=Node-80657ce3-0e73-9da0-8a3b-49a7c3a2e2f7 -13:18.965 [INFO] TestAgent.server: federation state anti-entropy synced -13:19.175 [INFO] TestAgent.server.connect: CA provider config updated -13:19.175 [INFO] TestAgent: Requesting shutdown -13:19.175 [INFO] TestAgent.server: shutting down server -13:19.175 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:19.175 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:19.175 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:19.175 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:19.175 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:19.175 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:19.175 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:19.175 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:19.175 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:19.175 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:19.175 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:19.175 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:19.175Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:19.176Z [INFO] agent.router.manager: shutting down -13:19.176 [INFO] TestAgent: consul server down -13:19.176 [INFO] TestAgent: shutdown complete -13:19.176 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16644 network=tcp -13:19.176 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16644 network=udp -13:19.176 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16652 network=tcp -13:19.677 [INFO] TestAgent: Waiting for endpoints to shut down -13:19.677 [INFO] TestAgent: Endpoints down -=== RUN TestConnectCAConfig/force_without_cross_sign_snake_case -2021-10-31T23:13:18.939Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:19.031Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:19.033 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:edd4672b-79c4-e817-37be-8b0e401708b1 Address:127.0.0.1:16663}]" -13:19.033 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16663 [Follower]" leader= -13:19.037 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-edd4672b-79c4-e817-37be-8b0e401708b1.dc1 127.0.0.1 -13:19.038 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-edd4672b-79c4-e817-37be-8b0e401708b1 127.0.0.1 -2021-10-31T23:13:19.038Z [INFO] agent.router: Initializing LAN area manager -13:19.039 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16619 network=udp -13:19.039 [INFO] TestAgent.server: Adding LAN server: server="Node-edd4672b-79c4-e817-37be-8b0e401708b1 (Addr: tcp/127.0.0.1:16663) (DC: dc1)" -13:19.039 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-edd4672b-79c4-e817-37be-8b0e401708b1.dc1 area=wan -13:19.039 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16619 network=tcp -13:19.040 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16631 network=tcp -13:19.040 [INFO] TestAgent: started state syncer -13:19.040 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16664 network=tcp -13:19.093 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:19.094 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16663 [Candidate]" term=2 -13:19.094 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:19.094 [DEBUG] TestAgent.server.raft: vote granted: from=edd4672b-79c4-e817-37be-8b0e401708b1 term=2 tally=1 -13:19.094 [INFO] TestAgent.server.raft: election won: tally=1 -13:19.094 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16663 [Leader]" -13:19.094 [INFO] TestAgent.server: cluster leadership acquired -13:19.095 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16663 -13:19.095 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:19.095 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:19.095 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-edd4672b-79c4-e817-37be-8b0e401708b1 -13:19.095 [INFO] TestAgent.server: member joined, marking health alive: member=Node-edd4672b-79c4-e817-37be-8b0e401708b1 -13:19.096 [INFO] TestAgent.server: New leader elected: payload=Node-edd4672b-79c4-e817-37be-8b0e401708b1 -13:19.097 [INFO] TestAgent.server: federation state anti-entropy synced -13:19.166 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:19.166 [INFO] TestAgent: Synced node info -13:19.166 [DEBUG] TestAgent: Node info in sync -13:19.279 [INFO] TestAgent: Requesting shutdown -13:19.279 [INFO] TestAgent.server: shutting down server -13:19.279 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:19.279 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:19.279 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:19.279 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:19.279 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:19.279 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:19.279 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:19.279Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:19.279Z [INFO] agent.router.manager: shutting down -13:19.280 [INFO] TestAgent: consul server down -13:19.280 [INFO] TestAgent: shutdown complete -13:19.280 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16619 network=tcp -13:19.280 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16619 network=udp -13:19.280 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16631 network=tcp -13:19.781 [INFO] TestAgent: Waiting for endpoints to shut down -13:19.781 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectCARoots_empty (0.89s) -=== CONT TestAgent_RegisterCheck_Service -=== RUN TestAgent_Monitor/unknown_log_level -=== RUN TestAgent_Monitor/stream_unstructured_logs -2021-10-31T23:13:19.004Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:19.032Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:19.034 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:05f8a3ef-8f59-d83f-d24f-3e8287aeca43 Address:127.0.0.1:16698}]" -13:19.035 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-05f8a3ef-8f59-d83f-d24f-3e8287aeca43.dc1 127.0.0.1 -13:19.036 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-05f8a3ef-8f59-d83f-d24f-3e8287aeca43 127.0.0.1 -2021-10-31T23:13:19.036Z [INFO] agent.router: Initializing LAN area manager -13:19.036 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16665 network=udp -13:19.036 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16698 [Follower]" leader= -13:19.036 [INFO] TestAgent.server: Adding LAN server: server="Node-05f8a3ef-8f59-d83f-d24f-3e8287aeca43 (Addr: tcp/127.0.0.1:16698) (DC: dc1)" -13:19.036 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-05f8a3ef-8f59-d83f-d24f-3e8287aeca43.dc1 area=wan -13:19.037 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16665 network=tcp -13:19.039 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16666 network=tcp -13:19.039 [INFO] TestAgent: started state syncer -13:19.039 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16699 network=tcp -13:19.086 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:19.086 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16698 [Candidate]" term=2 -13:19.086 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:19.086 [DEBUG] TestAgent.server.raft: vote granted: from=05f8a3ef-8f59-d83f-d24f-3e8287aeca43 term=2 tally=1 -13:19.086 [INFO] TestAgent.server.raft: election won: tally=1 -13:19.086 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16698 [Leader]" -13:19.098 [INFO] TestAgent.server: cluster leadership acquired -13:19.098 [INFO] TestAgent.server: New leader elected: payload=Node-05f8a3ef-8f59-d83f-d24f-3e8287aeca43 -13:19.098 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16698 -13:19.100 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:19.112 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:19.112 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:19.112 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:19.112 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:19.112 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:19.112 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-05f8a3ef-8f59-d83f-d24f-3e8287aeca43 -13:19.112 [INFO] TestAgent.server: member joined, marking health alive: member=Node-05f8a3ef-8f59-d83f-d24f-3e8287aeca43 -13:19.114 [INFO] TestAgent.server: federation state anti-entropy synced -13:19.307 [INFO] TestAgent: Requesting shutdown -13:19.308 [INFO] TestAgent.server: shutting down server -13:19.308 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:19.308 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:19.308 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:19.308 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:19.308 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:19.308 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:19.308 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:19.308 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:19.322 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:19.322 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:19.323Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:19.323Z [INFO] agent.router.manager: shutting down -13:19.323 [INFO] TestAgent: consul server down -13:19.323 [INFO] TestAgent: shutdown complete -13:19.323 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16665 network=tcp -13:19.323 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16665 network=udp -13:19.323 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16666 network=tcp -13:19.323 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:19.323 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:19.829 [INFO] TestAgent: Waiting for endpoints to shut down -13:19.829 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodeServices (0.87s) -=== CONT TestCatalogServiceNodes_NodeMetaFilter +=== RUN TestAgent_NodeMaintenance_ACLDeny/no_token +=== RUN TestAgent_NodeMaintenance_ACLDeny/root_token [WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestAgent_Monitor/stream_JSON_logs -2021-10-31T23:13:18.631Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:18.631Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:18.661Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:18.661Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:18.667 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:72438887-8c41-c555-f429-18ce0b43c668 Address:127.0.0.1:16628}]" -13:18.668 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-72438887-8c41-c555-f429-18ce0b43c668.dc2 127.0.0.1 -13:18.668 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-72438887-8c41-c555-f429-18ce0b43c668 127.0.0.1 -2021-10-31T23:13:18.668Z [INFO] agent.router: Initializing LAN area manager -13:18.668 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16651 network=udp -13:18.668 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16628 [Follower]" leader= -13:18.669 [INFO] TestAgent.server: Adding LAN server: server="Node-72438887-8c41-c555-f429-18ce0b43c668 (Addr: tcp/127.0.0.1:16628) (DC: dc2)" -13:18.669 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-72438887-8c41-c555-f429-18ce0b43c668.dc2 area=wan -13:18.669 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16651 network=tcp -13:18.674 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16638 network=tcp -13:18.674 [INFO] TestAgent: started state syncer -13:18.674 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16629 network=tcp -13:18.713 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:18.713 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16628 [Candidate]" term=2 -13:18.714 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:18.714 [DEBUG] TestAgent.server.raft: vote granted: from=72438887-8c41-c555-f429-18ce0b43c668 term=2 tally=1 -13:18.714 [INFO] TestAgent.server.raft: election won: tally=1 -13:18.714 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16628 [Leader]" -13:18.714 [INFO] TestAgent.server: cluster leadership acquired -13:18.714 [INFO] TestAgent.server: initializing acls -13:18.714 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:18.714 [INFO] TestAgent.server: New leader elected: payload=Node-72438887-8c41-c555-f429-18ce0b43c668 -13:18.715 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:18.715 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:18.715 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:18.715 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-72438887-8c41-c555-f429-18ce0b43c668 -13:18.715 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-72438887-8c41-c555-f429-18ce0b43c668.dc2 -13:18.715 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:18.717 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:18.717 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:18.717 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:18.717 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:18.717 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:18.717 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-72438887-8c41-c555-f429-18ce0b43c668 -13:18.717 [INFO] TestAgent.server: member joined, marking health alive: member=Node-72438887-8c41-c555-f429-18ce0b43c668 -13:18.747 [INFO] TestAgent.server: Updating LAN server: server="Node-72438887-8c41-c555-f429-18ce0b43c668 (Addr: tcp/127.0.0.1:16628) (DC: dc2)" -13:18.747 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-72438887-8c41-c555-f429-18ce0b43c668.dc2 area=wan -13:18.749 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-72438887-8c41-c555-f429-18ce0b43c668 -13:18.755 [INFO] TestAgent.server: federation state anti-entropy synced -13:18.846 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:16572] -13:18.850 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:16572 -13:18.851 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-37796cd2-74ba-b8bd-b0e6-22e407bd2742.dc1 127.0.0.1 -13:18.851 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -13:18.851 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-37796cd2-74ba-b8bd-b0e6-22e407bd2742.dc1 area=wan -13:18.882 [INFO] TestAgent: Requesting shutdown -13:18.882 [INFO] TestAgent.server: shutting down server -13:18.882 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:18.882 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:18.882 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:18.882 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:18.882 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:18.882 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:18.882 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:18.882 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:18.882 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:18.882 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:18.882 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:18.882 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:18.882 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:18.882 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:18.882 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:18.882 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:18.883Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:18.883Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:18.883Z [INFO] agent.router.manager: shutting down -13:18.883 [INFO] TestAgent: consul server down -13:18.883 [INFO] TestAgent: shutdown complete -13:18.883 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16651 network=tcp -13:18.883 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16651 network=udp -13:18.883 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16638 network=tcp -13:19.410 [INFO] TestAgent: Waiting for endpoints to shut down -13:19.410 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:18.372Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:18.372Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:18.483Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:18.483Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:18.515 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:37796cd2-74ba-b8bd-b0e6-22e407bd2742 Address:127.0.0.1:16573}]" -13:18.515 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-37796cd2-74ba-b8bd-b0e6-22e407bd2742.dc1 127.0.0.1 -13:18.516 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-37796cd2-74ba-b8bd-b0e6-22e407bd2742 127.0.0.1 -2021-10-31T23:13:18.516Z [INFO] agent.router: Initializing LAN area manager -13:18.516 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16505 network=udp -13:18.524 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16573 [Follower]" leader= -13:18.526 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-37796cd2-74ba-b8bd-b0e6-22e407bd2742.dc1 area=wan -13:18.526 [INFO] TestAgent.server: Adding LAN server: server="Node-37796cd2-74ba-b8bd-b0e6-22e407bd2742 (Addr: tcp/127.0.0.1:16573) (DC: dc1)" -13:18.526 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16505 network=tcp -13:18.527 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16506 network=tcp -13:18.527 [INFO] TestAgent: started state syncer -13:18.527 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16574 network=tcp -13:18.566 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:18.566 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16573 [Candidate]" term=2 -13:18.566 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:18.566 [DEBUG] TestAgent.server.raft: vote granted: from=37796cd2-74ba-b8bd-b0e6-22e407bd2742 term=2 tally=1 -13:18.566 [INFO] TestAgent.server.raft: election won: tally=1 -13:18.566 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16573 [Leader]" -13:18.566 [INFO] TestAgent.server: cluster leadership acquired -13:18.566 [INFO] TestAgent.server: initializing acls -13:18.566 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:18.567 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:18.567 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:18.567 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:18.567 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-37796cd2-74ba-b8bd-b0e6-22e407bd2742 -13:18.567 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-37796cd2-74ba-b8bd-b0e6-22e407bd2742.dc1 -13:18.568 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:18.568 [INFO] TestAgent.server: New leader elected: payload=Node-37796cd2-74ba-b8bd-b0e6-22e407bd2742 -13:18.568 [INFO] TestAgent.server: Updating LAN server: server="Node-37796cd2-74ba-b8bd-b0e6-22e407bd2742 (Addr: tcp/127.0.0.1:16573) (DC: dc1)" -13:18.568 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-37796cd2-74ba-b8bd-b0e6-22e407bd2742.dc1 area=wan -13:18.569 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:18.569 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:18.569 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:18.569 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:18.569 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:18.569 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-37796cd2-74ba-b8bd-b0e6-22e407bd2742 -13:18.569 [INFO] TestAgent.server: member joined, marking health alive: member=Node-37796cd2-74ba-b8bd-b0e6-22e407bd2742 -13:18.570 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-37796cd2-74ba-b8bd-b0e6-22e407bd2742 -13:18.570 [INFO] TestAgent.server: federation state anti-entropy synced -13:18.850 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:38270 -13:18.851 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-72438887-8c41-c555-f429-18ce0b43c668.dc2 127.0.0.1 -13:18.851 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-72438887-8c41-c555-f429-18ce0b43c668.dc2 area=wan -13:18.888 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:18.888 [INFO] TestAgent: Synced node info -13:19.410 [INFO] TestAgent: Requesting shutdown -13:19.410 [INFO] TestAgent.server: shutting down server -13:19.410 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:19.410 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:19.410 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:19.410 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:19.410 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:19.410 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:19.410 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:19.410 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:19.410 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:19.410 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:19.410 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:19.410 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:19.410 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:19.410 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:19.410 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:19.411Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:19.411Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:19.411Z [INFO] agent.router.manager: shutting down -13:19.411 [INFO] TestAgent: consul server down -13:19.411 [INFO] TestAgent: shutdown complete -13:19.411 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16505 network=tcp -13:19.411 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16505 network=udp -13:19.411 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16506 network=tcp -13:19.913 [INFO] TestAgent: Waiting for endpoints to shut down -13:19.913 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodeServices_WanTranslation (1.58s) -=== CONT TestAgent_NodeMaintenance_ACLDeny -2021-10-31T23:13:19.064Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:19.064Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:19.112Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:19.112Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:19.117 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:00f4e160-0fff-2c91-9b2f-c98fb42c805f Address:127.0.0.1:16251}]" -13:19.118 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16251 [Follower]" leader= -13:19.131 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-00f4e160-0fff-2c91-9b2f-c98fb42c805f.dc1 127.0.0.1 -13:19.132 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-00f4e160-0fff-2c91-9b2f-c98fb42c805f 127.0.0.1 -2021-10-31T23:13:19.132Z [INFO] agent.router: Initializing LAN area manager -13:19.132 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16700 network=udp -13:19.133 [INFO] TestAgent.server: Adding LAN server: server="Node-00f4e160-0fff-2c91-9b2f-c98fb42c805f (Addr: tcp/127.0.0.1:16251) (DC: dc1)" -13:19.133 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16700 network=tcp -13:19.165 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-00f4e160-0fff-2c91-9b2f-c98fb42c805f.dc1 area=wan -13:19.166 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16701 network=tcp -13:19.166 [INFO] TestAgent: started state syncer -13:19.166 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16252 network=tcp -13:19.181 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:19.181 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16251 [Candidate]" term=2 -13:19.182 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:19.182 [DEBUG] TestAgent.server.raft: vote granted: from=00f4e160-0fff-2c91-9b2f-c98fb42c805f term=2 tally=1 -13:19.182 [INFO] TestAgent.server.raft: election won: tally=1 -13:19.182 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16251 [Leader]" -13:19.182 [INFO] TestAgent.server: cluster leadership acquired -13:19.182 [INFO] TestAgent.server: initializing acls -13:19.182 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:19.182 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:19.183 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:19.183 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:19.183 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:19.183 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:19.183 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-00f4e160-0fff-2c91-9b2f-c98fb42c805f -13:19.183 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-00f4e160-0fff-2c91-9b2f-c98fb42c805f.dc1 -13:19.184 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:19.185 [INFO] TestAgent.server: New leader elected: payload=Node-00f4e160-0fff-2c91-9b2f-c98fb42c805f -13:19.185 [INFO] TestAgent.server: Updating LAN server: server="Node-00f4e160-0fff-2c91-9b2f-c98fb42c805f (Addr: tcp/127.0.0.1:16251) (DC: dc1)" -13:19.185 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-00f4e160-0fff-2c91-9b2f-c98fb42c805f.dc1 area=wan -13:19.186 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:19.186 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:19.186 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:19.186 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:19.186 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:19.186 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-00f4e160-0fff-2c91-9b2f-c98fb42c805f -13:19.186 [INFO] TestAgent.server: member joined, marking health alive: member=Node-00f4e160-0fff-2c91-9b2f-c98fb42c805f -13:19.187 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-00f4e160-0fff-2c91-9b2f-c98fb42c805f -13:19.187 [INFO] TestAgent.server: federation state anti-entropy synced -13:19.414 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -13:19.414 [WARN] TestAgent: Node info update blocked by ACLs: node=00f4e160-0fff-2c91-9b2f-c98fb42c805f accessorID=00000000-0000-0000-0000-000000000002 -13:19.418 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-00f4e160-0fff-2c91-9b2f-c98fb42c805f -13:19.418 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-00f4e160-0fff-2c91-9b2f-c98fb42c805f -13:19.419 [INFO] TestAgent: Updated agent's ACL token: token=acl_token -13:19.419 [INFO] TestAgent: Updated agent's ACL token: token=default -13:19.420 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_token -13:19.420 [INFO] TestAgent: Updated agent's ACL token: token=agent -13:19.420 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_master_token -13:19.420 [INFO] TestAgent: Updated agent's ACL token: token=agent_master -13:19.420 [INFO] TestAgent: Updated agent's ACL token: token=acl_replication_token -13:19.421 [INFO] TestAgent: Updated agent's ACL token: token=replication -13:19.421 [INFO] TestAgent: Updated agent's ACL token: token=acl_token -13:19.421 [INFO] TestAgent: Updated agent's ACL token: token=default -13:19.430 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_token -13:19.430 [INFO] TestAgent: Updated agent's ACL token: token=agent -13:19.430 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_master_token -13:19.431 [INFO] TestAgent: Updated agent's ACL token: token=agent_master -13:19.431 [INFO] TestAgent: Updated agent's ACL token: token=acl_replication_token -13:19.431 [INFO] TestAgent: Updated agent's ACL token: token=replication -13:19.432 [INFO] TestAgent: Requesting shutdown -13:19.432 [INFO] TestAgent.server: shutting down server -13:19.432 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:19.432 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:19.432 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:19.432 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:19.432 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:19.432 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:19.432 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:19.432 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:19.432 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:19.432 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:19.432 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:19.433 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:19.433 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:19.433 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:19.433 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:19.433Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:19.433Z [INFO] agent.router.manager: shutting down -13:19.433 [INFO] TestAgent: consul server down -13:19.433 [INFO] TestAgent: shutdown complete -13:19.433 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16700 network=tcp -13:19.438 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16700 network=udp -13:19.439 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16701 network=tcp -13:19.941 [INFO] TestAgent: Waiting for endpoints to shut down -13:19.941 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Token (0.92s) +2022-12-04T05:49:34.668Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:34.668Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.689Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:34.689Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.693 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:81c3361e-9082-c29e-4d70-02b160b7cbde Address:127.0.0.1:24021}]" +49:34.693 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24021 [Follower]" leader= +49:34.693 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-81c3361e-9082-c29e-4d70-02b160b7cbde.dc1 127.0.0.1 +49:34.694 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-81c3361e-9082-c29e-4d70-02b160b7cbde 127.0.0.1 +2022-12-04T05:49:34.694Z [INFO] agent.router: Initializing LAN area manager +49:34.694 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-81c3361e-9082-c29e-4d70-02b160b7cbde.dc1 area=wan +49:34.694 [INFO] TestAgent.server: Adding LAN server: server="Node-81c3361e-9082-c29e-4d70-02b160b7cbde (Addr: tcp/127.0.0.1:24021) (DC: dc1)" +49:34.694 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24016 network=udp +49:34.696 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24016 network=tcp +49:34.696 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24017 network=tcp +49:34.696 [INFO] TestAgent: started state syncer +49:34.697 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24078 network=tcp +49:34.738 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.738 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24021 [Candidate]" term=2 +49:34.739 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.739 [DEBUG] TestAgent.server.raft: vote granted: from=81c3361e-9082-c29e-4d70-02b160b7cbde term=2 tally=1 +49:34.739 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.739 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24021 [Leader]" +49:34.739 [INFO] TestAgent.server: cluster leadership acquired +49:34.739 [INFO] TestAgent.server: initializing acls +49:34.740 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:34.740 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:34.740 [INFO] TestAgent.server: New leader elected: payload=Node-81c3361e-9082-c29e-4d70-02b160b7cbde +49:34.745 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:34.745 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:34.745 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:34.745 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:34.745 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-81c3361e-9082-c29e-4d70-02b160b7cbde +49:34.746 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-81c3361e-9082-c29e-4d70-02b160b7cbde.dc1 +49:34.746 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-81c3361e-9082-c29e-4d70-02b160b7cbde.dc1 area=wan +49:34.746 [INFO] TestAgent.server: Updating LAN server: server="Node-81c3361e-9082-c29e-4d70-02b160b7cbde (Addr: tcp/127.0.0.1:24021) (DC: dc1)" +49:34.746 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.747 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.747 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.747 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.747 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.747 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.747 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-81c3361e-9082-c29e-4d70-02b160b7cbde +49:34.747 [INFO] TestAgent.server: member joined, marking health alive: member=Node-81c3361e-9082-c29e-4d70-02b160b7cbde +49:34.748 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.748 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-81c3361e-9082-c29e-4d70-02b160b7cbde +49:34.802 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-81c3361e-9082-c29e-4d70-02b160b7cbde +49:34.802 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-81c3361e-9082-c29e-4d70-02b160b7cbde +49:34.804 [INFO] TestAgent: Synced node info +49:34.804 [INFO] TestAgent: Synced service: service=test-id +49:34.804 [DEBUG] TestAgent: Check in sync: check=service:test-id +49:34.807 [INFO] TestAgent: Requesting shutdown +49:34.807 [INFO] TestAgent.server: shutting down server +49:34.807 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:34.807 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:34.807 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:34.807 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:34.807 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:34.807 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:34.807 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:34.807 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:34.807 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:34.807 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:34.807 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:34.807 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:34.807 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:34.807 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:34.807 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:34.807 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:34.807Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:34.807Z [INFO] agent.router.manager: shutting down +49:34.807 [INFO] TestAgent: consul server down +49:34.807 [INFO] TestAgent: shutdown complete +49:34.807 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24016 network=tcp +49:34.807 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24016 network=udp +49:34.807 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24017 network=tcp +49:35.309 [INFO] TestAgent: Waiting for endpoints to shut down +49:35.312 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectCALeafCert_aclServiceWrite (0.66s) +=== CONT TestAgent_NodeMaintenance_Disable +2022-12-04T05:49:34.845Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.859Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.861 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef480160-80fb-061d-b5d3-19aeb4454571 Address:127.0.0.1:24077}]" +49:34.861 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24077 [Follower]" leader= +49:34.861 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef480160-80fb-061d-b5d3-19aeb4454571.dc1 127.0.0.1 +49:34.861 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef480160-80fb-061d-b5d3-19aeb4454571 127.0.0.1 +2022-12-04T05:49:34.861Z [INFO] agent.router: Initializing LAN area manager +49:34.862 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24146 network=udp +49:34.862 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef480160-80fb-061d-b5d3-19aeb4454571.dc1 area=wan +49:34.862 [INFO] TestAgent.server: Adding LAN server: server="Node-ef480160-80fb-061d-b5d3-19aeb4454571 (Addr: tcp/127.0.0.1:24077) (DC: dc1)" +49:34.862 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24146 network=tcp +49:34.862 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24073 network=tcp +49:34.862 [INFO] TestAgent: started state syncer +49:34.862 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23959 network=tcp +49:34.908 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.908 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24077 [Candidate]" term=2 +49:34.909 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.909 [DEBUG] TestAgent.server.raft: vote granted: from=ef480160-80fb-061d-b5d3-19aeb4454571 term=2 tally=1 +49:34.909 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.909 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24077 [Leader]" +49:34.909 [INFO] TestAgent.server: cluster leadership acquired +49:34.909 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24077 +49:34.909 [INFO] TestAgent.server: New leader elected: payload=Node-ef480160-80fb-061d-b5d3-19aeb4454571 +49:34.909 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.909 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.909 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef480160-80fb-061d-b5d3-19aeb4454571 +49:34.909 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef480160-80fb-061d-b5d3-19aeb4454571 +49:34.910 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.932 [INFO] TestAgent: Requesting shutdown +49:34.933 [INFO] TestAgent.server: shutting down server +49:34.933 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:34.933 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:34.933 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:34.933 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:34.933 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:34.933 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:34.933 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:34.933 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:34.933Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:34.933Z [INFO] agent.router.manager: shutting down +49:34.933 [INFO] TestAgent: consul server down +49:34.933 [INFO] TestAgent: shutdown complete +49:34.933 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24146 network=tcp +49:34.933 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24146 network=udp +49:34.933 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24073 network=tcp +49:35.434 [INFO] TestAgent: Waiting for endpoints to shut down +49:35.434 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectCARoots_empty (0.61s) +=== CONT TestAgent_NodeMaintenance_Enable +2022-12-04T05:49:34.702Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:34.702Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.712Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:34.712Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.714 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f4ac9ad5-3f21-d2b5-b371-070c0180db47 Address:127.0.0.1:24063}]" +49:34.714 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24063 [Follower]" leader= +49:34.714 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f4ac9ad5-3f21-d2b5-b371-070c0180db47.dc1 127.0.0.1 +49:34.715 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f4ac9ad5-3f21-d2b5-b371-070c0180db47 127.0.0.1 +2022-12-04T05:49:34.715Z [INFO] agent.router: Initializing LAN area manager +49:34.715 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24065 network=udp +49:34.715 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f4ac9ad5-3f21-d2b5-b371-070c0180db47.dc1 area=wan +49:34.715 [INFO] TestAgent.server: Adding LAN server: server="Node-f4ac9ad5-3f21-d2b5-b371-070c0180db47 (Addr: tcp/127.0.0.1:24063) (DC: dc1)" +49:34.715 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24065 network=tcp +49:34.716 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24059 network=tcp +49:34.716 [INFO] TestAgent: started state syncer +49:34.716 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24064 network=tcp +49:34.752 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.752 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24063 [Candidate]" term=2 +49:34.752 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.752 [DEBUG] TestAgent.server.raft: vote granted: from=f4ac9ad5-3f21-d2b5-b371-070c0180db47 term=2 tally=1 +49:34.752 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.752 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24063 [Leader]" +49:34.752 [INFO] TestAgent.server: cluster leadership acquired +49:34.752 [INFO] TestAgent.server: initializing acls +49:34.752 [INFO] TestAgent.server: New leader elected: payload=Node-f4ac9ad5-3f21-d2b5-b371-070c0180db47 +49:34.753 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:34.753 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:34.753 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:34.753 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:34.753 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:34.753 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:34.753 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f4ac9ad5-3f21-d2b5-b371-070c0180db47 +49:34.753 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f4ac9ad5-3f21-d2b5-b371-070c0180db47.dc1 +49:34.753 [INFO] TestAgent.server: Updating LAN server: server="Node-f4ac9ad5-3f21-d2b5-b371-070c0180db47 (Addr: tcp/127.0.0.1:24063) (DC: dc1)" +49:34.754 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f4ac9ad5-3f21-d2b5-b371-070c0180db47.dc1 area=wan +49:34.754 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.755 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.755 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.755 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.755 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.755 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.755 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f4ac9ad5-3f21-d2b5-b371-070c0180db47 +49:34.755 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f4ac9ad5-3f21-d2b5-b371-070c0180db47 +49:34.756 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.756 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f4ac9ad5-3f21-d2b5-b371-070c0180db47 +49:34.846 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.846 [INFO] TestAgent: Synced node info +49:34.980 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-f4ac9ad5-3f21-d2b5-b371-070c0180db47 +49:34.980 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-f4ac9ad5-3f21-d2b5-b371-070c0180db47 +49:34.981 [DEBUG] TestAgent: Node info in sync +49:34.982 [INFO] TestAgent: Synced service: service=test-id +49:34.982 [DEBUG] TestAgent: Check in sync: check=service:test-id +49:34.984 [INFO] TestAgent: Requesting shutdown +49:34.984 [INFO] TestAgent.server: shutting down server +49:34.984 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:34.984 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:34.984 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:34.984 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:34.984 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:34.984 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:34.984 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:34.985 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:34.985 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:34.985 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:34.985 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:34.985 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:34.985 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:34.985 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:34.985 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:34.985Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:34.985Z [INFO] agent.router.manager: shutting down +49:34.985 [INFO] TestAgent: consul server down +49:34.985 [INFO] TestAgent: shutdown complete +49:34.985 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24065 network=tcp +49:34.985 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24065 network=udp +49:34.985 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24059 network=tcp +49:35.486 [INFO] TestAgent: Waiting for endpoints to shut down +49:35.486 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectCALeafCert_aclDefaultDeny (0.81s) +=== CONT TestAgent_NodeMaintenance_BadRequest +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:34.827Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.845Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.848 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2e93c74f-c3df-eecd-226c-77da41778a97 Address:127.0.0.1:24056}]" +49:34.848 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24056 [Follower]" leader= +49:34.849 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2e93c74f-c3df-eecd-226c-77da41778a97.dc1 127.0.0.1 +49:34.850 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2e93c74f-c3df-eecd-226c-77da41778a97 127.0.0.1 +2022-12-04T05:49:34.850Z [INFO] agent.router: Initializing LAN area manager +49:34.850 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24051 network=udp +49:34.850 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2e93c74f-c3df-eecd-226c-77da41778a97.dc1 area=wan +49:34.850 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24051 network=tcp +49:34.850 [INFO] TestAgent.server: Adding LAN server: server="Node-2e93c74f-c3df-eecd-226c-77da41778a97 (Addr: tcp/127.0.0.1:24056) (DC: dc1)" +49:34.851 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24052 network=tcp +49:34.852 [INFO] TestAgent: started state syncer +49:34.852 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24145 network=tcp +49:34.914 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.914 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24056 [Candidate]" term=2 +49:34.914 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.914 [DEBUG] TestAgent.server.raft: vote granted: from=2e93c74f-c3df-eecd-226c-77da41778a97 term=2 tally=1 +49:34.914 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.914 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24056 [Leader]" +49:34.914 [INFO] TestAgent.server: cluster leadership acquired +49:34.915 [INFO] TestAgent.server: New leader elected: payload=Node-2e93c74f-c3df-eecd-226c-77da41778a97 +49:34.915 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24056 +49:34.916 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.917 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.917 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.917 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.917 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.917 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.917 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2e93c74f-c3df-eecd-226c-77da41778a97 +49:34.917 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2e93c74f-c3df-eecd-226c-77da41778a97 +49:34.917 [INFO] TestAgent.server: federation state anti-entropy synced +49:35.053 [DEBUG] connect.ca.consul: consul CA provider configured: id=25:69:1e:09:89:08:00:5e:8c:ee:8e:ad:31:1d:a9:76:c6:fb:ab:46:18:55:ef:46:34:b3:b7:3d:51:84:cc:46 is_primary=true +49:35.055 [INFO] TestAgent.server.connect: CA rotated to new root under provider: provider=consul +49:35.056 [DEBUG] connect.ca.consul: consul CA provider configured: id=61:3d:5d:e5:a7:76:c3:fd:09:94:9f:57:d3:1c:ec:e5:da:83:bc:4f:2b:be:16:48:7e:2d:4f:c4:a9:6f:d8:ce is_primary=true +49:35.058 [INFO] TestAgent.server.connect: CA rotated to new root under provider: provider=consul +49:35.058 [INFO] TestAgent: Requesting shutdown +49:35.058 [INFO] TestAgent.server: shutting down server +49:35.058 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.058 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.058 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.058 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.058 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.058 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.058 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.058 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:35.058 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.058 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.058 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:35.058 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:49:35.058Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.058Z [INFO] agent.router.manager: shutting down +49:35.058 [INFO] TestAgent: consul server down +49:35.058 [INFO] TestAgent: shutdown complete +49:35.058 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24051 network=tcp +49:35.058 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24051 network=udp +49:35.058 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24052 network=tcp +49:35.560 [INFO] TestAgent: Waiting for endpoints to shut down +49:35.560 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectCARoots_list (0.75s) +=== CONT TestAgent_ServiceMaintenance_ACLDeny +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=1)_compressed=false#01 +2022-12-04T05:49:34.650Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:34.650Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.662Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:34.662Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.664 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b97ec2ae-e9cd-638d-96cb-55efdfb3620f Address:127.0.0.1:24035}]" +49:34.664 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24035 [Follower]" leader= +49:34.664 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b97ec2ae-e9cd-638d-96cb-55efdfb3620f.dc1 127.0.0.1 +49:34.665 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b97ec2ae-e9cd-638d-96cb-55efdfb3620f 127.0.0.1 +2022-12-04T05:49:34.665Z [INFO] agent.router: Initializing LAN area manager +49:34.665 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24037 network=udp +49:34.665 [INFO] TestAgent.server: Adding LAN server: server="Node-b97ec2ae-e9cd-638d-96cb-55efdfb3620f (Addr: tcp/127.0.0.1:24035) (DC: dc1)" +49:34.665 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b97ec2ae-e9cd-638d-96cb-55efdfb3620f.dc1 area=wan +49:34.665 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24037 network=tcp +49:34.666 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24031 network=tcp +49:34.666 [INFO] TestAgent: started state syncer +49:34.666 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24001 network=tcp +49:34.707 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.707 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24035 [Candidate]" term=2 +49:34.707 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.707 [DEBUG] TestAgent.server.raft: vote granted: from=b97ec2ae-e9cd-638d-96cb-55efdfb3620f term=2 tally=1 +49:34.707 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.707 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24035 [Leader]" +49:34.708 [INFO] TestAgent.server: cluster leadership acquired +49:34.708 [INFO] TestAgent.server: initializing acls +49:34.708 [INFO] TestAgent.server: New leader elected: payload=Node-b97ec2ae-e9cd-638d-96cb-55efdfb3620f +49:34.709 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:34.709 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:34.710 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:34.710 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:34.710 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:34.710 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:34.710 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b97ec2ae-e9cd-638d-96cb-55efdfb3620f +49:34.710 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b97ec2ae-e9cd-638d-96cb-55efdfb3620f.dc1 +49:34.710 [INFO] TestAgent.server: Updating LAN server: server="Node-b97ec2ae-e9cd-638d-96cb-55efdfb3620f (Addr: tcp/127.0.0.1:24035) (DC: dc1)" +49:34.710 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b97ec2ae-e9cd-638d-96cb-55efdfb3620f.dc1 area=wan +49:34.711 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.712 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.712 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.712 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.712 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.712 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.712 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b97ec2ae-e9cd-638d-96cb-55efdfb3620f +49:34.712 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b97ec2ae-e9cd-638d-96cb-55efdfb3620f +49:34.712 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.713 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b97ec2ae-e9cd-638d-96cb-55efdfb3620f +49:35.009 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.009 [INFO] TestAgent: Synced node info +49:35.092 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b97ec2ae-e9cd-638d-96cb-55efdfb3620f +49:35.092 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b97ec2ae-e9cd-638d-96cb-55efdfb3620f +49:35.093 [DEBUG] TestAgent: Node info in sync +49:35.094 [INFO] TestAgent: Synced service: service=test-id +49:35.094 [DEBUG] TestAgent: Check in sync: check=service:test-id +49:35.095 [INFO] TestAgent: Requesting shutdown +49:35.095 [INFO] TestAgent.server: shutting down server +49:35.095 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.095 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.095 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:35.095 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:35.095 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.095 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.095 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.095 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.095 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.095 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.095 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.095 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:35.095 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:35.095 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:35.096 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:35.096Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.096Z [INFO] agent.router.manager: shutting down +49:35.096 [INFO] TestAgent: consul server down +49:35.096 [INFO] TestAgent: shutdown complete +49:35.096 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24037 network=tcp +49:35.096 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24037 network=udp +49:35.096 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24031 network=tcp +49:35.596 [INFO] TestAgent: Waiting for endpoints to shut down +49:35.596 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentConnectCALeafCert_aclServiceReadDeny (0.97s) +=== CONT TestAgent_Self +=== RUN TestAgent_Self/no_grpc +2022-12-04T05:49:34.863Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:34.863Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.876Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:34.876Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.878 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f09172c1-e3d9-7add-daf4-a3605038ec90 Address:127.0.0.1:24144}]" +49:34.878 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24144 [Follower]" leader= +49:34.879 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f09172c1-e3d9-7add-daf4-a3605038ec90.dc1 127.0.0.1 +49:34.879 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f09172c1-e3d9-7add-daf4-a3605038ec90 127.0.0.1 +2022-12-04T05:49:34.879Z [INFO] agent.router: Initializing LAN area manager +49:34.879 [INFO] TestAgent.server: Adding LAN server: server="Node-f09172c1-e3d9-7add-daf4-a3605038ec90 (Addr: tcp/127.0.0.1:24144) (DC: dc1)" +49:34.879 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f09172c1-e3d9-7add-daf4-a3605038ec90.dc1 area=wan +49:34.879 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23960 network=udp +49:34.879 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23960 network=tcp +49:34.880 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24140 network=tcp +49:34.880 [INFO] TestAgent: started state syncer +49:34.880 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24015 network=tcp +49:34.932 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:34.948 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.948 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24144 [Candidate]" term=2 +49:34.948 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.948 [DEBUG] TestAgent.server.raft: vote granted: from=f09172c1-e3d9-7add-daf4-a3605038ec90 term=2 tally=1 +49:34.948 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.948 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24144 [Leader]" +49:34.949 [INFO] TestAgent.server: cluster leadership acquired +49:34.949 [INFO] TestAgent.server: initializing acls +49:34.949 [INFO] TestAgent.server: New leader elected: payload=Node-f09172c1-e3d9-7add-daf4-a3605038ec90 +49:34.950 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:34.950 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:34.950 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:34.952 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:34.952 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:34.952 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:34.953 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f09172c1-e3d9-7add-daf4-a3605038ec90 +49:34.953 [INFO] TestAgent.server: Updating LAN server: server="Node-f09172c1-e3d9-7add-daf4-a3605038ec90 (Addr: tcp/127.0.0.1:24144) (DC: dc1)" +49:34.953 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f09172c1-e3d9-7add-daf4-a3605038ec90.dc1 +49:34.954 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f09172c1-e3d9-7add-daf4-a3605038ec90.dc1 area=wan +49:34.954 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.956 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.956 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.956 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.956 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.956 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.956 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f09172c1-e3d9-7add-daf4-a3605038ec90 +49:34.956 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f09172c1-e3d9-7add-daf4-a3605038ec90 +49:34.957 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.957 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f09172c1-e3d9-7add-daf4-a3605038ec90 +49:34.966 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +49:34.966 [WARN] TestAgent: Node info update blocked by ACLs: node=f09172c1-e3d9-7add-daf4-a3605038ec90 accessorID=00000000-0000-0000-0000-000000000002 +49:34.966 [DEBUG] TestAgent: Node info in sync +49:35.246 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-f09172c1-e3d9-7add-daf4-a3605038ec90 +49:35.246 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-f09172c1-e3d9-7add-daf4-a3605038ec90 +49:35.247 [INFO] TestAgent: Updated agent's ACL token: token=acl_token +49:35.247 [INFO] TestAgent: Updated agent's ACL token: token=default +49:35.247 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_token +49:35.247 [INFO] TestAgent: Updated agent's ACL token: token=agent +49:35.247 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_master_token +49:35.248 [INFO] TestAgent: Updated agent's ACL token: token=agent_master +49:35.248 [INFO] TestAgent: Updated agent's ACL token: token=acl_replication_token +49:35.248 [INFO] TestAgent: Updated agent's ACL token: token=replication +49:35.248 [INFO] TestAgent: Updated agent's ACL token: token=acl_token +49:35.248 [INFO] TestAgent: Updated agent's ACL token: token=default +49:35.248 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_token +49:35.248 [INFO] TestAgent: Updated agent's ACL token: token=agent +49:35.249 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_master_token +49:35.249 [INFO] TestAgent: Updated agent's ACL token: token=agent_master +49:35.249 [INFO] TestAgent: Updated agent's ACL token: token=acl_replication_token +49:35.249 [INFO] TestAgent: Updated agent's ACL token: token=replication +49:35.249 [INFO] TestAgent: Requesting shutdown +49:35.249 [INFO] TestAgent.server: shutting down server +49:35.249 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:35.249 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:35.249 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.249 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.249 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.249 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.249 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.249 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.249 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.249 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:35.249 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:35.249 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:35.249 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.249 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.249 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:35.249Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.249Z [INFO] agent.router.manager: shutting down +49:35.250 [INFO] TestAgent: consul server down +49:35.250 [INFO] TestAgent: shutdown complete +49:35.250 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23960 network=tcp +49:35.250 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23960 network=udp +49:35.250 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24140 network=tcp +49:35.750 [INFO] TestAgent: Waiting for endpoints to shut down +49:35.750 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Token (0.90s) --- PASS: TestAgent_Token/bad_token_name (0.00s) --- PASS: TestAgent_Token/bad_JSON (0.00s) --- PASS: TestAgent_Token/set_user_legacy (0.00s) @@ -35567,7 +46533,7 @@ --- PASS: TestAgent_Token/set_repl_legacy (0.00s) --- PASS: TestAgent_Token/set_repl (0.00s) --- PASS: TestAgent_Token/clear_user_legacy (0.00s) - --- PASS: TestAgent_Token/clear_default (0.01s) + --- PASS: TestAgent_Token/clear_default (0.00s) --- PASS: TestAgent_Token/clear_agent_legacy (0.00s) --- PASS: TestAgent_Token/clear_agent (0.00s) --- PASS: TestAgent_Token/clear_master_legacy (0.00s) @@ -35575,3723 +46541,1659 @@ --- PASS: TestAgent_Token/clear_repl_legacy (0.00s) --- PASS: TestAgent_Token/clear_repl (0.00s) --- PASS: TestAgent_Token/permission_denied (0.00s) -=== CONT TestCatalogServiceNodes -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=false -2021-10-31T23:13:19.329Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:19.357Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:19.360 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cc6c1412-7b46-fa70-3f21-2e5646e47e6f Address:127.0.0.1:16684}]" -13:19.360 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16684 [Follower]" leader= -13:19.366 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cc6c1412-7b46-fa70-3f21-2e5646e47e6f.dc1 127.0.0.1 -13:19.367 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cc6c1412-7b46-fa70-3f21-2e5646e47e6f 127.0.0.1 -2021-10-31T23:13:19.367Z [INFO] agent.router: Initializing LAN area manager -13:19.367 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16735 network=udp -13:19.367 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cc6c1412-7b46-fa70-3f21-2e5646e47e6f.dc1 area=wan -13:19.367 [INFO] TestAgent.server: Adding LAN server: server="Node-cc6c1412-7b46-fa70-3f21-2e5646e47e6f (Addr: tcp/127.0.0.1:16684) (DC: dc1)" -13:19.367 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16735 network=tcp -13:19.368 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16715 network=tcp -13:19.368 [INFO] TestAgent: started state syncer -13:19.368 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16685 network=tcp -13:19.418 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:19.418 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16684 [Candidate]" term=2 -13:19.418 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:19.418 [DEBUG] TestAgent.server.raft: vote granted: from=cc6c1412-7b46-fa70-3f21-2e5646e47e6f term=2 tally=1 -13:19.418 [INFO] TestAgent.server.raft: election won: tally=1 -13:19.418 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16684 [Leader]" -13:19.418 [INFO] TestAgent.server: cluster leadership acquired -13:19.418 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16684 -13:19.419 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:19.421 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:19.421 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:19.421 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:19.421 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:19.421 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:19.421 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cc6c1412-7b46-fa70-3f21-2e5646e47e6f -13:19.421 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cc6c1412-7b46-fa70-3f21-2e5646e47e6f -13:19.421 [INFO] TestAgent.server: New leader elected: payload=Node-cc6c1412-7b46-fa70-3f21-2e5646e47e6f -13:19.422 [INFO] TestAgent.server: federation state anti-entropy synced -13:19.583 [INFO] TestAgent: Requesting shutdown -13:19.583 [INFO] TestAgent.server: shutting down server -13:19.584 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:19.584 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:19.584 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:19.584 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:19.584 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:19.585 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:19.585 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:19.585 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:19.585 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:19.588 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:19.588Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:19.588Z [INFO] agent.router.manager: shutting down -13:19.588 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:19.588 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:19.588 [INFO] TestAgent: consul server down -13:19.588 [INFO] TestAgent: shutdown complete -13:19.588 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16735 network=tcp -13:19.588 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16735 network=udp -13:19.588 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16715 network=tcp -13:20.089 [INFO] TestAgent: Waiting for endpoints to shut down -13:20.089 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogServiceNodes_ConnectProxy (0.83s) -=== CONT TestAgent_NodeMaintenance_Disable -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:19.327Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:19.327Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:19.373Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:19.373Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:19.375 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:68773aef-7249-b468-c02a-37f0c992233c Address:127.0.0.1:16733}]" -13:19.376 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-68773aef-7249-b468-c02a-37f0c992233c.dc1 127.0.0.1 -13:19.376 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-68773aef-7249-b468-c02a-37f0c992233c 127.0.0.1 -2021-10-31T23:13:19.377Z [INFO] agent.router: Initializing LAN area manager -13:19.377 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16162 network=udp -13:19.377 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16733 [Follower]" leader= -13:19.378 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-68773aef-7249-b468-c02a-37f0c992233c.dc1 area=wan -13:19.403 [INFO] TestAgent.server: Adding LAN server: server="Node-68773aef-7249-b468-c02a-37f0c992233c (Addr: tcp/127.0.0.1:16733) (DC: dc1)" -13:19.406 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16162 network=tcp -13:19.409 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16163 network=tcp -13:19.409 [INFO] TestAgent: started state syncer -13:19.410 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16734 network=tcp -13:19.430 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:19.446 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:19.446 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16733 [Candidate]" term=2 -13:19.446 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:19.446 [DEBUG] TestAgent.server.raft: vote granted: from=68773aef-7249-b468-c02a-37f0c992233c term=2 tally=1 -13:19.446 [INFO] TestAgent.server.raft: election won: tally=1 -13:19.446 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16733 [Leader]" -13:19.446 [INFO] TestAgent.server: cluster leadership acquired -13:19.446 [INFO] TestAgent.server: initializing acls -13:19.447 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:19.447 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:19.447 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:19.448 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:19.448 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:19.448 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:19.448 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-68773aef-7249-b468-c02a-37f0c992233c -13:19.448 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-68773aef-7249-b468-c02a-37f0c992233c.dc1 -13:19.450 [INFO] TestAgent.server: New leader elected: payload=Node-68773aef-7249-b468-c02a-37f0c992233c -13:19.450 [INFO] TestAgent.server: Updating LAN server: server="Node-68773aef-7249-b468-c02a-37f0c992233c (Addr: tcp/127.0.0.1:16733) (DC: dc1)" -13:19.450 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-68773aef-7249-b468-c02a-37f0c992233c.dc1 area=wan -13:19.464 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:19.466 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:19.466 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:19.466 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:19.466 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:19.466 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:19.466 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-68773aef-7249-b468-c02a-37f0c992233c -13:19.466 [INFO] TestAgent.server: member joined, marking health alive: member=Node-68773aef-7249-b468-c02a-37f0c992233c -13:19.474 [INFO] TestAgent.server: federation state anti-entropy synced -13:19.474 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-68773aef-7249-b468-c02a-37f0c992233c -13:19.570 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-68773aef-7249-b468-c02a-37f0c992233c -13:19.570 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-68773aef-7249-b468-c02a-37f0c992233c -13:19.573 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_token -13:19.573 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -13:19.574 [INFO] TestAgent: Synced node info -13:19.602 [INFO] TestAgent: Requesting shutdown -13:19.602 [INFO] TestAgent.server: shutting down server -13:19.602 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:19.602 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:19.602 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:19.602 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:19.602 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:19.602 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:19.602 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:19.603 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:19.603 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:19.603 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:19.603 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:19.603 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:19.603 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:19.603 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:19.603 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:19.603Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:19.603Z [INFO] agent.router.manager: shutting down -13:19.603 [INFO] TestAgent: consul server down -13:19.603 [INFO] TestAgent: shutdown complete -13:19.603 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16162 network=tcp -13:19.603 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16162 network=udp -13:19.603 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16163 network=tcp -13:20.105 [INFO] TestAgent: Waiting for endpoints to shut down -13:20.105 [INFO] TestAgent: Endpoints down +=== CONT TestAgent_ServiceMaintenance_Enable +2022-12-04T05:49:34.872Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:34.872Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.883Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:34.883Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.885 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:328d6743-d280-2115-82e8-d9266fe9fea4 Address:127.0.0.1:24028}]" +49:34.885 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24028 [Follower]" leader= +49:34.885 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-328d6743-d280-2115-82e8-d9266fe9fea4.dc1 127.0.0.1 +49:34.886 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-328d6743-d280-2115-82e8-d9266fe9fea4 127.0.0.1 +2022-12-04T05:49:34.886Z [INFO] agent.router: Initializing LAN area manager +49:34.886 [INFO] TestAgent.server: Adding LAN server: server="Node-328d6743-d280-2115-82e8-d9266fe9fea4 (Addr: tcp/127.0.0.1:24028) (DC: dc1)" +49:34.886 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-328d6743-d280-2115-82e8-d9266fe9fea4.dc1 area=wan +49:34.886 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24023 network=tcp +49:34.886 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24023 network=udp +49:34.886 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24024 network=tcp +49:34.886 [INFO] TestAgent: started state syncer +49:34.886 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24082 network=tcp +49:34.924 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.924 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24028 [Candidate]" term=2 +49:34.924 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.924 [DEBUG] TestAgent.server.raft: vote granted: from=328d6743-d280-2115-82e8-d9266fe9fea4 term=2 tally=1 +49:34.924 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.924 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24028 [Leader]" +49:34.924 [INFO] TestAgent.server: cluster leadership acquired +49:34.924 [INFO] TestAgent.server: New leader elected: payload=Node-328d6743-d280-2115-82e8-d9266fe9fea4 +49:34.924 [INFO] TestAgent.server: initializing acls +49:34.925 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:34.925 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:34.925 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:34.925 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:34.925 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:34.925 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:34.925 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-328d6743-d280-2115-82e8-d9266fe9fea4 +49:34.925 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-328d6743-d280-2115-82e8-d9266fe9fea4.dc1 +49:34.926 [INFO] TestAgent.server: Updating LAN server: server="Node-328d6743-d280-2115-82e8-d9266fe9fea4 (Addr: tcp/127.0.0.1:24028) (DC: dc1)" +49:34.926 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-328d6743-d280-2115-82e8-d9266fe9fea4.dc1 area=wan +49:34.932 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.934 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.934 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.934 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.934 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.934 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.934 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-328d6743-d280-2115-82e8-d9266fe9fea4 +49:34.934 [INFO] TestAgent.server: member joined, marking health alive: member=Node-328d6743-d280-2115-82e8-d9266fe9fea4 +49:34.935 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-328d6743-d280-2115-82e8-d9266fe9fea4 +49:34.936 [INFO] TestAgent.server: federation state anti-entropy synced +49:35.012 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-328d6743-d280-2115-82e8-d9266fe9fea4 +49:35.012 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-328d6743-d280-2115-82e8-d9266fe9fea4 +49:35.014 [INFO] TestAgent: Updated agent's ACL token: token=acl_agent_token +49:35.267 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +49:35.268 [INFO] TestAgent: Synced node info +49:35.268 [DEBUG] TestAgent: Node info in sync +49:35.279 [INFO] TestAgent: Requesting shutdown +49:35.279 [INFO] TestAgent.server: shutting down server +49:35.279 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.279 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.279 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.279 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.279 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:35.279 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:35.279 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.279 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.279 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.279 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.279 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:35.279 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:35.279 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.279 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:35.279 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:35.279Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.279Z [INFO] agent.router.manager: shutting down +49:35.279 [INFO] TestAgent: consul server down +49:35.279 [INFO] TestAgent: shutdown complete +49:35.279 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24023 network=tcp +49:35.279 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24023 network=udp +49:35.279 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24024 network=tcp +49:35.780 [INFO] TestAgent: Waiting for endpoints to shut down +49:35.780 [INFO] TestAgent: Endpoints down === RUN TestAgent_TokenTriggersFullSync/agent -2021-10-31T23:13:19.141Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:19.208Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:19.210 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:986153bb-960f-a6e9-8e1b-21583221709a Address:127.0.0.1:16174}]" -13:19.211 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-986153bb-960f-a6e9-8e1b-21583221709a.dc1 127.0.0.1 -13:19.211 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-986153bb-960f-a6e9-8e1b-21583221709a 127.0.0.1 -2021-10-31T23:13:19.211Z [INFO] agent.router: Initializing LAN area manager -13:19.212 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16174 [Follower]" leader= -13:19.212 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16253 network=udp -13:19.212 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16253 network=tcp -13:19.212 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-986153bb-960f-a6e9-8e1b-21583221709a.dc1 area=wan -13:19.212 [INFO] TestAgent.server: Adding LAN server: server="Node-986153bb-960f-a6e9-8e1b-21583221709a (Addr: tcp/127.0.0.1:16174) (DC: dc1)" -13:19.213 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16254 network=tcp -13:19.213 [INFO] TestAgent: started state syncer -13:19.213 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16175 network=tcp -13:19.270 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:19.271 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16174 [Candidate]" term=2 -13:19.271 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:19.271 [DEBUG] TestAgent.server.raft: vote granted: from=986153bb-960f-a6e9-8e1b-21583221709a term=2 tally=1 -13:19.271 [INFO] TestAgent.server.raft: election won: tally=1 -13:19.271 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16174 [Leader]" -13:19.271 [INFO] TestAgent.server: cluster leadership acquired -13:19.272 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16174 -13:19.273 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:19.274 [INFO] TestAgent.server: New leader elected: payload=Node-986153bb-960f-a6e9-8e1b-21583221709a -13:19.295 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:19.295 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:19.295 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:19.295 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:19.295 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:19.295 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-986153bb-960f-a6e9-8e1b-21583221709a -13:19.295 [INFO] TestAgent.server: member joined, marking health alive: member=Node-986153bb-960f-a6e9-8e1b-21583221709a -13:19.296 [INFO] TestAgent.server: federation state anti-entropy synced -13:19.366 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:19.367 [INFO] TestAgent: Synced node info -13:19.446 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:19.446 [DEBUG] TestAgent: Node info in sync -13:19.446 [DEBUG] TestAgent: Node info in sync -13:19.653 [INFO] TestAgent: Requesting shutdown -13:19.654 [INFO] TestAgent.server: shutting down server -13:19.654 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:19.654 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:19.654 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:19.654 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:19.654 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:19.654 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:19.654 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:19.654 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:19.654 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:19.654 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:19.654 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:19.654Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:19.654Z [INFO] agent.router.manager: shutting down -13:19.654 [INFO] TestAgent: consul server down -13:19.654 [INFO] TestAgent: shutdown complete -13:19.654 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16253 network=tcp -13:19.654 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16253 network=udp -13:19.654 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16254 network=tcp -13:20.188 [INFO] TestAgent: Waiting for endpoints to shut down -13:20.188 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogConnectServiceNodes_good (1.08s) -=== CONT TestAgent_NodeMaintenance_Enable -2021-10-31T23:13:19.476Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:19.476Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:19.518Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:19.518Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:19.520 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ffd6f941-ac30-c768-378a-d691f0fc7159 Address:127.0.0.1:16677}]" -13:19.521 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ffd6f941-ac30-c768-378a-d691f0fc7159.dc1 127.0.0.1 -13:19.521 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16677 [Follower]" leader= -13:19.534 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ffd6f941-ac30-c768-378a-d691f0fc7159 127.0.0.1 -2021-10-31T23:13:19.534Z [INFO] agent.router: Initializing LAN area manager -13:19.534 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16672 network=udp -13:19.535 [INFO] TestAgent.server: Adding LAN server: server="Node-ffd6f941-ac30-c768-378a-d691f0fc7159 (Addr: tcp/127.0.0.1:16677) (DC: dc1)" -13:19.535 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ffd6f941-ac30-c768-378a-d691f0fc7159.dc1 area=wan -13:19.535 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16672 network=tcp -13:19.536 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16673 network=tcp -13:19.536 [INFO] TestAgent: started state syncer -13:19.536 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16678 network=tcp -13:19.589 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:19.589 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16677 [Candidate]" term=2 -13:19.589 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:19.589 [DEBUG] TestAgent.server.raft: vote granted: from=ffd6f941-ac30-c768-378a-d691f0fc7159 term=2 tally=1 -13:19.589 [INFO] TestAgent.server.raft: election won: tally=1 -13:19.589 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16677 [Leader]" -13:19.593 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=2 mode=2 found=true leader=127.0.0.1:16677 -13:19.593 [INFO] TestAgent.server: cluster leadership acquired -13:19.594 [INFO] TestAgent.server: initializing acls -13:19.594 [INFO] TestAgent.server: New leader elected: payload=Node-ffd6f941-ac30-c768-378a-d691f0fc7159 -13:19.594 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:19.594 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:19.601 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:19.602 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:19.602 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:19.602 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:19.602 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ffd6f941-ac30-c768-378a-d691f0fc7159 -13:19.602 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ffd6f941-ac30-c768-378a-d691f0fc7159.dc1 -13:19.603 [INFO] TestAgent.server: Updating LAN server: server="Node-ffd6f941-ac30-c768-378a-d691f0fc7159 (Addr: tcp/127.0.0.1:16677) (DC: dc1)" -13:19.603 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ffd6f941-ac30-c768-378a-d691f0fc7159.dc1 area=wan -13:19.603 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:19.654 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:19.654 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:19.654 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:19.654 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:19.654 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:19.654 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ffd6f941-ac30-c768-378a-d691f0fc7159 -13:19.654 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ffd6f941-ac30-c768-378a-d691f0fc7159 -13:19.655 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ffd6f941-ac30-c768-378a-d691f0fc7159 -13:19.655 [INFO] TestAgent.server: federation state anti-entropy synced -13:19.663 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:19.663 [INFO] TestAgent: Synced node info -13:19.746 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ffd6f941-ac30-c768-378a-d691f0fc7159 -13:19.746 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ffd6f941-ac30-c768-378a-d691f0fc7159 -13:19.746 [INFO] TestAgent: Requesting shutdown -13:19.746 [INFO] TestAgent.server: shutting down server -13:19.746 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:19.746 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:19.746 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:19.746 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:19.746 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:19.746 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:19.746 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:19.746 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:19.746 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:19.746 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:19.746 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:19.746 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:19.746 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:19.746 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:19.746 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2021-10-31T23:13:19.747Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:19.747Z [INFO] agent.router.manager: shutting down -13:19.747 [INFO] TestAgent: consul server down -13:19.747 [INFO] TestAgent: shutdown complete -13:19.747 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16672 network=tcp -13:19.747 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16672 network=udp -13:19.747 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16673 network=tcp -13:20.250 [INFO] TestAgent: Waiting for endpoints to shut down -13:20.250 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Monitor_ACLDeny (0.84s) -=== CONT TestCatalogRegister_checkRegistration -2021-10-31T23:13:19.456Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:19.513Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:19.535 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:879368a7-1055-479d-e2a3-dc22e24ef8f1 Address:127.0.0.1:16670}]" -13:19.536 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16670 [Follower]" leader= -13:19.536 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-879368a7-1055-479d-e2a3-dc22e24ef8f1.dc1 127.0.0.1 -13:19.554 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-879368a7-1055-479d-e2a3-dc22e24ef8f1 127.0.0.1 -2021-10-31T23:13:19.554Z [INFO] agent.router: Initializing LAN area manager -13:19.555 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16686 network=udp -13:19.555 [INFO] TestAgent.server: Adding LAN server: server="Node-879368a7-1055-479d-e2a3-dc22e24ef8f1 (Addr: tcp/127.0.0.1:16670) (DC: dc1)" -13:19.555 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-879368a7-1055-479d-e2a3-dc22e24ef8f1.dc1 area=wan -13:19.555 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16686 network=tcp -13:19.556 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16694 network=tcp -13:19.556 [INFO] TestAgent: started state syncer -13:19.557 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16671 network=tcp -13:19.591 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:19.591 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16670 [Candidate]" term=2 -13:19.592 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:19.592 [DEBUG] TestAgent.server.raft: vote granted: from=879368a7-1055-479d-e2a3-dc22e24ef8f1 term=2 tally=1 -13:19.592 [INFO] TestAgent.server.raft: election won: tally=1 -13:19.592 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16670 [Leader]" -13:19.592 [INFO] TestAgent.server: cluster leadership acquired -13:19.592 [INFO] TestAgent.server: New leader elected: payload=Node-879368a7-1055-479d-e2a3-dc22e24ef8f1 -13:19.592 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16670 -13:19.593 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:19.652 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:19.652 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:19.652 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:19.652 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:19.652 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:19.652 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-879368a7-1055-479d-e2a3-dc22e24ef8f1 -13:19.652 [INFO] TestAgent.server: member joined, marking health alive: member=Node-879368a7-1055-479d-e2a3-dc22e24ef8f1 -13:19.653 [INFO] TestAgent.server: federation state anti-entropy synced -13:19.762 [INFO] TestAgent: Requesting shutdown -13:19.762 [INFO] TestAgent.server: shutting down server -13:19.762 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:19.762 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:19.762 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:19.762 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:19.762 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:19.762 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:19.762 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:19.762 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:19.762 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:19.762 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:19.762 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:19.762 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:19.762Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:19.763Z [INFO] agent.router.manager: shutting down -13:19.763 [INFO] TestAgent: consul server down -13:19.763 [INFO] TestAgent: shutdown complete -13:19.763 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16686 network=tcp -13:19.763 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16686 network=udp -13:19.763 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16694 network=tcp -13:20.265 [INFO] TestAgent: Waiting for endpoints to shut down -13:20.265 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogServiceNodes_DistanceSort (0.87s) -=== CONT TestAgent_NodeMaintenance_BadRequest -=== RUN TestAgent_NodeMaintenance_ACLDeny/no_token -=== RUN TestAgent_NodeMaintenance_ACLDeny/root_token -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:19.563Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:19.652Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:19.654 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:234fd4af-e83e-d686-d82a-b0ad6ac3f095 Address:127.0.0.1:16705}]" -13:19.655 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16705 [Follower]" leader= -13:19.655 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-234fd4af-e83e-d686-d82a-b0ad6ac3f095.dc1 127.0.0.1 -13:19.656 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-234fd4af-e83e-d686-d82a-b0ad6ac3f095 127.0.0.1 -2021-10-31T23:13:19.656Z [INFO] agent.router: Initializing LAN area manager -13:19.656 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16714 network=udp -13:19.656 [INFO] TestAgent.server: Adding LAN server: server="Node-234fd4af-e83e-d686-d82a-b0ad6ac3f095 (Addr: tcp/127.0.0.1:16705) (DC: dc1)" -13:19.656 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-234fd4af-e83e-d686-d82a-b0ad6ac3f095.dc1 area=wan -13:19.656 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16714 network=tcp -13:19.657 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16743 network=tcp -13:19.657 [INFO] TestAgent: started state syncer -13:19.657 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16706 network=tcp -13:19.732 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:19.732 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16705 [Candidate]" term=2 -13:19.733 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:19.733 [DEBUG] TestAgent.server.raft: vote granted: from=234fd4af-e83e-d686-d82a-b0ad6ac3f095 term=2 tally=1 -13:19.733 [INFO] TestAgent.server.raft: election won: tally=1 -13:19.733 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16705 [Leader]" -13:19.733 [INFO] TestAgent.server: cluster leadership acquired -13:19.733 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16705 -13:19.734 [INFO] TestAgent.server: New leader elected: payload=Node-234fd4af-e83e-d686-d82a-b0ad6ac3f095 -13:19.735 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:19.746 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:19.746 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:19.746 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:19.746 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:19.746 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:19.746 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-234fd4af-e83e-d686-d82a-b0ad6ac3f095 -13:19.746 [INFO] TestAgent.server: member joined, marking health alive: member=Node-234fd4af-e83e-d686-d82a-b0ad6ac3f095 -13:19.747 [INFO] TestAgent.server: federation state anti-entropy synced -13:19.968 [INFO] TestAgent: Requesting shutdown -13:19.968 [INFO] TestAgent.server: shutting down server -13:19.968 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:19.968 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:19.968 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:19.968 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:19.968 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:19.968 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:19.968 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:19.968 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:19.968 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:19.968 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:19.968 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:19.968 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:19.969Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:19.969Z [INFO] agent.router.manager: shutting down -13:19.969 [INFO] TestAgent: consul server down -13:19.969 [INFO] TestAgent: shutdown complete -13:19.969 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16714 network=tcp -13:19.969 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16714 network=udp -13:19.969 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16743 network=tcp -13:20.469 [INFO] TestAgent: Waiting for endpoints to shut down -13:20.469 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:19.852Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:19.906Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:19.909 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:701a7808-542c-a592-2424-18ceca9bc034 Address:127.0.0.1:16719}]" -13:19.909 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16719 [Follower]" leader= -13:19.914 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-701a7808-542c-a592-2424-18ceca9bc034.dc1 127.0.0.1 -13:19.915 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-701a7808-542c-a592-2424-18ceca9bc034 127.0.0.1 -2021-10-31T23:13:19.915Z [INFO] agent.router: Initializing LAN area manager -13:19.915 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16749 network=udp -13:19.915 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-701a7808-542c-a592-2424-18ceca9bc034.dc1 area=wan -13:19.916 [INFO] TestAgent.server: Adding LAN server: server="Node-701a7808-542c-a592-2424-18ceca9bc034 (Addr: tcp/127.0.0.1:16719) (DC: dc1)" -13:19.916 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16749 network=tcp -13:19.917 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16736 network=tcp -13:19.917 [INFO] TestAgent: started state syncer -13:19.918 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16720 network=tcp -13:19.953 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:19.954 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16719 [Candidate]" term=2 -13:19.954 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:19.954 [DEBUG] TestAgent.server.raft: vote granted: from=701a7808-542c-a592-2424-18ceca9bc034 term=2 tally=1 -13:19.954 [INFO] TestAgent.server.raft: election won: tally=1 -13:19.954 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16719 [Leader]" -13:19.954 [INFO] TestAgent.server: cluster leadership acquired -13:19.954 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16719 -13:19.955 [INFO] TestAgent.server: New leader elected: payload=Node-701a7808-542c-a592-2424-18ceca9bc034 -13:19.955 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:19.958 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:19.958 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:19.958 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:19.958 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:19.958 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:19.958 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-701a7808-542c-a592-2424-18ceca9bc034 -13:19.958 [INFO] TestAgent.server: member joined, marking health alive: member=Node-701a7808-542c-a592-2424-18ceca9bc034 -13:19.958 [INFO] TestAgent.server: federation state anti-entropy synced -13:19.964 [INFO] TestAgent: Requesting shutdown -13:19.964 [INFO] TestAgent.server: shutting down server -13:19.964 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:19.964 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:19.964 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:19.964 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:19.964 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:19.965 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:19.965 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:19.965 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:19.965 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:19.965 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:19.965 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:19.965 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:19.965Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:19.965Z [INFO] agent.router.manager: shutting down -13:19.965 [INFO] TestAgent: consul server down -13:19.965 [INFO] TestAgent: shutdown complete -13:19.965 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16749 network=tcp -13:19.965 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16749 network=udp -13:19.965 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16736 network=tcp -13:20.470 [INFO] TestAgent: Waiting for endpoints to shut down -13:20.470 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogServiceNodes_Filter (0.95s) -=== CONT TestCatalogServices_NodeMetaFilter ---- PASS: TestCatalogServiceNodes_NodeMetaFilter (0.64s) -=== CONT TestAgent_ServiceMaintenance_ACLDeny -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:19.994Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:20.020Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:20.022 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:27c58a43-1863-1af4-e302-d54a7ae4b498 Address:127.0.0.1:16754}]" -13:20.023 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-27c58a43-1863-1af4-e302-d54a7ae4b498.dc1 127.0.0.1 -13:20.024 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-27c58a43-1863-1af4-e302-d54a7ae4b498 127.0.0.1 -2021-10-31T23:13:20.024Z [INFO] agent.router: Initializing LAN area manager -13:20.024 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16728 network=udp -13:20.024 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16754 [Follower]" leader= -13:20.024 [INFO] TestAgent.server: Adding LAN server: server="Node-27c58a43-1863-1af4-e302-d54a7ae4b498 (Addr: tcp/127.0.0.1:16754) (DC: dc1)" -13:20.024 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-27c58a43-1863-1af4-e302-d54a7ae4b498.dc1 area=wan -13:20.024 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16728 network=tcp -13:20.026 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16708 network=tcp -13:20.026 [INFO] TestAgent: started state syncer -13:20.026 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16755 network=tcp -13:20.093 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:20.093 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16754 [Candidate]" term=2 -13:20.094 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:20.094 [DEBUG] TestAgent.server.raft: vote granted: from=27c58a43-1863-1af4-e302-d54a7ae4b498 term=2 tally=1 -13:20.094 [INFO] TestAgent.server.raft: election won: tally=1 -13:20.094 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16754 [Leader]" -13:20.094 [INFO] TestAgent.server: cluster leadership acquired -13:20.094 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16754 -13:20.094 [INFO] TestAgent.server: New leader elected: payload=Node-27c58a43-1863-1af4-e302-d54a7ae4b498 -13:20.095 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:20.096 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:20.096 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:20.096 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:20.096 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:20.096 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:20.096 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-27c58a43-1863-1af4-e302-d54a7ae4b498 -13:20.096 [INFO] TestAgent.server: member joined, marking health alive: member=Node-27c58a43-1863-1af4-e302-d54a7ae4b498 -13:20.097 [INFO] TestAgent.server: federation state anti-entropy synced -13:20.148 [INFO] TestAgent: Requesting shutdown -13:20.148 [INFO] TestAgent.server: shutting down server -13:20.148 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:20.148 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:20.148 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:20.148 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:20.148 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:20.149 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:20.149 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:20.149 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:20.149 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:20.149 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:20.149 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:20.149 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:20.149Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:20.149Z [INFO] agent.router.manager: shutting down -13:20.149 [INFO] TestAgent: consul server down -13:20.149 [INFO] TestAgent: shutdown complete -13:20.149 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16728 network=tcp -13:20.149 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16728 network=udp -13:20.149 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16708 network=tcp -13:20.650 [INFO] TestAgent: Waiting for endpoints to shut down -13:20.650 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogServiceNodes (0.71s) -=== CONT TestCatalogServices -2021-10-31T23:13:19.834Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:19.861Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:19.863 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:56e48337-33ce-8f02-e069-7b4ccbeab75e Address:127.0.0.1:16747}]" -13:19.864 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-56e48337-33ce-8f02-e069-7b4ccbeab75e.dc1 127.0.0.1 -13:19.865 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-56e48337-33ce-8f02-e069-7b4ccbeab75e 127.0.0.1 -2021-10-31T23:13:19.865Z [INFO] agent.router: Initializing LAN area manager -13:19.865 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16763 network=udp -13:19.865 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16747 [Follower]" leader= -13:19.865 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-56e48337-33ce-8f02-e069-7b4ccbeab75e.dc1 area=wan -13:19.866 [INFO] TestAgent.server: Adding LAN server: server="Node-56e48337-33ce-8f02-e069-7b4ccbeab75e (Addr: tcp/127.0.0.1:16747) (DC: dc1)" -13:19.866 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16763 network=tcp -13:19.867 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16750 network=tcp -13:19.867 [INFO] TestAgent: started state syncer -13:19.867 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16748 network=tcp -13:19.933 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:19.934 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16747 [Candidate]" term=2 -13:19.934 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:19.934 [DEBUG] TestAgent.server.raft: vote granted: from=56e48337-33ce-8f02-e069-7b4ccbeab75e term=2 tally=1 -13:19.934 [INFO] TestAgent.server.raft: election won: tally=1 -13:19.934 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16747 [Leader]" -13:19.934 [INFO] TestAgent.server: cluster leadership acquired -13:19.934 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16747 -13:19.936 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:19.937 [INFO] TestAgent.server: New leader elected: payload=Node-56e48337-33ce-8f02-e069-7b4ccbeab75e -13:19.949 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:19.949 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:19.949 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:19.949 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:19.949 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:19.949 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-56e48337-33ce-8f02-e069-7b4ccbeab75e -13:19.949 [INFO] TestAgent.server: member joined, marking health alive: member=Node-56e48337-33ce-8f02-e069-7b4ccbeab75e -13:19.962 [INFO] TestAgent.server: federation state anti-entropy synced -13:20.030 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:20.030 [INFO] TestAgent: Synced node info -13:20.210 [DEBUG] TestAgent: Node info in sync -13:20.214 [INFO] TestAgent: Synced service: service=memcache -13:20.214 [DEBUG] TestAgent: Check in sync: check=service:memcache -13:20.215 [DEBUG] TestAgent: Node info in sync -13:20.215 [DEBUG] TestAgent: Service in sync: service=memcache -13:20.215 [DEBUG] TestAgent: Check in sync: check=service:memcache -13:20.216 [INFO] TestAgent: Synced check: check=memcache_check2 -13:20.216 [INFO] TestAgent: Requesting shutdown -13:20.216 [INFO] TestAgent.server: shutting down server -13:20.216 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:20.216 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:20.216 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:20.216 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:20.216 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:20.216 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:20.216 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:20.216 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:20.216 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" -13:20.216 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:20.216 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:20.216Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:20.216Z [INFO] agent.router.manager: shutting down -13:20.217 [INFO] TestAgent: consul server down -13:20.217 [INFO] TestAgent: shutdown complete -13:20.217 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16763 network=tcp -13:20.217 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16763 network=udp -13:20.217 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16750 network=tcp -13:20.717 [INFO] TestAgent: Waiting for endpoints to shut down -13:20.717 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RegisterCheck_Service (0.94s) -=== CONT TestAgent_ServiceMaintenance_Disable -2021-10-31T23:13:19.778Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:19.830Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:19.832 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:429f8eb3-15a6-6175-8998-a0aa466fd327 Address:127.0.0.1:16761}]" -13:19.833 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-429f8eb3-15a6-6175-8998-a0aa466fd327.dc1 127.0.0.1 -13:19.833 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-429f8eb3-15a6-6175-8998-a0aa466fd327 127.0.0.1 -2021-10-31T23:13:19.833Z [INFO] agent.router: Initializing LAN area manager -13:19.833 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16761 [Follower]" leader= -13:19.834 [INFO] TestAgent.server: Adding LAN server: server="Node-429f8eb3-15a6-6175-8998-a0aa466fd327 (Addr: tcp/127.0.0.1:16761) (DC: dc1)" -13:19.834 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-429f8eb3-15a6-6175-8998-a0aa466fd327.dc1 area=wan -13:19.838 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16582 network=udp -13:19.838 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16582 network=tcp -13:19.839 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16583 network=tcp -13:19.839 [INFO] TestAgent: started state syncer -13:19.839 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16762 network=tcp -13:19.893 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:19.893 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16761 [Candidate]" term=2 -13:19.893 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:19.893 [DEBUG] TestAgent.server.raft: vote granted: from=429f8eb3-15a6-6175-8998-a0aa466fd327 term=2 tally=1 -13:19.893 [INFO] TestAgent.server.raft: election won: tally=1 -13:19.893 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16761 [Leader]" -13:19.906 [INFO] TestAgent.server: cluster leadership acquired -13:19.906 [INFO] TestAgent.server: New leader elected: payload=Node-429f8eb3-15a6-6175-8998-a0aa466fd327 -13:19.906 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16761 -13:19.908 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:19.918 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:19.918 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:19.918 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:19.918 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:19.918 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:19.918 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-429f8eb3-15a6-6175-8998-a0aa466fd327 -13:19.918 [INFO] TestAgent.server: member joined, marking health alive: member=Node-429f8eb3-15a6-6175-8998-a0aa466fd327 -13:19.919 [INFO] TestAgent.server: federation state anti-entropy synced -13:20.217 [INFO] TestAgent.server.connect: CA provider config updated -13:20.217 [INFO] TestAgent: Requesting shutdown -13:20.225 [INFO] TestAgent.server: shutting down server -13:20.225 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:20.225 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:20.225 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:20.225 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:20.225 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:20.225 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:20.225 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:20.225 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:20.225 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:20.225 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:20.225 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:20.225 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:20.226Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:20.226Z [INFO] agent.router.manager: shutting down -13:20.226 [INFO] TestAgent: consul server down -13:20.226 [INFO] TestAgent: shutdown complete -13:20.226 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16582 network=tcp -13:20.226 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16582 network=udp -13:20.226 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16583 network=tcp -13:20.729 [INFO] TestAgent: Waiting for endpoints to shut down -13:20.729 [INFO] TestAgent: Endpoints down -=== RUN TestConnectCAConfig/setting_state_fails -2021-10-31T23:13:20.112Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:20.145Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:20.147 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fa35700a-1320-7f57-1aa8-9fd909bcbfb5 Address:127.0.0.1:16740}]" -13:20.148 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16740 [Follower]" leader= -13:20.154 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fa35700a-1320-7f57-1aa8-9fd909bcbfb5.dc1 127.0.0.1 -13:20.155 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fa35700a-1320-7f57-1aa8-9fd909bcbfb5 127.0.0.1 -2021-10-31T23:13:20.155Z [INFO] agent.router: Initializing LAN area manager -13:20.179 [INFO] TestAgent.server: Adding LAN server: server="Node-fa35700a-1320-7f57-1aa8-9fd909bcbfb5 (Addr: tcp/127.0.0.1:16740) (DC: dc1)" -13:20.180 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fa35700a-1320-7f57-1aa8-9fd909bcbfb5.dc1 area=wan -13:20.180 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16756 network=udp -13:20.180 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16756 network=tcp -13:20.181 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16778 network=tcp -13:20.181 [INFO] TestAgent: started state syncer -13:20.181 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16741 network=tcp -13:20.193 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:20.193 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16740 [Candidate]" term=2 -13:20.214 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:20.214 [DEBUG] TestAgent.server.raft: vote granted: from=fa35700a-1320-7f57-1aa8-9fd909bcbfb5 term=2 tally=1 -13:20.214 [INFO] TestAgent.server.raft: election won: tally=1 -13:20.214 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16740 [Leader]" -13:20.215 [INFO] TestAgent.server: cluster leadership acquired -13:20.215 [INFO] TestAgent.server: New leader elected: payload=Node-fa35700a-1320-7f57-1aa8-9fd909bcbfb5 -13:20.215 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16740 -13:20.216 [INFO] TestAgent: Synced node info -13:20.216 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:20.227 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:20.227 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:20.227 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:20.227 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:20.227 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:20.227 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fa35700a-1320-7f57-1aa8-9fd909bcbfb5 -13:20.227 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fa35700a-1320-7f57-1aa8-9fd909bcbfb5 -13:20.228 [INFO] TestAgent.server: federation state anti-entropy synced -13:20.259 [INFO] TestAgent: Node entered maintenance mode -13:20.260 [DEBUG] TestAgent: removed check: check=_node_maintenance -13:20.260 [INFO] TestAgent: Node left maintenance mode -13:20.260 [DEBUG] TestAgent: Node info in sync -13:20.260 [INFO] TestAgent: Deregistered check: check=_node_maintenance -13:20.260 [INFO] TestAgent: Requesting shutdown -13:20.260 [INFO] TestAgent.server: shutting down server -13:20.260 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:20.260 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:20.260 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:20.260 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:20.260 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:20.260 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:20.260 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:20.260 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:20.260 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:20.260 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:20.260 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:20.261Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:20.261Z [INFO] agent.router.manager: shutting down -13:20.261 [INFO] TestAgent: consul server down -13:20.261 [INFO] TestAgent: shutdown complete -13:20.261 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16756 network=tcp -13:20.261 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16756 network=udp -13:20.261 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16778 network=tcp -13:20.761 [INFO] TestAgent: Waiting for endpoints to shut down -13:20.762 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_NodeMaintenance_Disable (0.68s) -=== CONT TestCatalogNodes_DistanceSort -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=true -2021-10-31T23:13:19.993Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:19.993Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:20.028Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:20.028Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:20.038 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:beec9cb3-5e9e-cb0c-42c2-4dc22ed2c198 Address:127.0.0.1:16726}]" -13:20.039 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-beec9cb3-5e9e-cb0c-42c2-4dc22ed2c198.dc1 127.0.0.1 -13:20.040 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-beec9cb3-5e9e-cb0c-42c2-4dc22ed2c198 127.0.0.1 -2021-10-31T23:13:20.040Z [INFO] agent.router: Initializing LAN area manager -13:20.040 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16721 network=udp -13:20.040 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16726 [Follower]" leader= -13:20.040 [INFO] TestAgent.server: Adding LAN server: server="Node-beec9cb3-5e9e-cb0c-42c2-4dc22ed2c198 (Addr: tcp/127.0.0.1:16726) (DC: dc1)" -13:20.040 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-beec9cb3-5e9e-cb0c-42c2-4dc22ed2c198.dc1 area=wan -13:20.040 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16721 network=tcp -13:20.041 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16722 network=tcp -13:20.041 [INFO] TestAgent: started state syncer -13:20.041 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16727 network=tcp -13:20.093 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:20.109 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:20.109 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16726 [Candidate]" term=2 -13:20.114 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:20.114 [DEBUG] TestAgent.server.raft: vote granted: from=beec9cb3-5e9e-cb0c-42c2-4dc22ed2c198 term=2 tally=1 -13:20.114 [INFO] TestAgent.server.raft: election won: tally=1 -13:20.114 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16726 [Leader]" -13:20.114 [INFO] TestAgent.server: cluster leadership acquired -13:20.114 [INFO] TestAgent.server: initializing acls -13:20.114 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:20.114 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:20.115 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:20.115 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:20.115 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:20.115 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:20.115 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-beec9cb3-5e9e-cb0c-42c2-4dc22ed2c198 -13:20.115 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-beec9cb3-5e9e-cb0c-42c2-4dc22ed2c198.dc1 -13:20.116 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:20.117 [INFO] TestAgent.server: New leader elected: payload=Node-beec9cb3-5e9e-cb0c-42c2-4dc22ed2c198 -13:20.117 [INFO] TestAgent.server: Updating LAN server: server="Node-beec9cb3-5e9e-cb0c-42c2-4dc22ed2c198 (Addr: tcp/127.0.0.1:16726) (DC: dc1)" -13:20.118 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-beec9cb3-5e9e-cb0c-42c2-4dc22ed2c198.dc1 area=wan -13:20.122 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:20.122 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:20.122 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:20.122 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:20.122 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:20.122 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-beec9cb3-5e9e-cb0c-42c2-4dc22ed2c198 -13:20.122 [INFO] TestAgent.server: member joined, marking health alive: member=Node-beec9cb3-5e9e-cb0c-42c2-4dc22ed2c198 -13:20.123 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-beec9cb3-5e9e-cb0c-42c2-4dc22ed2c198 -13:20.123 [INFO] TestAgent.server: federation state anti-entropy synced -13:20.289 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-beec9cb3-5e9e-cb0c-42c2-4dc22ed2c198 -13:20.312 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-beec9cb3-5e9e-cb0c-42c2-4dc22ed2c198 -13:20.313 [INFO] TestAgent: Node entered maintenance mode -13:20.313 [INFO] TestAgent: Synced node info -13:20.314 [INFO] TestAgent: Synced check: check=_node_maintenance -13:20.314 [INFO] TestAgent: Requesting shutdown -13:20.314 [INFO] TestAgent.server: shutting down server -13:20.314 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:20.314 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:20.314 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:20.314 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:20.314 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:20.314 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:20.314 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:20.314 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:20.314 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:20.314 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:20.314 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:20.314 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:20.314 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:20.314 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:20.314 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:20.314 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:20.315Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:20.315Z [INFO] agent.router.manager: shutting down -13:20.315 [INFO] TestAgent: consul server down -13:20.315 [INFO] TestAgent: shutdown complete -13:20.315 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16721 network=tcp -13:20.315 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16721 network=udp -13:20.315 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16722 network=tcp -13:20.817 [INFO] TestAgent: Waiting for endpoints to shut down -13:20.817 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_NodeMaintenance_ACLDeny (0.90s) +2022-12-04T05:49:35.136Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:35.136Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:35.145Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:35.145Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:35.147 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:02ecc161-5c7f-c806-b808-b64dca029a8c Address:127.0.0.1:24151}]" +49:35.147 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24151 [Follower]" leader= +49:35.147 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-02ecc161-5c7f-c806-b808-b64dca029a8c.dc1 127.0.0.1 +49:35.165 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-02ecc161-5c7f-c806-b808-b64dca029a8c 127.0.0.1 +2022-12-04T05:49:35.165Z [INFO] agent.router: Initializing LAN area manager +49:35.165 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24139 network=udp +49:35.166 [INFO] TestAgent.server: Adding LAN server: server="Node-02ecc161-5c7f-c806-b808-b64dca029a8c (Addr: tcp/127.0.0.1:24151) (DC: dc1)" +49:35.166 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-02ecc161-5c7f-c806-b808-b64dca029a8c.dc1 area=wan +49:35.168 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24139 network=tcp +49:35.170 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24147 network=tcp +49:35.170 [INFO] TestAgent: started state syncer +49:35.170 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24117 network=tcp +49:35.200 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.200 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24151 [Candidate]" term=2 +49:35.200 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:35.200 [DEBUG] TestAgent.server.raft: vote granted: from=02ecc161-5c7f-c806-b808-b64dca029a8c term=2 tally=1 +49:35.200 [INFO] TestAgent.server.raft: election won: tally=1 +49:35.200 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24151 [Leader]" +49:35.200 [INFO] TestAgent.server: cluster leadership acquired +49:35.200 [INFO] TestAgent.server: initializing acls +49:35.200 [INFO] TestAgent.server: New leader elected: payload=Node-02ecc161-5c7f-c806-b808-b64dca029a8c +49:35.201 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:35.201 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:35.201 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:35.201 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:35.201 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:35.201 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:35.201 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-02ecc161-5c7f-c806-b808-b64dca029a8c +49:35.202 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-02ecc161-5c7f-c806-b808-b64dca029a8c.dc1 +49:35.202 [INFO] TestAgent.server: Updating LAN server: server="Node-02ecc161-5c7f-c806-b808-b64dca029a8c (Addr: tcp/127.0.0.1:24151) (DC: dc1)" +49:35.202 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-02ecc161-5c7f-c806-b808-b64dca029a8c.dc1 area=wan +49:35.202 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:35.203 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:35.203 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:35.203 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:35.203 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:35.203 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:35.203 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-02ecc161-5c7f-c806-b808-b64dca029a8c +49:35.203 [INFO] TestAgent.server: member joined, marking health alive: member=Node-02ecc161-5c7f-c806-b808-b64dca029a8c +49:35.203 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-02ecc161-5c7f-c806-b808-b64dca029a8c +49:35.204 [INFO] TestAgent.server: federation state anti-entropy synced +49:35.226 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.226 [INFO] TestAgent: Synced node info +49:35.282 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-02ecc161-5c7f-c806-b808-b64dca029a8c +49:35.282 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-02ecc161-5c7f-c806-b808-b64dca029a8c +49:35.283 [INFO] TestAgent: Node entered maintenance mode +49:35.283 [DEBUG] TestAgent: Node info in sync +49:35.283 [INFO] TestAgent: Synced check: check=_node_maintenance +49:35.283 [INFO] TestAgent: Requesting shutdown +49:35.283 [INFO] TestAgent.server: shutting down server +49:35.283 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.283 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.283 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:35.283 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:35.283 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.283 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.283 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:35.283 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.283 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:35.283 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.283 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:35.283 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:35.283Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.283Z [INFO] agent.router.manager: shutting down +49:35.283 [INFO] TestAgent: consul server down +49:35.283 [INFO] TestAgent: shutdown complete +49:35.283 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24139 network=tcp +49:35.283 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24139 network=udp +49:35.283 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24147 network=tcp +49:35.785 [INFO] TestAgent: Waiting for endpoints to shut down +49:35.785 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_NodeMaintenance_ACLDeny (0.66s) --- PASS: TestAgent_NodeMaintenance_ACLDeny/no_token (0.00s) --- PASS: TestAgent_NodeMaintenance_ACLDeny/root_token (0.00s) -=== CONT TestAgent_ServiceMaintenance_Enable -[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:19.673Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:19.673Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:19.700Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:19.700Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:19.723 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ceba6a04-ee06-8144-6549-9da34b393c5e Address:127.0.0.1:16580}]" -13:19.724 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ceba6a04-ee06-8144-6549-9da34b393c5e.dc2 127.0.0.1 -13:19.725 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ceba6a04-ee06-8144-6549-9da34b393c5e 127.0.0.1 -2021-10-31T23:13:19.725Z [INFO] agent.router: Initializing LAN area manager -13:19.730 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16580 [Follower]" leader= -13:19.732 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16707 network=tcp -13:19.732 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16707 network=udp -13:19.732 [INFO] TestAgent.server: Adding LAN server: server="Node-ceba6a04-ee06-8144-6549-9da34b393c5e (Addr: tcp/127.0.0.1:16580) (DC: dc2)" -13:19.732 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ceba6a04-ee06-8144-6549-9da34b393c5e.dc2 area=wan -13:19.737 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16729 network=tcp -13:19.738 [INFO] TestAgent: started state syncer -13:19.738 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16581 network=tcp -13:19.781 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:19.787 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:19.787 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16580 [Candidate]" term=2 -13:19.788 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:19.788 [DEBUG] TestAgent.server.raft: vote granted: from=ceba6a04-ee06-8144-6549-9da34b393c5e term=2 tally=1 -13:19.788 [INFO] TestAgent.server.raft: election won: tally=1 -13:19.788 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16580 [Leader]" -13:19.788 [INFO] TestAgent.server: cluster leadership acquired -13:19.788 [INFO] TestAgent.server: initializing acls -13:19.788 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:19.789 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:19.789 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:19.789 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:19.789 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ceba6a04-ee06-8144-6549-9da34b393c5e -13:19.789 [INFO] TestAgent.server: New leader elected: payload=Node-ceba6a04-ee06-8144-6549-9da34b393c5e -13:19.789 [INFO] TestAgent.server: Updating LAN server: server="Node-ceba6a04-ee06-8144-6549-9da34b393c5e (Addr: tcp/127.0.0.1:16580) (DC: dc2)" -13:19.798 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ceba6a04-ee06-8144-6549-9da34b393c5e.dc2 -13:19.799 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:19.801 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ceba6a04-ee06-8144-6549-9da34b393c5e.dc2 area=wan -13:19.810 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:19.810 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:19.810 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:19.810 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:19.810 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:19.810 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ceba6a04-ee06-8144-6549-9da34b393c5e -13:19.810 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ceba6a04-ee06-8144-6549-9da34b393c5e -13:19.811 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ceba6a04-ee06-8144-6549-9da34b393c5e -13:19.811 [INFO] TestAgent.server: federation state anti-entropy synced -13:19.891 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:16690] -13:19.892 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:16690 -13:19.903 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c7e28652-242f-79fd-e522-44273a73b489.dc1 127.0.0.1 -13:19.903 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -13:19.905 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c7e28652-242f-79fd-e522-44273a73b489.dc1 area=wan -13:19.919 [INFO] TestAgent: Requesting shutdown -13:19.919 [INFO] TestAgent.server: shutting down server -13:19.919 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:19.919 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:19.919 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:19.919 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:19.919 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:19.919 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:19.919 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:19.920 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:19.920 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:19.920 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:19.920 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:19.920 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:19.920 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:19.920 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:19.920 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:19.920 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:19.920Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:19.920Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:19.920Z [INFO] agent.router.manager: shutting down -13:19.920 [INFO] TestAgent: consul server down -13:19.920 [INFO] TestAgent: shutdown complete -13:19.920 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16707 network=tcp -13:19.920 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16707 network=udp -13:19.920 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16729 network=tcp -13:20.421 [INFO] TestAgent: Waiting for endpoints to shut down -13:20.438 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:19.478Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:19.478Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:19.518Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:19.518Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:19.520 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c7e28652-242f-79fd-e522-44273a73b489 Address:127.0.0.1:16691}]" -13:19.521 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c7e28652-242f-79fd-e522-44273a73b489.dc1 127.0.0.1 -13:19.521 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16691 [Follower]" leader= -13:19.527 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c7e28652-242f-79fd-e522-44273a73b489 127.0.0.1 -2021-10-31T23:13:19.527Z [INFO] agent.router: Initializing LAN area manager -13:19.527 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16679 network=udp -13:19.527 [INFO] TestAgent.server: Adding LAN server: server="Node-c7e28652-242f-79fd-e522-44273a73b489 (Addr: tcp/127.0.0.1:16691) (DC: dc1)" -13:19.527 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c7e28652-242f-79fd-e522-44273a73b489.dc1 area=wan -13:19.527 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16679 network=tcp -13:19.528 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16687 network=tcp -13:19.528 [INFO] TestAgent: started state syncer -13:19.528 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16692 network=tcp -13:19.577 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:19.584 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:19.584 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16691 [Candidate]" term=2 -13:19.584 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:19.584 [DEBUG] TestAgent.server.raft: vote granted: from=c7e28652-242f-79fd-e522-44273a73b489 term=2 tally=1 -13:19.584 [INFO] TestAgent.server.raft: election won: tally=1 -13:19.584 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16691 [Leader]" -13:19.585 [INFO] TestAgent.server: cluster leadership acquired -13:19.588 [INFO] TestAgent.server: initializing acls -13:19.589 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:19.589 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:19.589 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:19.589 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:19.590 [INFO] TestAgent.server: New leader elected: payload=Node-c7e28652-242f-79fd-e522-44273a73b489 -13:19.590 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c7e28652-242f-79fd-e522-44273a73b489 -13:19.590 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c7e28652-242f-79fd-e522-44273a73b489.dc1 -13:19.590 [INFO] TestAgent.server: Updating LAN server: server="Node-c7e28652-242f-79fd-e522-44273a73b489 (Addr: tcp/127.0.0.1:16691) (DC: dc1)" -13:19.591 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c7e28652-242f-79fd-e522-44273a73b489.dc1 area=wan -13:19.592 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:19.647 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:19.647 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:19.647 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:19.647 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:19.647 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:19.650 [INFO] TestAgent.server: federation state anti-entropy synced -13:19.652 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c7e28652-242f-79fd-e522-44273a73b489 -13:19.652 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c7e28652-242f-79fd-e522-44273a73b489 -13:19.654 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c7e28652-242f-79fd-e522-44273a73b489 -13:19.892 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:34742 -13:19.903 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ceba6a04-ee06-8144-6549-9da34b393c5e.dc2 127.0.0.1 -13:19.903 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ceba6a04-ee06-8144-6549-9da34b393c5e.dc2 area=wan -13:19.918 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:19.919 [INFO] TestAgent: Synced node info -13:19.919 [DEBUG] TestAgent: Node info in sync -13:20.438 [INFO] TestAgent: Requesting shutdown -13:20.438 [INFO] TestAgent.server: shutting down server -13:20.438 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:20.438 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:20.438 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:20.438 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:20.438 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:20.438 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:20.438 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:20.438 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:20.438 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:20.438 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:20.438 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:20.438 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:20.438 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:20.438 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:20.438 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:20.439Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:20.439Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:20.439Z [INFO] agent.router.manager: shutting down -13:20.439 [INFO] TestAgent: consul server down -13:20.439 [INFO] TestAgent: shutdown complete -13:20.439 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16679 network=tcp -13:20.439 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16679 network=udp -13:20.439 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16687 network=tcp -13:20.941 [INFO] TestAgent: Waiting for endpoints to shut down -13:20.941 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogServiceNodes_WanTranslation (1.52s) -=== CONT TestCatalogNodes_Blocking -=== RUN TestAgent_ServiceMaintenance_ACLDeny/no_token -=== RUN TestAgent_ServiceMaintenance_ACLDeny/root_token -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=false -2021-10-31T23:13:20.306Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:20.335Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:20.361 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ce65ba18-b07d-4f81-bd5b-f78e1877a723 Address:127.0.0.1:16775}]" -13:20.365 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16775 [Follower]" leader= -13:20.370 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ce65ba18-b07d-4f81-bd5b-f78e1877a723.dc1 127.0.0.1 -13:20.371 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ce65ba18-b07d-4f81-bd5b-f78e1877a723 127.0.0.1 -2021-10-31T23:13:20.371Z [INFO] agent.router: Initializing LAN area manager -13:20.371 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16805 network=udp -13:20.372 [INFO] TestAgent.server: Adding LAN server: server="Node-ce65ba18-b07d-4f81-bd5b-f78e1877a723 (Addr: tcp/127.0.0.1:16775) (DC: dc1)" -13:20.372 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ce65ba18-b07d-4f81-bd5b-f78e1877a723.dc1 area=wan -13:20.374 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16805 network=tcp -13:20.376 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16806 network=tcp -13:20.376 [INFO] TestAgent: started state syncer -13:20.377 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16776 network=tcp -13:20.435 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:20.435 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16775 [Candidate]" term=2 -13:20.436 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:20.436 [DEBUG] TestAgent.server.raft: vote granted: from=ce65ba18-b07d-4f81-bd5b-f78e1877a723 term=2 tally=1 -13:20.436 [INFO] TestAgent.server.raft: election won: tally=1 -13:20.436 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16775 [Leader]" -13:20.436 [INFO] TestAgent.server: cluster leadership acquired -13:20.436 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16775 -13:20.437 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:20.437 [INFO] TestAgent.server: New leader elected: payload=Node-ce65ba18-b07d-4f81-bd5b-f78e1877a723 -13:20.442 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:20.442 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:20.442 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:20.442 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:20.442 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:20.442 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ce65ba18-b07d-4f81-bd5b-f78e1877a723 -13:20.442 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ce65ba18-b07d-4f81-bd5b-f78e1877a723 -13:20.454 [INFO] TestAgent.server: federation state anti-entropy synced -13:20.579 [INFO] TestAgent: Requesting shutdown -13:20.579 [INFO] TestAgent.server: shutting down server -13:20.579 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:20.579 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:20.579 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:20.579 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:20.579 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:20.579 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:20.579 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:20.579 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:20.579 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:20.579 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:20.579 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:20.579 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:20.580Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:20.580Z [INFO] agent.router.manager: shutting down -13:20.580 [INFO] TestAgent: consul server down -13:20.580 [INFO] TestAgent: shutdown complete -13:20.580 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16805 network=tcp -13:20.580 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16805 network=udp -13:20.580 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16806 network=tcp -13:21.081 [INFO] TestAgent: Waiting for endpoints to shut down -13:21.081 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_NodeMaintenance_BadRequest (0.81s) === CONT TestAgent_ServiceMaintenance_BadRequest -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestAgent_Monitor/serf_shutdown_logging -2021-10-31T23:13:20.265Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:20.363Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:20.366 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:65de0f8f-7cdb-c229-2120-92322a2b0827 Address:127.0.0.1:16782}]" -13:20.366 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16782 [Follower]" leader= -13:20.367 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-65de0f8f-7cdb-c229-2120-92322a2b0827.dc1 127.0.0.1 -13:20.367 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-65de0f8f-7cdb-c229-2120-92322a2b0827 127.0.0.1 -2021-10-31T23:13:20.367Z [INFO] agent.router: Initializing LAN area manager -13:20.368 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16770 network=udp -13:20.368 [INFO] TestAgent.server: Adding LAN server: server="Node-65de0f8f-7cdb-c229-2120-92322a2b0827 (Addr: tcp/127.0.0.1:16782) (DC: dc1)" -13:20.368 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-65de0f8f-7cdb-c229-2120-92322a2b0827.dc1 area=wan -13:20.368 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16770 network=tcp -13:20.394 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16785 network=tcp -13:20.394 [INFO] TestAgent: started state syncer -13:20.394 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16783 network=tcp -13:20.434 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:20.434 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16782 [Candidate]" term=2 -13:20.434 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:20.434 [DEBUG] TestAgent.server.raft: vote granted: from=65de0f8f-7cdb-c229-2120-92322a2b0827 term=2 tally=1 -13:20.434 [INFO] TestAgent.server.raft: election won: tally=1 -13:20.434 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16782 [Leader]" -13:20.434 [INFO] TestAgent.server: cluster leadership acquired -13:20.435 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16782 -13:20.435 [INFO] TestAgent.server: New leader elected: payload=Node-65de0f8f-7cdb-c229-2120-92322a2b0827 -13:20.436 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:20.442 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:20.442 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:20.442 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:20.442 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:20.442 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:20.443 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-65de0f8f-7cdb-c229-2120-92322a2b0827 -13:20.443 [INFO] TestAgent.server: member joined, marking health alive: member=Node-65de0f8f-7cdb-c229-2120-92322a2b0827 -13:20.444 [INFO] TestAgent.server: federation state anti-entropy synced -13:20.611 [INFO] TestAgent: Node entered maintenance mode -13:20.611 [INFO] TestAgent: Synced node info -13:20.612 [INFO] TestAgent: Synced check: check=_node_maintenance -13:20.612 [INFO] TestAgent: Requesting shutdown -13:20.612 [INFO] TestAgent.server: shutting down server -13:20.612 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:20.612 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:20.612 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:20.612 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:20.612 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:20.612 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:20.612 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:20.612 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:20.612 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:20.612 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:20.612 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:20.612 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:20.612Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:20.612Z [INFO] agent.router.manager: shutting down -13:20.612 [INFO] TestAgent: consul server down -13:20.612 [INFO] TestAgent: shutdown complete -13:20.612 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16770 network=tcp -13:20.612 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16770 network=udp -13:20.613 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16785 network=tcp -13:21.114 [INFO] TestAgent: Waiting for endpoints to shut down -13:21.114 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_NodeMaintenance_Enable (0.93s) -=== CONT TestCatalogNodes_WanTranslation -2021-10-31T23:13:20.132Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:20.132Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:20.244Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:20.244Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:20.256 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a43b12f8-8aaf-62c9-6f2f-17e91546c519 Address:127.0.0.1:16768}]" -13:20.256 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16768 [Follower]" leader= -13:20.269 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a43b12f8-8aaf-62c9-6f2f-17e91546c519.dc1 127.0.0.1 -13:20.270 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a43b12f8-8aaf-62c9-6f2f-17e91546c519 127.0.0.1 -2021-10-31T23:13:20.270Z [INFO] agent.router: Initializing LAN area manager -13:20.270 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16742 network=udp -13:20.271 [INFO] TestAgent.server: Adding LAN server: server="Node-a43b12f8-8aaf-62c9-6f2f-17e91546c519 (Addr: tcp/127.0.0.1:16768) (DC: dc1)" -13:20.271 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a43b12f8-8aaf-62c9-6f2f-17e91546c519.dc1 area=wan -13:20.271 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16742 network=tcp -13:20.272 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16757 network=tcp -13:20.272 [INFO] TestAgent: started state syncer -13:20.272 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16769 network=tcp -13:20.306 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:20.306 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16768 [Candidate]" term=2 -13:20.306 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:20.306 [DEBUG] TestAgent.server.raft: vote granted: from=a43b12f8-8aaf-62c9-6f2f-17e91546c519 term=2 tally=1 -13:20.306 [INFO] TestAgent.server.raft: election won: tally=1 -13:20.306 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16768 [Leader]" -13:20.306 [INFO] TestAgent.server: cluster leadership acquired -13:20.306 [INFO] TestAgent.server: initializing acls -13:20.307 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:20.307 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:20.307 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:20.307 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:20.307 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:20.307 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:20.308 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a43b12f8-8aaf-62c9-6f2f-17e91546c519 -13:20.308 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a43b12f8-8aaf-62c9-6f2f-17e91546c519.dc1 -13:20.308 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:20.309 [INFO] TestAgent.server: New leader elected: payload=Node-a43b12f8-8aaf-62c9-6f2f-17e91546c519 -13:20.310 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a43b12f8-8aaf-62c9-6f2f-17e91546c519.dc1 area=wan -13:20.310 [INFO] TestAgent.server: Updating LAN server: server="Node-a43b12f8-8aaf-62c9-6f2f-17e91546c519 (Addr: tcp/127.0.0.1:16768) (DC: dc1)" -13:20.310 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:20.310 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:20.310 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:20.310 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:20.310 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:20.310 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a43b12f8-8aaf-62c9-6f2f-17e91546c519 -13:20.310 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a43b12f8-8aaf-62c9-6f2f-17e91546c519 -13:20.311 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a43b12f8-8aaf-62c9-6f2f-17e91546c519 -13:20.311 [INFO] TestAgent.server: federation state anti-entropy synced -13:20.518 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a43b12f8-8aaf-62c9-6f2f-17e91546c519 -13:20.518 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a43b12f8-8aaf-62c9-6f2f-17e91546c519 -13:20.520 [INFO] TestAgent: Updated agent's ACL token: token=agent -13:20.610 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -13:20.611 [INFO] TestAgent: Synced node info -13:20.630 [INFO] TestAgent: Requesting shutdown -13:20.630 [INFO] TestAgent.server: shutting down server -13:20.630 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:20.630 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:20.630 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:20.630 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:20.630 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:20.630 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:20.630 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:20.630 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:20.630 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:20.630 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:20.630 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:20.630 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:20.630 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:20.630 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:20.630 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:20.631Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:20.631Z [INFO] agent.router.manager: shutting down -13:20.631 [INFO] TestAgent: consul server down -13:20.631 [INFO] TestAgent: shutdown complete -13:20.631 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16742 network=tcp -13:20.631 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16742 network=udp -13:20.631 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16757 network=tcp -13:21.133 [INFO] TestAgent: Waiting for endpoints to shut down -13:21.133 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_TokenTriggersFullSync/acl_token -2021-10-31T23:13:20.312Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:20.368Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:20.371 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3fef0c34-40fd-da98-f611-d8b0caffc1b2 Address:127.0.0.1:16803}]" -13:20.372 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3fef0c34-40fd-da98-f611-d8b0caffc1b2.dc1 127.0.0.1 -13:20.373 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16803 [Follower]" leader= -13:20.373 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3fef0c34-40fd-da98-f611-d8b0caffc1b2 127.0.0.1 -2021-10-31T23:13:20.373Z [INFO] agent.router: Initializing LAN area manager -13:20.377 [INFO] TestAgent.server: Adding LAN server: server="Node-3fef0c34-40fd-da98-f611-d8b0caffc1b2 (Addr: tcp/127.0.0.1:16803) (DC: dc1)" -13:20.400 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16784 network=udp -13:20.400 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3fef0c34-40fd-da98-f611-d8b0caffc1b2.dc1 area=wan -13:20.400 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16784 network=tcp -13:20.402 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16764 network=tcp -13:20.403 [INFO] TestAgent: started state syncer -13:20.403 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16804 network=tcp -13:20.430 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:20.430 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16803 [Candidate]" term=2 -13:20.431 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:20.431 [DEBUG] TestAgent.server.raft: vote granted: from=3fef0c34-40fd-da98-f611-d8b0caffc1b2 term=2 tally=1 -13:20.431 [INFO] TestAgent.server.raft: election won: tally=1 -13:20.431 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16803 [Leader]" -13:20.431 [INFO] TestAgent.server: cluster leadership acquired -13:20.431 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16803 -13:20.432 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:20.433 [INFO] TestAgent.server: New leader elected: payload=Node-3fef0c34-40fd-da98-f611-d8b0caffc1b2 -13:20.434 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:20.434 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:20.434 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:20.434 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:20.434 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:20.434 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3fef0c34-40fd-da98-f611-d8b0caffc1b2 -13:20.434 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3fef0c34-40fd-da98-f611-d8b0caffc1b2 -13:20.435 [INFO] TestAgent.server: federation state anti-entropy synced -13:20.598 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:20.598 [INFO] TestAgent: Synced node info -13:20.598 [DEBUG] TestAgent: Node info in sync -13:20.704 [INFO] TestAgent: Requesting shutdown -13:20.704 [INFO] TestAgent.server: shutting down server -13:20.704 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:20.704 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:20.704 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:20.704 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:20.704 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:20.704 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:20.704 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:20.704 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:20.704 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:20.704 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:20.704 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:20.704Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:20.704Z [INFO] agent.router.manager: shutting down -13:20.704 [INFO] TestAgent: consul server down -13:20.704 [INFO] TestAgent: shutdown complete -13:20.704 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16784 network=tcp -13:20.705 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16784 network=udp -13:20.705 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16764 network=tcp -13:21.205 [INFO] TestAgent: Waiting for endpoints to shut down -13:21.205 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogRegister_checkRegistration (0.96s) -=== CONT TestCatalogNodes_Filter -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=255)_compressed=true -2021-10-31T23:13:20.492Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:20.550Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:20.552 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3bddebd1-e545-83d8-f18b-e8f5ec2e467e Address:127.0.0.1:16817}]" -13:20.553 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16817 [Follower]" leader= -13:20.563 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3bddebd1-e545-83d8-f18b-e8f5ec2e467e.dc1 127.0.0.1 -13:20.564 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3bddebd1-e545-83d8-f18b-e8f5ec2e467e 127.0.0.1 -2021-10-31T23:13:20.564Z [INFO] agent.router: Initializing LAN area manager -13:20.564 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16777 network=udp -13:20.564 [INFO] TestAgent.server: Adding LAN server: server="Node-3bddebd1-e545-83d8-f18b-e8f5ec2e467e (Addr: tcp/127.0.0.1:16817) (DC: dc1)" -13:20.564 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3bddebd1-e545-83d8-f18b-e8f5ec2e467e.dc1 area=wan -13:20.564 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16777 network=tcp -13:20.565 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16792 network=tcp -13:20.565 [INFO] TestAgent: started state syncer -13:20.565 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16818 network=tcp -13:20.621 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:20.621 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16817 [Candidate]" term=2 -13:20.622 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:20.622 [DEBUG] TestAgent.server.raft: vote granted: from=3bddebd1-e545-83d8-f18b-e8f5ec2e467e term=2 tally=1 -13:20.622 [INFO] TestAgent.server.raft: election won: tally=1 -13:20.622 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16817 [Leader]" -13:20.622 [INFO] TestAgent.server: cluster leadership acquired -13:20.622 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16817 -13:20.623 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:20.624 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:20.624 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:20.624 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:20.624 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:20.624 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:20.625 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3bddebd1-e545-83d8-f18b-e8f5ec2e467e -13:20.625 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3bddebd1-e545-83d8-f18b-e8f5ec2e467e -13:20.625 [INFO] TestAgent.server: New leader elected: payload=Node-3bddebd1-e545-83d8-f18b-e8f5ec2e467e -13:20.626 [INFO] TestAgent.server: federation state anti-entropy synced -13:20.711 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:20.712 [INFO] TestAgent: Synced node info -13:20.712 [DEBUG] TestAgent: Node info in sync -13:20.767 [INFO] TestAgent: Requesting shutdown -13:20.767 [INFO] TestAgent.server: shutting down server -13:20.767 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:20.767 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:20.767 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:20.767 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:20.767 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:20.767 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:20.767 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:20.767 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:20.767 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:20.767 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:20.767 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:20.767Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:20.767Z [INFO] agent.router.manager: shutting down -13:20.768 [INFO] TestAgent: consul server down -13:20.768 [INFO] TestAgent: shutdown complete -13:20.768 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16777 network=tcp -13:20.768 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16777 network=udp -13:20.768 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16792 network=tcp -13:21.269 [INFO] TestAgent: Waiting for endpoints to shut down -13:21.269 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogServices_NodeMetaFilter (0.80s) +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:35.121Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:35.133Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:35.135 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a9f56812-9b2b-bdf6-b213-0de0ea65caac Address:127.0.0.1:24095}]" +49:35.135 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24095 [Follower]" leader= +49:35.135 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a9f56812-9b2b-bdf6-b213-0de0ea65caac.dc1 127.0.0.1 +49:35.136 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a9f56812-9b2b-bdf6-b213-0de0ea65caac 127.0.0.1 +2022-12-04T05:49:35.136Z [INFO] agent.router: Initializing LAN area manager +49:35.136 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a9f56812-9b2b-bdf6-b213-0de0ea65caac.dc1 area=wan +49:35.136 [INFO] TestAgent.server: Adding LAN server: server="Node-a9f56812-9b2b-bdf6-b213-0de0ea65caac (Addr: tcp/127.0.0.1:24095) (DC: dc1)" +49:35.136 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24088 network=udp +49:35.136 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24088 network=tcp +49:35.137 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24105 network=tcp +49:35.137 [INFO] TestAgent: started state syncer +49:35.137 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24138 network=tcp +49:35.194 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.194 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24095 [Candidate]" term=2 +49:35.194 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:35.194 [DEBUG] TestAgent.server.raft: vote granted: from=a9f56812-9b2b-bdf6-b213-0de0ea65caac term=2 tally=1 +49:35.194 [INFO] TestAgent.server.raft: election won: tally=1 +49:35.194 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24095 [Leader]" +49:35.194 [INFO] TestAgent.server: cluster leadership acquired +49:35.194 [INFO] TestAgent.server: New leader elected: payload=Node-a9f56812-9b2b-bdf6-b213-0de0ea65caac +49:35.194 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24095 +49:35.195 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:35.197 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:35.197 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:35.197 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:35.197 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:35.197 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:35.197 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a9f56812-9b2b-bdf6-b213-0de0ea65caac +49:35.197 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a9f56812-9b2b-bdf6-b213-0de0ea65caac +49:35.197 [INFO] TestAgent.server: federation state anti-entropy synced +49:35.315 [INFO] TestAgent: Synced node info +49:35.316 [INFO] TestAgent: Synced service: service=memcache +49:35.316 [DEBUG] TestAgent: Check in sync: check=service:memcache +49:35.316 [DEBUG] TestAgent: Node info in sync +49:35.316 [DEBUG] TestAgent: Service in sync: service=memcache +49:35.316 [DEBUG] TestAgent: Check in sync: check=service:memcache +49:35.317 [INFO] TestAgent: Synced check: check=memcache_check2 +49:35.317 [INFO] TestAgent: Requesting shutdown +49:35.317 [INFO] TestAgent.server: shutting down server +49:35.317 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.317 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.317 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.317 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.317 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.317 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.317 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.317 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:35.317 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.317 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:35.317 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.317 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:35.318Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.318Z [INFO] agent.router.manager: shutting down +49:35.318 [INFO] TestAgent: consul server down +49:35.318 [INFO] TestAgent: shutdown complete +49:35.318 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24088 network=tcp +49:35.318 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24088 network=udp +49:35.318 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24105 network=tcp +49:35.818 [INFO] TestAgent: Waiting for endpoints to shut down +49:35.818 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RegisterCheck_Service (0.71s) === CONT TestAgent_DeregisterService_ACLDeny -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:20.686Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:20.723Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:20.738 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:889f0b9e-d8a8-6ad1-9940-9e4a03221530 Address:127.0.0.1:16810}]" -13:20.739 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-889f0b9e-d8a8-6ad1-9940-9e4a03221530.dc1 127.0.0.1 -13:20.739 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-889f0b9e-d8a8-6ad1-9940-9e4a03221530 127.0.0.1 -2021-10-31T23:13:20.739Z [INFO] agent.router: Initializing LAN area manager -13:20.740 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16840 network=udp -13:20.740 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16810 [Follower]" leader= -13:20.740 [INFO] TestAgent.server: Adding LAN server: server="Node-889f0b9e-d8a8-6ad1-9940-9e4a03221530 (Addr: tcp/127.0.0.1:16810) (DC: dc1)" -13:20.740 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-889f0b9e-d8a8-6ad1-9940-9e4a03221530.dc1 area=wan -13:20.740 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16840 network=tcp -13:20.741 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16841 network=tcp -13:20.742 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16811 network=tcp -13:20.742 [INFO] TestAgent: started state syncer -13:20.806 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:20.806 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16810 [Candidate]" term=2 -13:20.806 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:20.806 [DEBUG] TestAgent.server.raft: vote granted: from=889f0b9e-d8a8-6ad1-9940-9e4a03221530 term=2 tally=1 -13:20.806 [INFO] TestAgent.server.raft: election won: tally=1 -13:20.806 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16810 [Leader]" -13:20.806 [INFO] TestAgent.server: cluster leadership acquired -13:20.806 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16810 -13:20.807 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:20.809 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:20.809 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:20.809 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:20.809 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:20.809 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:20.809 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-889f0b9e-d8a8-6ad1-9940-9e4a03221530 -13:20.809 [INFO] TestAgent.server: member joined, marking health alive: member=Node-889f0b9e-d8a8-6ad1-9940-9e4a03221530 -13:20.810 [INFO] TestAgent.server: New leader elected: payload=Node-889f0b9e-d8a8-6ad1-9940-9e4a03221530 -13:20.814 [INFO] TestAgent.server: federation state anti-entropy synced -13:20.854 [INFO] TestAgent: Requesting shutdown -13:20.854 [INFO] TestAgent.server: shutting down server -13:20.854 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:20.854 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:20.854 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:20.854 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:20.854 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:20.854 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:20.854 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:20.855 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:20.855 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:20.855 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:20.855 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:20.855 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:20.855Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:20.855Z [INFO] agent.router.manager: shutting down -13:20.855 [INFO] TestAgent: consul server down -13:20.855 [INFO] TestAgent: shutdown complete -13:20.855 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16840 network=tcp -13:20.855 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16840 network=udp -13:20.855 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16841 network=tcp -13:21.356 [INFO] TestAgent: Waiting for endpoints to shut down -13:21.356 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogServices (0.71s) -=== CONT TestCatalogNodes_MetaFilter -2021-10-31T23:13:20.773Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:20.802Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:20.804 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7907ffe4-8a20-d917-7198-f567ef7586b5 Address:127.0.0.1:16789}]" -13:20.804 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7907ffe4-8a20-d917-7198-f567ef7586b5.dc1 127.0.0.1 -13:20.805 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7907ffe4-8a20-d917-7198-f567ef7586b5 127.0.0.1 -2021-10-31T23:13:20.805Z [INFO] agent.router: Initializing LAN area manager -13:20.805 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16812 network=udp -13:20.805 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16789 [Follower]" leader= -13:20.805 [INFO] TestAgent.server: Adding LAN server: server="Node-7907ffe4-8a20-d917-7198-f567ef7586b5 (Addr: tcp/127.0.0.1:16789) (DC: dc1)" -13:20.805 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7907ffe4-8a20-d917-7198-f567ef7586b5.dc1 area=wan -13:20.805 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16812 network=tcp -13:20.811 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16813 network=tcp -13:20.811 [INFO] TestAgent: started state syncer -13:20.811 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16790 network=tcp -13:20.845 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:20.846 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16789 [Candidate]" term=2 -13:20.846 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:20.846 [DEBUG] TestAgent.server.raft: vote granted: from=7907ffe4-8a20-d917-7198-f567ef7586b5 term=2 tally=1 -13:20.846 [INFO] TestAgent.server.raft: election won: tally=1 -13:20.846 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16789 [Leader]" -13:20.846 [INFO] TestAgent.server: cluster leadership acquired -13:20.846 [INFO] TestAgent.server: New leader elected: payload=Node-7907ffe4-8a20-d917-7198-f567ef7586b5 -13:20.846 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16789 -13:20.848 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:20.850 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:20.850 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:20.850 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:20.850 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:20.850 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:20.850 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7907ffe4-8a20-d917-7198-f567ef7586b5 -13:20.850 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7907ffe4-8a20-d917-7198-f567ef7586b5 -13:20.851 [INFO] TestAgent.server: federation state anti-entropy synced -13:20.930 [INFO] TestAgent: Service entered maintenance mode: service=test -13:20.930 [DEBUG] TestAgent: removed check: check=_service_maintenance:test -13:20.930 [INFO] TestAgent: Service left maintenance mode: service=test -13:20.931 [INFO] TestAgent: Synced node info -13:20.931 [INFO] TestAgent: Synced service: service=test -13:20.931 [INFO] TestAgent: Deregistered check: check=_service_maintenance:test -13:20.931 [INFO] TestAgent: Requesting shutdown -13:20.932 [INFO] TestAgent.server: shutting down server -13:20.932 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:20.932 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:20.932 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:20.932 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:20.932 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:20.932 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:20.932 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:20.932 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:20.932 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:20.932 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:20.932 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:20.932 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:20.932Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:20.932Z [INFO] agent.router.manager: shutting down -13:20.932 [INFO] TestAgent: consul server down -13:20.932 [INFO] TestAgent: shutdown complete -13:20.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16812 network=tcp -13:20.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16812 network=udp -13:20.932 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16813 network=tcp -13:21.433 [INFO] TestAgent: Waiting for endpoints to shut down -13:21.433 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ServiceMaintenance_Disable (0.72s) +2022-12-04T05:49:34.966Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:34.966Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.985Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:34.985Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.988 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:15a7d954-b70a-10fe-e003-b7b79b46da74 Address:127.0.0.1:24109}]" +49:34.989 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24109 [Follower]" leader= +49:34.989 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-15a7d954-b70a-10fe-e003-b7b79b46da74.dc1 127.0.0.1 +49:34.990 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-15a7d954-b70a-10fe-e003-b7b79b46da74 127.0.0.1 +2022-12-04T05:49:34.990Z [INFO] agent.router: Initializing LAN area manager +49:34.990 [INFO] TestAgent.server: Adding LAN server: server="Node-15a7d954-b70a-10fe-e003-b7b79b46da74 (Addr: tcp/127.0.0.1:24109) (DC: dc1)" +49:34.990 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24072 network=udp +49:34.990 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-15a7d954-b70a-10fe-e003-b7b79b46da74.dc1 area=wan +49:34.990 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24072 network=tcp +49:34.991 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24098 network=tcp +49:34.992 [INFO] TestAgent: started state syncer +49:34.992 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24110 network=tcp +49:35.040 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:35.055 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.055 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24109 [Candidate]" term=2 +49:35.055 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:35.055 [DEBUG] TestAgent.server.raft: vote granted: from=15a7d954-b70a-10fe-e003-b7b79b46da74 term=2 tally=1 +49:35.055 [INFO] TestAgent.server.raft: election won: tally=1 +49:35.055 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24109 [Leader]" +49:35.055 [INFO] TestAgent.server: cluster leadership acquired +49:35.055 [INFO] TestAgent.server: initializing acls +49:35.055 [INFO] TestAgent.server: New leader elected: payload=Node-15a7d954-b70a-10fe-e003-b7b79b46da74 +49:35.055 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:35.055 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:35.056 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:35.056 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:35.056 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:35.056 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:35.056 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-15a7d954-b70a-10fe-e003-b7b79b46da74 +49:35.056 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-15a7d954-b70a-10fe-e003-b7b79b46da74.dc1 +49:35.057 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-15a7d954-b70a-10fe-e003-b7b79b46da74.dc1 area=wan +49:35.057 [INFO] TestAgent.server: Updating LAN server: server="Node-15a7d954-b70a-10fe-e003-b7b79b46da74 (Addr: tcp/127.0.0.1:24109) (DC: dc1)" +49:35.057 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:35.058 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:35.058 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:35.058 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:35.058 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:35.058 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:35.058 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-15a7d954-b70a-10fe-e003-b7b79b46da74 +49:35.058 [INFO] TestAgent.server: member joined, marking health alive: member=Node-15a7d954-b70a-10fe-e003-b7b79b46da74 +49:35.059 [INFO] TestAgent.server: federation state anti-entropy synced +49:35.059 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-15a7d954-b70a-10fe-e003-b7b79b46da74 +49:35.253 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.254 [INFO] TestAgent: Synced node info +49:35.338 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-15a7d954-b70a-10fe-e003-b7b79b46da74 +49:35.339 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-15a7d954-b70a-10fe-e003-b7b79b46da74 +49:35.339 [INFO] TestAgent: Requesting shutdown +49:35.339 [INFO] TestAgent.server: shutting down server +49:35.339 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:35.339 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:35.339 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.339 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.339 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.339 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.339 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.339 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:35.339 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:35.339 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.339 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.339 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.339 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.339 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:35.339 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:35.339Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.339Z [INFO] agent.router.manager: shutting down +49:35.339 [INFO] TestAgent: consul server down +49:35.339 [INFO] TestAgent: shutdown complete +49:35.339 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24072 network=tcp +49:35.339 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24072 network=udp +49:35.339 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24098 network=tcp +49:35.840 [INFO] TestAgent: Waiting for endpoints to shut down +49:35.840 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Monitor_ACLDeny (0.89s) === CONT TestAgent_DeregisterService +=== RUN TestAgent_ServiceMaintenance_ACLDeny/no_token +=== RUN TestAgent_ServiceMaintenance_ACLDeny/root_token === RUN TestAgent_ServiceMaintenance_BadRequest/not_enabled === RUN TestAgent_ServiceMaintenance_BadRequest/no_service_id === RUN TestAgent_ServiceMaintenance_BadRequest/bad_service_id -2021-10-31T23:13:20.517Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:20.517Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:20.593Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:20.593Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:20.597 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:694ff2e2-ca55-4b45-5b07-56cef911ed21 Address:127.0.0.1:16838}]" -13:20.597 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16838 [Follower]" leader= -13:20.602 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-694ff2e2-ca55-4b45-5b07-56cef911ed21.dc1 127.0.0.1 -13:20.603 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-694ff2e2-ca55-4b45-5b07-56cef911ed21 127.0.0.1 -2021-10-31T23:13:20.603Z [INFO] agent.router: Initializing LAN area manager -13:20.604 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16819 network=udp -13:20.604 [INFO] TestAgent.server: Adding LAN server: server="Node-694ff2e2-ca55-4b45-5b07-56cef911ed21 (Addr: tcp/127.0.0.1:16838) (DC: dc1)" -13:20.604 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-694ff2e2-ca55-4b45-5b07-56cef911ed21.dc1 area=wan -13:20.604 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16819 network=tcp -13:20.610 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16820 network=tcp -13:20.611 [INFO] TestAgent: started state syncer -13:20.611 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16839 network=tcp -13:20.646 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:20.646 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16838 [Candidate]" term=2 -13:20.646 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:20.646 [DEBUG] TestAgent.server.raft: vote granted: from=694ff2e2-ca55-4b45-5b07-56cef911ed21 term=2 tally=1 -13:20.646 [INFO] TestAgent.server.raft: election won: tally=1 -13:20.646 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16838 [Leader]" -13:20.646 [INFO] TestAgent.server: cluster leadership acquired -13:20.647 [INFO] TestAgent.server: initializing acls -13:20.647 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:20.647 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:20.648 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:20.648 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:20.648 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:20.648 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:20.648 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-694ff2e2-ca55-4b45-5b07-56cef911ed21 -13:20.648 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-694ff2e2-ca55-4b45-5b07-56cef911ed21.dc1 -13:20.649 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:20.649 [INFO] TestAgent.server: New leader elected: payload=Node-694ff2e2-ca55-4b45-5b07-56cef911ed21 -13:20.649 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-694ff2e2-ca55-4b45-5b07-56cef911ed21.dc1 area=wan -13:20.650 [INFO] TestAgent.server: Updating LAN server: server="Node-694ff2e2-ca55-4b45-5b07-56cef911ed21 (Addr: tcp/127.0.0.1:16838) (DC: dc1)" -13:20.651 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:20.651 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:20.651 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:20.651 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:20.651 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:20.651 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-694ff2e2-ca55-4b45-5b07-56cef911ed21 -13:20.651 [INFO] TestAgent.server: member joined, marking health alive: member=Node-694ff2e2-ca55-4b45-5b07-56cef911ed21 -13:20.651 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-694ff2e2-ca55-4b45-5b07-56cef911ed21 -13:20.652 [INFO] TestAgent.server: federation state anti-entropy synced -13:20.990 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:20.991 [INFO] TestAgent: Synced node info -13:20.991 [DEBUG] TestAgent: Node info in sync -13:21.022 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-694ff2e2-ca55-4b45-5b07-56cef911ed21 -13:21.022 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-694ff2e2-ca55-4b45-5b07-56cef911ed21 -13:21.022 [INFO] TestAgent: Service entered maintenance mode: service=test -13:21.022 [DEBUG] TestAgent: Node info in sync -13:21.022 [WARN] TestAgent: Service registration blocked by ACLs: service=test accessorID=00000000-0000-0000-0000-000000000002 -13:21.023 [INFO] TestAgent: Synced check: check=_service_maintenance:test -13:21.023 [INFO] TestAgent: Requesting shutdown -13:21.023 [INFO] TestAgent.server: shutting down server -13:21.023 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:21.023 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.023 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.023 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.023 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.023 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:21.023 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.023 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.023 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:21.023 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:21.023 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.023 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:21.023 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.023 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:21.023 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:21.023Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.023Z [INFO] agent.router.manager: shutting down -13:21.024 [INFO] TestAgent: consul server down -13:21.024 [INFO] TestAgent: shutdown complete -13:21.024 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16819 network=tcp -13:21.024 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16819 network=udp -13:21.024 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16820 network=tcp -13:21.529 [INFO] TestAgent: Waiting for endpoints to shut down -13:21.530 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ServiceMaintenance_ACLDeny (1.06s) - --- PASS: TestAgent_ServiceMaintenance_ACLDeny/no_token (0.00s) - --- PASS: TestAgent_ServiceMaintenance_ACLDeny/root_token (0.00s) -=== CONT TestCatalogNodes -=== RUN TestAgent_DeregisterService_ACLDeny/no_token -=== RUN TestAgent_DeregisterService_ACLDeny/root_token -2021-10-31T23:13:20.898Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:20.923Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:20.924 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2bd80a00-5c00-10b1-1d1b-34c13652ebea Address:127.0.0.1:16873}]" -13:20.925 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2bd80a00-5c00-10b1-1d1b-34c13652ebea.dc1 127.0.0.1 -13:20.925 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2bd80a00-5c00-10b1-1d1b-34c13652ebea 127.0.0.1 -2021-10-31T23:13:20.925Z [INFO] agent.router: Initializing LAN area manager -13:20.926 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16873 [Follower]" leader= -13:20.938 [INFO] TestAgent.server: Adding LAN server: server="Node-2bd80a00-5c00-10b1-1d1b-34c13652ebea (Addr: tcp/127.0.0.1:16873) (DC: dc1)" -13:20.938 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2bd80a00-5c00-10b1-1d1b-34c13652ebea.dc1 area=wan -13:20.938 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16861 network=udp -13:20.938 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16861 network=tcp -13:20.939 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16848 network=tcp -13:20.939 [INFO] TestAgent: started state syncer -13:20.939 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16874 network=tcp -13:20.981 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:20.982 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16873 [Candidate]" term=2 -13:20.982 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:20.982 [DEBUG] TestAgent.server.raft: vote granted: from=2bd80a00-5c00-10b1-1d1b-34c13652ebea term=2 tally=1 -13:20.982 [INFO] TestAgent.server.raft: election won: tally=1 -13:20.982 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16873 [Leader]" -13:20.982 [INFO] TestAgent.server: cluster leadership acquired -13:20.982 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16873 -13:20.983 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:20.985 [INFO] TestAgent.server: New leader elected: payload=Node-2bd80a00-5c00-10b1-1d1b-34c13652ebea -13:20.985 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:20.985 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:20.985 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:20.985 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:20.985 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:20.985 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2bd80a00-5c00-10b1-1d1b-34c13652ebea -13:20.985 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2bd80a00-5c00-10b1-1d1b-34c13652ebea -13:20.986 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.042 [INFO] TestAgent: Service entered maintenance mode: service=test -13:21.043 [INFO] TestAgent: Synced node info -13:21.043 [INFO] TestAgent: Synced service: service=test -13:21.043 [INFO] TestAgent: Synced check: check=_service_maintenance:test -13:21.043 [INFO] TestAgent: Requesting shutdown -13:21.043 [INFO] TestAgent.server: shutting down server -13:21.043 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.043 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.043 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.043 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.043 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.044 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.044 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:21.044 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:21.044 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.044 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:21.044 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.044 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:21.044Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.044Z [INFO] agent.router.manager: shutting down -13:21.044 [INFO] TestAgent: consul server down -13:21.044 [INFO] TestAgent: shutdown complete -13:21.044 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16861 network=tcp -13:21.044 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16861 network=udp -13:21.044 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16848 network=tcp -13:21.545 [INFO] TestAgent: Waiting for endpoints to shut down -13:21.545 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ServiceMaintenance_Enable (0.73s) +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=1)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=1)_compressed=false#01 +2022-12-04T05:49:34.578Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.587Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.589 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a6c2ea86-2566-a9ce-715d-f6ec879d106d Address:127.0.0.1:24014}]" +49:34.589 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24014 [Follower]" leader= +49:34.589 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a6c2ea86-2566-a9ce-715d-f6ec879d106d.dc1 127.0.0.1 +49:34.590 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a6c2ea86-2566-a9ce-715d-f6ec879d106d 127.0.0.1 +2022-12-04T05:49:34.590Z [INFO] agent.router: Initializing LAN area manager +49:34.590 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24044 network=udp +49:34.590 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24044 network=tcp +49:34.590 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a6c2ea86-2566-a9ce-715d-f6ec879d106d.dc1 area=wan +49:34.590 [INFO] TestAgent.server: Adding LAN server: server="Node-a6c2ea86-2566-a9ce-715d-f6ec879d106d (Addr: tcp/127.0.0.1:24014) (DC: dc1)" +49:34.591 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24010 network=tcp +49:34.591 [INFO] TestAgent: started state syncer +49:34.591 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24029 network=tcp +49:34.653 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.653 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24014 [Candidate]" term=2 +49:34.653 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.653 [DEBUG] TestAgent.server.raft: vote granted: from=a6c2ea86-2566-a9ce-715d-f6ec879d106d term=2 tally=1 +49:34.653 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.653 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24014 [Leader]" +49:34.653 [INFO] TestAgent.server: cluster leadership acquired +49:34.653 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24014 +49:34.653 [INFO] TestAgent.server: New leader elected: payload=Node-a6c2ea86-2566-a9ce-715d-f6ec879d106d +49:34.654 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.656 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.656 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.656 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.656 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.656 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.656 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a6c2ea86-2566-a9ce-715d-f6ec879d106d +49:34.656 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a6c2ea86-2566-a9ce-715d-f6ec879d106d +49:34.657 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.746 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.747 [INFO] TestAgent: Synced node info +49:34.747 [DEBUG] TestAgent: Node info in sync +49:35.015 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.015 [DEBUG] TestAgent: Node info in sync +49:35.016 [INFO] TestAgent: Synced service: service=redis +49:35.016 [DEBUG] TestAgent: Check in sync: check=redis-check +49:35.016 [INFO] TestAgent: Synced check: check=node-check +49:35.516 [DEBUG] TestAgent: Registered node: node="Node-level check" +49:35.516 [DEBUG] TestAgent: Registered node: node="Serf Health Status" +49:35.516 [DEBUG] TestAgent: Registered node: node="Service-level check" +49:35.516 [INFO] TestAgent: Sync in progress: iteration=1 +49:35.517 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.517 [DEBUG] TestAgent: Node info in sync +49:35.517 [DEBUG] TestAgent: Service in sync: service=redis +49:35.517 [DEBUG] TestAgent: Check in sync: check=redis-check +49:35.517 [DEBUG] TestAgent: Check in sync: check=node-check +49:35.517 [INFO] TestAgent: Sync in progress: iteration=2 +49:35.517 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.517 [DEBUG] TestAgent: Node info in sync +49:35.517 [DEBUG] TestAgent: Service in sync: service=redis +49:35.517 [DEBUG] TestAgent: Check in sync: check=redis-check +49:35.517 [DEBUG] TestAgent: Check in sync: check=node-check +49:35.517 [INFO] TestAgent: Sync in progress: iteration=3 +49:35.517 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.517 [DEBUG] TestAgent: Node info in sync +49:35.517 [DEBUG] TestAgent: Service in sync: service=redis +49:35.517 [DEBUG] TestAgent: Check in sync: check=redis-check +49:35.517 [DEBUG] TestAgent: Check in sync: check=node-check +49:35.517 [INFO] TestAgent: Sync in progress: iteration=4 +49:35.517 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.517 [DEBUG] TestAgent: Node info in sync +49:35.517 [DEBUG] TestAgent: Service in sync: service=redis +49:35.517 [DEBUG] TestAgent: Check in sync: check=redis-check +49:35.517 [DEBUG] TestAgent: Check in sync: check=node-check +49:35.517 [INFO] TestAgent: Sync in progress: iteration=5 +49:35.517 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.517 [DEBUG] TestAgent: Node info in sync +49:35.517 [DEBUG] TestAgent: Service in sync: service=redis +49:35.517 [DEBUG] TestAgent: Check in sync: check=redis-check +49:35.517 [DEBUG] TestAgent: Check in sync: check=node-check +49:35.517 [INFO] TestAgent: Sync in progress: iteration=6 +49:35.517 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.517 [DEBUG] TestAgent: Node info in sync +49:35.517 [DEBUG] TestAgent: Service in sync: service=redis +49:35.517 [DEBUG] TestAgent: Check in sync: check=redis-check +49:35.517 [DEBUG] TestAgent: Check in sync: check=node-check +49:35.517 [INFO] TestAgent: Sync in progress: iteration=7 +49:35.517 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.517 [DEBUG] TestAgent: Node info in sync +49:35.517 [DEBUG] TestAgent: Service in sync: service=redis +49:35.517 [DEBUG] TestAgent: Check in sync: check=redis-check +49:35.517 [DEBUG] TestAgent: Check in sync: check=node-check +49:35.517 [INFO] TestAgent: Sync in progress: iteration=8 +49:35.517 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.517 [DEBUG] TestAgent: Node info in sync +49:35.517 [DEBUG] TestAgent: Service in sync: service=redis +49:35.517 [DEBUG] TestAgent: Check in sync: check=redis-check +49:35.517 [DEBUG] TestAgent: Check in sync: check=node-check +49:35.517 [INFO] TestAgent: Sync in progress: iteration=9 +49:35.517 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.517 [DEBUG] TestAgent: Node info in sync +49:35.517 [DEBUG] TestAgent: Service in sync: service=redis +49:35.517 [DEBUG] TestAgent: Check in sync: check=redis-check +49:35.517 [DEBUG] TestAgent: Check in sync: check=node-check +49:35.517 [INFO] TestAgent: Sync in progress: iteration=10 +49:35.517 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.517 [DEBUG] TestAgent: Node info in sync +49:35.517 [DEBUG] TestAgent: Service in sync: service=redis +49:35.517 [DEBUG] TestAgent: Check in sync: check=node-check +49:35.517 [DEBUG] TestAgent: Check in sync: check=redis-check +49:35.518 [INFO] TestAgent: Requesting shutdown +49:35.518 [INFO] TestAgent.server: shutting down server +49:35.518 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.518 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.518 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.518 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.518 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.518 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.518 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.518 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:35.518 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.518 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.518 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:35.518Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.518Z [INFO] agent.router.manager: shutting down +49:35.519 [INFO] TestAgent: consul server down +49:35.519 [INFO] TestAgent: shutdown complete +49:35.519 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24044 network=tcp +49:35.519 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24044 network=udp +49:35.519 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24010 network=tcp +49:36.019 [INFO] TestAgent: Waiting for endpoints to shut down +49:36.019 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_IndexChurn (2.70s) + --- PASS: TestAgent_IndexChurn/no_tags (1.25s) + --- PASS: TestAgent_IndexChurn/with_tags (1.45s) === CONT TestAgent_UpdateCheck_ACLDeny -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:20.784Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:20.816Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:20.819 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:618fa219-213e-8c3e-39b8-68f2298c774e Address:127.0.0.1:16796}]" -13:20.819 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-618fa219-213e-8c3e-39b8-68f2298c774e.dc1 127.0.0.1 -13:20.820 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16796 [Follower]" leader= -13:20.831 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-618fa219-213e-8c3e-39b8-68f2298c774e 127.0.0.1 -2021-10-31T23:13:20.831Z [INFO] agent.router: Initializing LAN area manager -13:20.846 [INFO] TestAgent.server: Adding LAN server: server="Node-618fa219-213e-8c3e-39b8-68f2298c774e (Addr: tcp/127.0.0.1:16796) (DC: dc1)" -13:20.846 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-618fa219-213e-8c3e-39b8-68f2298c774e.dc1 area=wan -13:20.846 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16791 network=tcp -13:20.846 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16791 network=udp -13:20.847 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16771 network=tcp -13:20.847 [INFO] TestAgent: started state syncer -13:20.848 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16797 network=tcp -13:20.889 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:20.890 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16796 [Candidate]" term=2 -13:20.890 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:20.890 [DEBUG] TestAgent.server.raft: vote granted: from=618fa219-213e-8c3e-39b8-68f2298c774e term=2 tally=1 -13:20.890 [INFO] TestAgent.server.raft: election won: tally=1 -13:20.890 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16796 [Leader]" -13:20.890 [INFO] TestAgent.server: cluster leadership acquired -13:20.890 [INFO] TestAgent.server: New leader elected: payload=Node-618fa219-213e-8c3e-39b8-68f2298c774e -13:20.890 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16796 -13:20.895 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:20.896 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:20.897 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:20.897 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:20.897 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:20.897 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:20.897 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-618fa219-213e-8c3e-39b8-68f2298c774e -13:20.897 [INFO] TestAgent.server: member joined, marking health alive: member=Node-618fa219-213e-8c3e-39b8-68f2298c774e -13:20.898 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.158 [INFO] TestAgent: Requesting shutdown -13:21.158 [INFO] TestAgent.server: shutting down server -13:21.158 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.158 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.158 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.158 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.158 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.158 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.158 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:21.158 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:21.158 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.158 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:21.158 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.158 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:21.158Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.159Z [INFO] agent.router.manager: shutting down -13:21.159 [INFO] TestAgent: consul server down -13:21.159 [INFO] TestAgent: shutdown complete -13:21.159 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16791 network=tcp -13:21.159 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16791 network=udp -13:21.159 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16771 network=tcp -13:21.661 [INFO] TestAgent: Waiting for endpoints to shut down -13:21.672 [INFO] TestAgent: Endpoints down -=== RUN TestConnectCAConfig/updating_config_with_same_state -2021-10-31T23:13:19.513Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:19.557Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:19.568 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c794c680-57cb-19eb-5c58-19ed57461999 Address:127.0.0.1:16712}]" -13:19.568 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16712 [Follower]" leader= -13:19.569 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c794c680-57cb-19eb-5c58-19ed57461999.dc1 127.0.0.1 -13:19.574 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c794c680-57cb-19eb-5c58-19ed57461999 127.0.0.1 -2021-10-31T23:13:19.574Z [INFO] agent.router: Initializing LAN area manager -13:19.574 [INFO] TestAgent.server: Adding LAN server: server="Node-c794c680-57cb-19eb-5c58-19ed57461999 (Addr: tcp/127.0.0.1:16712) (DC: dc1)" -13:19.574 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c794c680-57cb-19eb-5c58-19ed57461999.dc1 area=wan -13:19.574 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16693 network=udp -13:19.575 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16693 network=tcp -13:19.576 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16680 network=tcp -13:19.576 [INFO] TestAgent: started state syncer -13:19.577 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16713 network=tcp -13:19.614 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:19.614 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16712 [Candidate]" term=2 -13:19.614 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:19.614 [DEBUG] TestAgent.server.raft: vote granted: from=c794c680-57cb-19eb-5c58-19ed57461999 term=2 tally=1 -13:19.614 [INFO] TestAgent.server.raft: election won: tally=1 -13:19.614 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16712 [Leader]" -13:19.615 [INFO] TestAgent.server: cluster leadership acquired -13:19.638 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16712 -13:19.639 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:19.640 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:19.640 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:19.640 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:19.640 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:19.640 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:19.641 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c794c680-57cb-19eb-5c58-19ed57461999 -13:19.641 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c794c680-57cb-19eb-5c58-19ed57461999 -13:19.642 [INFO] TestAgent.server: federation state anti-entropy synced -13:19.642 [INFO] TestAgent.server: New leader elected: payload=Node-c794c680-57cb-19eb-5c58-19ed57461999 -13:19.785 [INFO] TestAgent: Synced node info -13:19.802 [INFO] TestAgent: Synced service: service=monitor -13:19.802 [DEBUG] TestAgent: Check in sync: check=service:monitor -13:19.834 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:19.834 [DEBUG] TestAgent: Node info in sync -13:19.834 [DEBUG] TestAgent: Service in sync: service=monitor -13:19.834 [DEBUG] TestAgent: Check in sync: check=service:monitor -13:19.904 [DEBUG] TestAgent: Node info in sync -13:19.905 [INFO] TestAgent: Synced service: service=monitor -13:19.905 [DEBUG] TestAgent: Check in sync: check=service:monitor -13:21.042 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:21.042 [DEBUG] TestAgent: Node info in sync -13:21.042 [DEBUG] TestAgent: Service in sync: service=monitor -13:21.042 [DEBUG] TestAgent: Check in sync: check=service:monitor -13:21.042 [DEBUG] TestAgent: Node info in sync -13:21.042 [DEBUG] TestAgent: Service in sync: service=monitor -13:21.042 [DEBUG] TestAgent: Check in sync: check=service:monitor -13:21.106 [INFO] TestAgent: Requesting shutdown -13:21.106 [INFO] TestAgent.server: shutting down server -13:21.106 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.106 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.106 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.106 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.106 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.107 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.107 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:21.107 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.107 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:21.107 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.107 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:21.107Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.107Z [INFO] agent.router.manager: shutting down -13:21.107 [INFO] TestAgent: consul server down -13:21.107 [INFO] TestAgent: shutdown complete -13:21.107 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16693 network=tcp -13:21.107 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16693 network=udp -13:21.107 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16680 network=tcp -13:21.609 [INFO] TestAgent: Waiting for endpoints to shut down -13:21.610 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Monitor (2.26s) - --- PASS: TestAgent_Monitor/unknown_log_level (0.00s) - --- PASS: TestAgent_Monitor/stream_unstructured_logs (0.12s) - --- PASS: TestAgent_Monitor/stream_JSON_logs (1.20s) - --- PASS: TestAgent_Monitor/serf_shutdown_logging (0.61s) -=== CONT TestCatalogDatacenters -2021-10-31T23:13:20.962Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:20.976Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:20.982 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2d13a839-ed44-a2d0-d64b-4ca4efbb6677 Address:127.0.0.1:16852}]" -13:20.983 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2d13a839-ed44-a2d0-d64b-4ca4efbb6677.dc1 127.0.0.1 -13:20.983 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2d13a839-ed44-a2d0-d64b-4ca4efbb6677 127.0.0.1 -2021-10-31T23:13:20.983Z [INFO] agent.router: Initializing LAN area manager -13:20.983 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16875 network=udp -13:20.983 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16852 [Follower]" leader= -13:20.984 [INFO] TestAgent.server: Adding LAN server: server="Node-2d13a839-ed44-a2d0-d64b-4ca4efbb6677 (Addr: tcp/127.0.0.1:16852) (DC: dc1)" -13:20.984 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2d13a839-ed44-a2d0-d64b-4ca4efbb6677.dc1 area=wan -13:20.984 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16875 network=tcp -13:20.984 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16876 network=tcp -13:20.984 [INFO] TestAgent: started state syncer -13:20.984 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16853 network=tcp -13:21.041 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:21.041 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16852 [Candidate]" term=2 -13:21.042 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:21.042 [DEBUG] TestAgent.server.raft: vote granted: from=2d13a839-ed44-a2d0-d64b-4ca4efbb6677 term=2 tally=1 -13:21.042 [INFO] TestAgent.server.raft: election won: tally=1 -13:21.042 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16852 [Leader]" -13:21.042 [INFO] TestAgent.server: cluster leadership acquired -13:21.042 [INFO] TestAgent.server: New leader elected: payload=Node-2d13a839-ed44-a2d0-d64b-4ca4efbb6677 -13:21.042 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16852 -13:21.043 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:21.044 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:21.044 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:21.044 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:21.044 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:21.044 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:21.044 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2d13a839-ed44-a2d0-d64b-4ca4efbb6677 -13:21.044 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2d13a839-ed44-a2d0-d64b-4ca4efbb6677 -13:21.045 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.114 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:21.114 [INFO] TestAgent: Synced node info -13:21.223 [INFO] TestAgent: Requesting shutdown -13:21.223 [INFO] TestAgent.server: shutting down server -13:21.223 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.223 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.223 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.223 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.223 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.223 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.223 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.223 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:21.224 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.224 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:21.225 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -2021-10-31T23:13:21.226Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.226Z [INFO] agent.router.manager: shutting down -13:21.226 [INFO] TestAgent: consul server down -13:21.226 [INFO] TestAgent: shutdown complete -13:21.226 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16875 network=tcp -13:21.229 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16875 network=udp -13:21.237 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16876 network=tcp -13:21.738 [INFO] TestAgent: Waiting for endpoints to shut down -13:21.738 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodes_Blocking (0.80s) +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=true#01 +2022-12-04T05:49:35.453Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:35.474Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:35.477 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:25d14b38-091f-6401-453e-63a2d1f606e7 Address:127.0.0.1:24165}]" +49:35.477 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24165 [Follower]" leader= +49:35.478 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-25d14b38-091f-6401-453e-63a2d1f606e7.dc1 127.0.0.1 +49:35.478 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-25d14b38-091f-6401-453e-63a2d1f606e7 127.0.0.1 +2022-12-04T05:49:35.478Z [INFO] agent.router: Initializing LAN area manager +49:35.478 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24153 network=udp +49:35.478 [INFO] TestAgent.server: Adding LAN server: server="Node-25d14b38-091f-6401-453e-63a2d1f606e7 (Addr: tcp/127.0.0.1:24165) (DC: dc1)" +49:35.478 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24153 network=tcp +49:35.478 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-25d14b38-091f-6401-453e-63a2d1f606e7.dc1 area=wan +49:35.479 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24161 network=tcp +49:35.479 [INFO] TestAgent: started state syncer +49:35.479 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24096 network=tcp +49:35.514 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.514 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24165 [Candidate]" term=2 +49:35.514 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:35.514 [DEBUG] TestAgent.server.raft: vote granted: from=25d14b38-091f-6401-453e-63a2d1f606e7 term=2 tally=1 +49:35.514 [INFO] TestAgent.server.raft: election won: tally=1 +49:35.514 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24165 [Leader]" +49:35.514 [INFO] TestAgent.server: cluster leadership acquired +49:35.514 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24165 +49:35.515 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:35.516 [INFO] TestAgent.server: New leader elected: payload=Node-25d14b38-091f-6401-453e-63a2d1f606e7 +49:35.518 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:35.518 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:35.518 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:35.518 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:35.518 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:35.518 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-25d14b38-091f-6401-453e-63a2d1f606e7 +49:35.518 [INFO] TestAgent.server: member joined, marking health alive: member=Node-25d14b38-091f-6401-453e-63a2d1f606e7 +49:35.519 [INFO] TestAgent.server: federation state anti-entropy synced +49:35.612 [INFO] TestAgent: Node entered maintenance mode +49:35.612 [INFO] TestAgent: Synced node info +49:35.613 [INFO] TestAgent: Synced check: check=_node_maintenance +49:35.613 [INFO] TestAgent: Requesting shutdown +49:35.613 [INFO] TestAgent.server: shutting down server +49:35.613 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.613 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.613 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.613 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.613 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.613 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:35.613 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.613 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.613 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.613 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.613 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:35.613 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:35.613Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.613Z [INFO] agent.router.manager: shutting down +49:35.613 [INFO] TestAgent: consul server down +49:35.613 [INFO] TestAgent: shutdown complete +49:35.613 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24153 network=tcp +49:35.613 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24153 network=udp +49:35.613 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24161 network=tcp +49:36.114 [INFO] TestAgent: Waiting for endpoints to shut down +49:36.114 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_NodeMaintenance_Enable (0.68s) === CONT TestAgent_FailCheck_ACLDeny +2022-12-04T05:49:35.337Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:35.347Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:35.349 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9bf17845-8c77-3df9-2527-fb0ea28a438c Address:127.0.0.1:24123}]" +49:35.349 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24123 [Follower]" leader= +49:35.349 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9bf17845-8c77-3df9-2527-fb0ea28a438c.dc1 127.0.0.1 +49:35.350 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9bf17845-8c77-3df9-2527-fb0ea28a438c 127.0.0.1 +2022-12-04T05:49:35.350Z [INFO] agent.router: Initializing LAN area manager +49:35.350 [INFO] TestAgent.server: Adding LAN server: server="Node-9bf17845-8c77-3df9-2527-fb0ea28a438c (Addr: tcp/127.0.0.1:24123) (DC: dc1)" +49:35.350 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9bf17845-8c77-3df9-2527-fb0ea28a438c.dc1 area=wan +49:35.350 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24118 network=udp +49:35.350 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24118 network=tcp +49:35.350 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24119 network=tcp +49:35.350 [INFO] TestAgent: started state syncer +49:35.350 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24152 network=tcp +49:35.412 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.412 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24123 [Candidate]" term=2 +49:35.412 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:35.412 [DEBUG] TestAgent.server.raft: vote granted: from=9bf17845-8c77-3df9-2527-fb0ea28a438c term=2 tally=1 +49:35.412 [INFO] TestAgent.server.raft: election won: tally=1 +49:35.412 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24123 [Leader]" +49:35.413 [INFO] TestAgent.server: cluster leadership acquired +49:35.413 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24123 +49:35.415 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:35.416 [INFO] TestAgent.server: New leader elected: payload=Node-9bf17845-8c77-3df9-2527-fb0ea28a438c +49:35.418 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:35.418 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:35.418 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:35.418 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:35.418 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:35.418 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9bf17845-8c77-3df9-2527-fb0ea28a438c +49:35.418 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9bf17845-8c77-3df9-2527-fb0ea28a438c +49:35.419 [INFO] TestAgent.server: federation state anti-entropy synced +49:35.535 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.536 [INFO] TestAgent: Synced node info +49:35.536 [DEBUG] TestAgent: Node info in sync +49:35.693 [INFO] TestAgent: Node entered maintenance mode +49:35.693 [DEBUG] TestAgent: removed check: check=_node_maintenance +49:35.693 [INFO] TestAgent: Node left maintenance mode +49:35.693 [DEBUG] TestAgent: Node info in sync +49:35.693 [INFO] TestAgent: Deregistered check: check=_node_maintenance +49:35.693 [INFO] TestAgent: Requesting shutdown +49:35.693 [INFO] TestAgent.server: shutting down server +49:35.693 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.693 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.693 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.693 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.693 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.693 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.693 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.693 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:35.693 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.693 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.693 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:35.694Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.694Z [INFO] agent.router.manager: shutting down +49:35.694 [INFO] TestAgent: consul server down +49:35.694 [INFO] TestAgent: shutdown complete +49:35.694 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24118 network=tcp +49:35.694 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24118 network=udp +49:35.694 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24119 network=tcp +49:36.195 [INFO] TestAgent: Waiting for endpoints to shut down +49:36.195 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_NodeMaintenance_Disable (0.88s) +=== CONT TestAgent_FailCheck +=== RUN TestAgent_DeregisterService_ACLDeny/no_token +=== RUN TestAgent_DeregisterService_ACLDeny/root_token +2022-12-04T05:49:35.504Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:35.522Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:35.525 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ff5621c3-30bb-3657-8057-324ab5ea9f83 Address:127.0.0.1:24102}]" +49:35.526 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ff5621c3-30bb-3657-8057-324ab5ea9f83.dc1 127.0.0.1 +49:35.526 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ff5621c3-30bb-3657-8057-324ab5ea9f83 127.0.0.1 +2022-12-04T05:49:35.526Z [INFO] agent.router: Initializing LAN area manager +49:35.527 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24097 network=udp +49:35.527 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24102 [Follower]" leader= +49:35.527 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24097 network=tcp +49:35.527 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ff5621c3-30bb-3657-8057-324ab5ea9f83.dc1 area=wan +49:35.527 [INFO] TestAgent.server: Adding LAN server: server="Node-ff5621c3-30bb-3657-8057-324ab5ea9f83 (Addr: tcp/127.0.0.1:24102) (DC: dc1)" +49:35.528 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24084 network=tcp +49:35.528 [INFO] TestAgent: started state syncer +49:35.528 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24222 network=tcp +49:35.576 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.576 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24102 [Candidate]" term=2 +49:35.576 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:35.576 [DEBUG] TestAgent.server.raft: vote granted: from=ff5621c3-30bb-3657-8057-324ab5ea9f83 term=2 tally=1 +49:35.576 [INFO] TestAgent.server.raft: election won: tally=1 +49:35.576 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24102 [Leader]" +49:35.576 [INFO] TestAgent.server: cluster leadership acquired +49:35.577 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24102 +49:35.577 [INFO] TestAgent.server: New leader elected: payload=Node-ff5621c3-30bb-3657-8057-324ab5ea9f83 +49:35.578 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:35.580 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:35.580 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:35.580 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:35.580 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:35.580 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:35.581 [INFO] TestAgent.server: federation state anti-entropy synced +49:35.585 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ff5621c3-30bb-3657-8057-324ab5ea9f83 +49:35.585 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ff5621c3-30bb-3657-8057-324ab5ea9f83 +49:35.620 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.620 [INFO] TestAgent: Synced node info +49:35.620 [DEBUG] TestAgent: Node info in sync +49:35.794 [INFO] TestAgent: Requesting shutdown +49:35.794 [INFO] TestAgent.server: shutting down server +49:35.794 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.794 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.794 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.794 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.794 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.794 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.794 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.794 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.794 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:35.794 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.794 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:35.800Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.800Z [INFO] agent.router.manager: shutting down +49:35.800 [INFO] TestAgent: consul server down +49:35.800 [INFO] TestAgent: shutdown complete +49:35.800 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24097 network=tcp +49:35.800 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24097 network=udp +49:35.801 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24084 network=tcp +49:36.303 [INFO] TestAgent: Waiting for endpoints to shut down +49:36.303 [INFO] TestAgent: Endpoints down [WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestAgent_UpdateCheck_ACLDeny/no_token -=== RUN TestAgent_UpdateCheck_ACLDeny/root_token -2021-10-31T23:13:20.796Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:20.863Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:20.872 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b3834164-db62-0744-66f7-ab72e7f25066 Address:127.0.0.1:16859}]" -13:20.873 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b3834164-db62-0744-66f7-ab72e7f25066.dc1 127.0.0.1 -13:20.877 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16859 [Follower]" leader= -13:20.879 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b3834164-db62-0744-66f7-ab72e7f25066 127.0.0.1 -2021-10-31T23:13:20.879Z [INFO] agent.router: Initializing LAN area manager -13:20.880 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16798 network=udp -13:20.880 [INFO] TestAgent.server: Adding LAN server: server="Node-b3834164-db62-0744-66f7-ab72e7f25066 (Addr: tcp/127.0.0.1:16859) (DC: dc1)" -13:20.880 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b3834164-db62-0744-66f7-ab72e7f25066.dc1 area=wan -13:20.881 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16798 network=tcp -13:20.882 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16799 network=tcp -13:20.882 [INFO] TestAgent: started state syncer -13:20.882 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16860 network=tcp -13:20.936 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:20.936 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16859 [Candidate]" term=2 -13:20.937 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:20.937 [DEBUG] TestAgent.server.raft: vote granted: from=b3834164-db62-0744-66f7-ab72e7f25066 term=2 tally=1 -13:20.937 [INFO] TestAgent.server.raft: election won: tally=1 -13:20.937 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16859 [Leader]" -13:20.937 [INFO] TestAgent.server: cluster leadership acquired -13:20.937 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16859 -13:20.937 [INFO] TestAgent.server: New leader elected: payload=Node-b3834164-db62-0744-66f7-ab72e7f25066 -13:20.946 [INFO] TestAgent: Synced node info -13:20.947 [DEBUG] TestAgent: Node info in sync -13:20.947 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:20.948 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:20.948 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:20.948 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:20.948 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:20.948 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:20.948 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b3834164-db62-0744-66f7-ab72e7f25066 -13:20.948 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b3834164-db62-0744-66f7-ab72e7f25066 -13:20.949 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.454 [INFO] TestAgent: Requesting shutdown -13:21.454 [INFO] TestAgent.server: shutting down server -13:21.454 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.454 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.454 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.454 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.454 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.454 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.454 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:21.454 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.454 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:21.454 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.454 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:21.454Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.454Z [INFO] agent.router.manager: shutting down -13:21.454 [INFO] TestAgent: consul server down -13:21.454 [INFO] TestAgent: shutdown complete -13:21.454 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16798 network=tcp -13:21.455 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16798 network=udp -13:21.455 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16799 network=tcp -13:21.955 [INFO] TestAgent: Waiting for endpoints to shut down -13:21.955 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodes_DistanceSort (1.19s) -=== CONT TestCatalogDeregister -2021-10-31T23:13:21.105Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:21.130Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:21.132 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7a7a04f2-f0c8-e75f-b053-10fd352b452c Address:127.0.0.1:16831}]" -13:21.133 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7a7a04f2-f0c8-e75f-b053-10fd352b452c.dc1 127.0.0.1 -13:21.133 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16831 [Follower]" leader= -13:21.134 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7a7a04f2-f0c8-e75f-b053-10fd352b452c 127.0.0.1 -2021-10-31T23:13:21.134Z [INFO] agent.router: Initializing LAN area manager -13:21.134 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16854 network=udp -13:21.134 [INFO] TestAgent.server: Adding LAN server: server="Node-7a7a04f2-f0c8-e75f-b053-10fd352b452c (Addr: tcp/127.0.0.1:16831) (DC: dc1)" -13:21.134 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7a7a04f2-f0c8-e75f-b053-10fd352b452c.dc1 area=wan -13:21.134 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16854 network=tcp -13:21.135 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16862 network=tcp -13:21.135 [INFO] TestAgent: started state syncer -13:21.135 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16832 network=tcp -13:21.186 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:21.186 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16831 [Candidate]" term=2 -13:21.187 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:21.187 [DEBUG] TestAgent.server.raft: vote granted: from=7a7a04f2-f0c8-e75f-b053-10fd352b452c term=2 tally=1 -13:21.187 [INFO] TestAgent.server.raft: election won: tally=1 -13:21.187 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16831 [Leader]" -13:21.189 [INFO] TestAgent.server: cluster leadership acquired -13:21.189 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16831 -13:21.190 [INFO] TestAgent.server: New leader elected: payload=Node-7a7a04f2-f0c8-e75f-b053-10fd352b452c -13:21.190 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:21.192 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:21.192 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:21.192 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:21.192 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:21.192 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:21.192 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7a7a04f2-f0c8-e75f-b053-10fd352b452c -13:21.192 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7a7a04f2-f0c8-e75f-b053-10fd352b452c -13:21.193 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.368 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:21.368 [INFO] TestAgent: Synced node info -13:21.522 [INFO] TestAgent: Requesting shutdown -13:21.522 [INFO] TestAgent.server: shutting down server -13:21.522 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.522 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.522 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.522 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.523 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.523 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.523 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:21.523 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.523 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.523 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:21.523 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:21.523Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.523Z [INFO] agent.router.manager: shutting down -13:21.523 [INFO] TestAgent: consul server down -13:21.523 [INFO] TestAgent: shutdown complete -13:21.523 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16854 network=tcp -13:21.523 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16854 network=udp -13:21.523 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16862 network=tcp -13:22.025 [INFO] TestAgent: Waiting for endpoints to shut down -13:22.025 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ServiceMaintenance_BadRequest (0.94s) +--- PASS: TestAgent_NodeMaintenance_BadRequest (0.82s) +=== CONT TestAgent_WarnCheck_ACLDeny +2022-12-04T05:49:35.609Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:35.620Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:35.621 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:91380d2f-ea6a-4056-7db9-566d90bd5595 Address:127.0.0.1:24221}]" +49:35.621 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24221 [Follower]" leader= +49:35.622 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-91380d2f-ea6a-4056-7db9-566d90bd5595.dc1 127.0.0.1 +49:35.622 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-91380d2f-ea6a-4056-7db9-566d90bd5595 127.0.0.1 +2022-12-04T05:49:35.622Z [INFO] agent.router: Initializing LAN area manager +49:35.623 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24188 network=udp +49:35.623 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-91380d2f-ea6a-4056-7db9-566d90bd5595.dc1 area=wan +49:35.623 [INFO] TestAgent.server: Adding LAN server: server="Node-91380d2f-ea6a-4056-7db9-566d90bd5595 (Addr: tcp/127.0.0.1:24221) (DC: dc1)" +49:35.623 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24188 network=tcp +49:35.623 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24217 network=tcp +49:35.623 [INFO] TestAgent: started state syncer +49:35.684 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.684 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24221 [Candidate]" term=2 +49:35.684 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:35.684 [DEBUG] TestAgent.server.raft: vote granted: from=91380d2f-ea6a-4056-7db9-566d90bd5595 term=2 tally=1 +49:35.684 [INFO] TestAgent.server.raft: election won: tally=1 +49:35.684 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24221 [Leader]" +49:35.684 [INFO] TestAgent.server: cluster leadership acquired +49:35.684 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24221 +49:35.686 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:35.688 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:35.688 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:35.688 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:35.688 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:35.688 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:35.688 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-91380d2f-ea6a-4056-7db9-566d90bd5595 +49:35.688 [INFO] TestAgent.server: member joined, marking health alive: member=Node-91380d2f-ea6a-4056-7db9-566d90bd5595 +49:35.688 [INFO] TestAgent.server: New leader elected: payload=Node-91380d2f-ea6a-4056-7db9-566d90bd5595 +49:35.689 [INFO] TestAgent.server: federation state anti-entropy synced +49:35.815 [INFO] TestAgent: Requesting shutdown +49:35.815 [INFO] TestAgent.server: shutting down server +49:35.815 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.815 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.815 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.815 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.815 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.815 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:35.815 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.815 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.815 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:35.815 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.815 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.815 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:35.815Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.815Z [INFO] agent.router.manager: shutting down +49:35.816 [INFO] TestAgent: consul server down +49:35.816 [INFO] TestAgent: shutdown complete +49:35.816 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24188 network=tcp +49:35.816 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24188 network=udp +49:35.816 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24217 network=tcp +49:36.316 [INFO] TestAgent: Waiting for endpoints to shut down +49:36.316 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_Self/normal +2022-12-04T05:49:35.586Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:35.586Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:35.596Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:35.596Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:35.597 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e715967a-fbe1-aa2d-73a4-4bc78070aef3 Address:127.0.0.1:24207}]" +49:35.597 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24207 [Follower]" leader= +49:35.598 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e715967a-fbe1-aa2d-73a4-4bc78070aef3.dc1 127.0.0.1 +49:35.598 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e715967a-fbe1-aa2d-73a4-4bc78070aef3 127.0.0.1 +2022-12-04T05:49:35.598Z [INFO] agent.router: Initializing LAN area manager +49:35.598 [INFO] TestAgent.server: Adding LAN server: server="Node-e715967a-fbe1-aa2d-73a4-4bc78070aef3 (Addr: tcp/127.0.0.1:24207) (DC: dc1)" +49:35.598 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e715967a-fbe1-aa2d-73a4-4bc78070aef3.dc1 area=wan +49:35.599 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24223 network=udp +49:35.599 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24223 network=tcp +49:35.599 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24203 network=tcp +49:35.599 [INFO] TestAgent: started state syncer +49:35.599 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24187 network=tcp +49:35.649 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:35.659 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.659 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24207 [Candidate]" term=2 +49:35.659 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:35.659 [DEBUG] TestAgent.server.raft: vote granted: from=e715967a-fbe1-aa2d-73a4-4bc78070aef3 term=2 tally=1 +49:35.659 [INFO] TestAgent.server.raft: election won: tally=1 +49:35.659 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24207 [Leader]" +49:35.659 [INFO] TestAgent.server: cluster leadership acquired +49:35.659 [INFO] TestAgent.server: New leader elected: payload=Node-e715967a-fbe1-aa2d-73a4-4bc78070aef3 +49:35.659 [INFO] TestAgent.server: initializing acls +49:35.659 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:35.659 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:35.660 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:35.660 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:35.660 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:35.660 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:35.660 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e715967a-fbe1-aa2d-73a4-4bc78070aef3 +49:35.660 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e715967a-fbe1-aa2d-73a4-4bc78070aef3.dc1 +49:35.661 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:35.661 [INFO] TestAgent.server: Updating LAN server: server="Node-e715967a-fbe1-aa2d-73a4-4bc78070aef3 (Addr: tcp/127.0.0.1:24207) (DC: dc1)" +49:35.661 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e715967a-fbe1-aa2d-73a4-4bc78070aef3.dc1 area=wan +49:35.662 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:35.662 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:35.662 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:35.662 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:35.662 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:35.662 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e715967a-fbe1-aa2d-73a4-4bc78070aef3 +49:35.662 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e715967a-fbe1-aa2d-73a4-4bc78070aef3 +49:35.662 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e715967a-fbe1-aa2d-73a4-4bc78070aef3 +49:35.663 [INFO] TestAgent.server: federation state anti-entropy synced +49:35.865 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e715967a-fbe1-aa2d-73a4-4bc78070aef3 +49:35.865 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e715967a-fbe1-aa2d-73a4-4bc78070aef3 +49:35.866 [INFO] TestAgent: Service entered maintenance mode: service=test +49:35.867 [INFO] TestAgent: Synced node info +49:35.867 [WARN] TestAgent: Service registration blocked by ACLs: service=test accessorID=00000000-0000-0000-0000-000000000002 +49:35.867 [INFO] TestAgent: Synced check: check=_service_maintenance:test +49:35.867 [INFO] TestAgent: Requesting shutdown +49:35.867 [INFO] TestAgent.server: shutting down server +49:35.867 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:35.867 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:35.867 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.867 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.867 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.867 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.867 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.867 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:35.867 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.867 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.867 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:35.867 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.867 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:35.867 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:35.867 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.867 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:35.867Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.867Z [INFO] agent.router.manager: shutting down +49:35.868 [INFO] TestAgent: consul server down +49:35.868 [INFO] TestAgent: shutdown complete +49:35.868 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24223 network=tcp +49:35.868 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24223 network=udp +49:35.868 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24203 network=tcp +49:36.368 [INFO] TestAgent: Waiting for endpoints to shut down +49:36.368 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ServiceMaintenance_ACLDeny (0.81s) + --- PASS: TestAgent_ServiceMaintenance_ACLDeny/no_token (0.00s) + --- PASS: TestAgent_ServiceMaintenance_ACLDeny/root_token (0.00s) +=== CONT TestAgent_WarnCheck +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=false#01 +2022-12-04T05:49:35.801Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:35.813Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:35.815 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:015f914f-23f8-22c3-dec8-a84dffc436bb Address:127.0.0.1:24179}]" +49:35.815 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24179 [Follower]" leader= +49:35.815 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-015f914f-23f8-22c3-dec8-a84dffc436bb.dc1 127.0.0.1 +49:35.816 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-015f914f-23f8-22c3-dec8-a84dffc436bb 127.0.0.1 +2022-12-04T05:49:35.816Z [INFO] agent.router: Initializing LAN area manager +49:35.816 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-015f914f-23f8-22c3-dec8-a84dffc436bb.dc1 area=wan +49:35.816 [INFO] TestAgent.server: Adding LAN server: server="Node-015f914f-23f8-22c3-dec8-a84dffc436bb (Addr: tcp/127.0.0.1:24179) (DC: dc1)" +49:35.816 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24167 network=udp +49:35.816 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24167 network=tcp +49:35.817 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24175 network=tcp +49:35.817 [INFO] TestAgent: started state syncer +49:35.817 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24043 network=tcp +49:35.852 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.852 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24179 [Candidate]" term=2 +49:35.852 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:35.852 [DEBUG] TestAgent.server.raft: vote granted: from=015f914f-23f8-22c3-dec8-a84dffc436bb term=2 tally=1 +49:35.852 [INFO] TestAgent.server.raft: election won: tally=1 +49:35.852 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24179 [Leader]" +49:35.852 [INFO] TestAgent.server: cluster leadership acquired +49:35.853 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24179 +49:35.853 [INFO] TestAgent.server: New leader elected: payload=Node-015f914f-23f8-22c3-dec8-a84dffc436bb +49:35.854 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:35.856 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:35.856 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:35.856 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:35.856 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:35.856 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:35.856 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-015f914f-23f8-22c3-dec8-a84dffc436bb +49:35.856 [INFO] TestAgent.server: member joined, marking health alive: member=Node-015f914f-23f8-22c3-dec8-a84dffc436bb +49:35.857 [INFO] TestAgent.server: federation state anti-entropy synced +49:35.937 [INFO] TestAgent: Requesting shutdown +49:35.937 [INFO] TestAgent.server: shutting down server +49:35.937 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.937 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.937 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.937 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.937 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.937 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.937 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.938 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:35.938 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.938 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.938 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:35.938 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:35.938Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.938Z [INFO] agent.router.manager: shutting down +49:35.939 [INFO] TestAgent: consul server down +49:35.939 [INFO] TestAgent: shutdown complete +49:35.939 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24167 network=tcp +49:35.939 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24167 network=udp +49:35.939 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24175 network=tcp +49:36.440 [INFO] TestAgent: Waiting for endpoints to shut down +49:36.440 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ServiceMaintenance_BadRequest (0.66s) --- PASS: TestAgent_ServiceMaintenance_BadRequest/not_enabled (0.00s) --- PASS: TestAgent_ServiceMaintenance_BadRequest/no_service_id (0.00s) --- PASS: TestAgent_ServiceMaintenance_BadRequest/bad_service_id (0.00s) -=== CONT TestAgent_FailCheck +=== CONT TestAgent_PassCheck_ACLDeny +=== RUN TestAgent_UpdateCheck_ACLDeny/no_token +=== RUN TestAgent_UpdateCheck_ACLDeny/root_token +2022-12-04T05:49:35.800Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:35.800Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:35.811Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:35.811Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:35.813 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:248f1f1b-e517-3f10-61d7-0f74e55b046c Address:127.0.0.1:24186}]" +49:35.813 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24186 [Follower]" leader= +49:35.814 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-248f1f1b-e517-3f10-61d7-0f74e55b046c.dc1 127.0.0.1 +49:35.814 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-248f1f1b-e517-3f10-61d7-0f74e55b046c 127.0.0.1 +2022-12-04T05:49:35.814Z [INFO] agent.router: Initializing LAN area manager +49:35.814 [INFO] TestAgent.server: Adding LAN server: server="Node-248f1f1b-e517-3f10-61d7-0f74e55b046c (Addr: tcp/127.0.0.1:24186) (DC: dc1)" +49:35.814 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-248f1f1b-e517-3f10-61d7-0f74e55b046c.dc1 area=wan +49:35.814 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24181 network=udp +49:35.814 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24181 network=tcp +49:35.815 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24182 network=tcp +49:35.815 [INFO] TestAgent: started state syncer +49:35.815 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24166 network=tcp +49:35.864 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.864 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24186 [Candidate]" term=2 +49:35.864 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:35.864 [DEBUG] TestAgent.server.raft: vote granted: from=248f1f1b-e517-3f10-61d7-0f74e55b046c term=2 tally=1 +49:35.864 [INFO] TestAgent.server.raft: election won: tally=1 +49:35.864 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24186 [Leader]" +49:35.864 [INFO] TestAgent.server: cluster leadership acquired +49:35.864 [INFO] TestAgent.server: initializing acls +49:35.864 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:35.864 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:35.865 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:35.865 [INFO] TestAgent.server: New leader elected: payload=Node-248f1f1b-e517-3f10-61d7-0f74e55b046c +49:35.865 [INFO] TestAgent.server: initializing acls +49:35.865 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:35.865 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:35.865 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:35.865 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:35.865 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-248f1f1b-e517-3f10-61d7-0f74e55b046c +49:35.866 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-248f1f1b-e517-3f10-61d7-0f74e55b046c.dc1 +49:35.866 [INFO] TestAgent.server: Updating LAN server: server="Node-248f1f1b-e517-3f10-61d7-0f74e55b046c (Addr: tcp/127.0.0.1:24186) (DC: dc1)" +49:35.866 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-248f1f1b-e517-3f10-61d7-0f74e55b046c.dc1 area=wan +49:35.866 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:35.866 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +49:35.866 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-248f1f1b-e517-3f10-61d7-0f74e55b046c +49:35.866 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:35.866 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-248f1f1b-e517-3f10-61d7-0f74e55b046c.dc1 +49:35.866 [INFO] TestAgent.server: Updating LAN server: server="Node-248f1f1b-e517-3f10-61d7-0f74e55b046c (Addr: tcp/127.0.0.1:24186) (DC: dc1)" +49:35.867 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-248f1f1b-e517-3f10-61d7-0f74e55b046c.dc1 area=wan +49:35.868 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:35.868 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:35.868 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:35.868 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:35.868 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:35.868 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-248f1f1b-e517-3f10-61d7-0f74e55b046c +49:35.868 [INFO] TestAgent.server: member joined, marking health alive: member=Node-248f1f1b-e517-3f10-61d7-0f74e55b046c +49:35.869 [INFO] TestAgent.server: federation state anti-entropy synced +49:35.869 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-248f1f1b-e517-3f10-61d7-0f74e55b046c +49:35.869 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-248f1f1b-e517-3f10-61d7-0f74e55b046c +49:35.888 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-248f1f1b-e517-3f10-61d7-0f74e55b046c +49:35.888 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-248f1f1b-e517-3f10-61d7-0f74e55b046c +49:35.891 [INFO] TestAgent: Updated agent's ACL token: token=agent +49:35.991 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +49:35.992 [INFO] TestAgent: Synced node info +49:35.992 [DEBUG] TestAgent: Node info in sync +49:35.996 [INFO] TestAgent: Requesting shutdown +49:35.996 [INFO] TestAgent.server: shutting down server +49:35.996 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.996 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.996 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:35.996 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:35.996 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.996 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.996 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.996 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.996 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.996 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.996 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.996 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:35.996 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:35.996 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:35.996 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:35.997Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.997Z [INFO] agent.router.manager: shutting down +49:35.997 [INFO] TestAgent: consul server down +49:35.997 [INFO] TestAgent: shutdown complete +49:35.997 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24181 network=tcp +49:35.997 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24181 network=udp +49:35.997 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24182 network=tcp +49:36.498 [INFO] TestAgent: Waiting for endpoints to shut down +49:36.498 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_TokenTriggersFullSync/acl_token +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestAgent_FailCheck_ACLDeny/no_token === RUN TestAgent_FailCheck_ACLDeny/root_token -2021-10-31T23:13:21.293Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:21.293Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:21.319Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:21.319Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:21.322 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fd71fa99-bf11-d31f-797c-7ab1e92975ee Address:127.0.0.1:16880}]" -13:21.323 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fd71fa99-bf11-d31f-797c-7ab1e92975ee.dc1 127.0.0.1 -13:21.323 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fd71fa99-bf11-d31f-797c-7ab1e92975ee 127.0.0.1 -2021-10-31T23:13:21.323Z [INFO] agent.router: Initializing LAN area manager -13:21.324 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16903 network=udp -13:21.324 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16880 [Follower]" leader= -13:21.324 [INFO] TestAgent.server: Adding LAN server: server="Node-fd71fa99-bf11-d31f-797c-7ab1e92975ee (Addr: tcp/127.0.0.1:16880) (DC: dc1)" -13:21.324 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fd71fa99-bf11-d31f-797c-7ab1e92975ee.dc1 area=wan -13:21.324 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16903 network=tcp -13:21.325 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16911 network=tcp -13:21.325 [INFO] TestAgent: started state syncer -13:21.325 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16881 network=tcp -13:21.360 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:21.360 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16880 [Candidate]" term=2 -13:21.360 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:21.360 [DEBUG] TestAgent.server.raft: vote granted: from=fd71fa99-bf11-d31f-797c-7ab1e92975ee term=2 tally=1 -13:21.360 [INFO] TestAgent.server.raft: election won: tally=1 -13:21.360 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16880 [Leader]" -13:21.360 [INFO] TestAgent.server: cluster leadership acquired -13:21.360 [INFO] TestAgent.server: initializing acls -13:21.361 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:21.361 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:21.361 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:21.361 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:21.361 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:21.361 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:21.361 [INFO] TestAgent.server: New leader elected: payload=Node-fd71fa99-bf11-d31f-797c-7ab1e92975ee -13:21.362 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-fd71fa99-bf11-d31f-797c-7ab1e92975ee -13:21.362 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-fd71fa99-bf11-d31f-797c-7ab1e92975ee.dc1 -13:21.362 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:21.362 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-fd71fa99-bf11-d31f-797c-7ab1e92975ee.dc1 area=wan -13:21.363 [INFO] TestAgent.server: Updating LAN server: server="Node-fd71fa99-bf11-d31f-797c-7ab1e92975ee (Addr: tcp/127.0.0.1:16880) (DC: dc1)" -13:21.364 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:21.364 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:21.364 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:21.364 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:21.364 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:21.364 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fd71fa99-bf11-d31f-797c-7ab1e92975ee -13:21.364 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fd71fa99-bf11-d31f-797c-7ab1e92975ee -13:21.364 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fd71fa99-bf11-d31f-797c-7ab1e92975ee -13:21.364 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.471 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:21.472 [INFO] TestAgent: Synced node info -13:21.534 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fd71fa99-bf11-d31f-797c-7ab1e92975ee -13:21.542 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fd71fa99-bf11-d31f-797c-7ab1e92975ee -13:21.542 [DEBUG] TestAgent: removed service: service=test -13:21.542 [DEBUG] TestAgent: Node info in sync -13:21.542 [INFO] TestAgent: Deregistered service: service=test -13:21.542 [INFO] TestAgent: Requesting shutdown -13:21.542 [INFO] TestAgent.server: shutting down server -13:21.542 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.542 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.542 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:21.542 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:21.542 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.542 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.542 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.542 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.542 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:21.542 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:21.542 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.542 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:21.542 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:21.542 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.542 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:21.542Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.542Z [INFO] agent.router.manager: shutting down -13:21.543 [INFO] TestAgent: consul server down -13:21.543 [INFO] TestAgent: shutdown complete -13:21.543 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16903 network=tcp -13:21.543 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16903 network=udp -13:21.543 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16911 network=tcp -13:22.043 [INFO] TestAgent: Waiting for endpoints to shut down -13:22.043 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_DeregisterService_ACLDeny (0.77s) - --- PASS: TestAgent_DeregisterService_ACLDeny/no_token (0.00s) - --- PASS: TestAgent_DeregisterService_ACLDeny/root_token (0.00s) -=== CONT TestCatalogRegister_Service_InvalidAddress -2021-10-31T23:13:21.158Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:21.158Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:21.209Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:21.209Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:21.219 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5b4adf07-4c88-a625-9027-edb4f8b6b887 Address:127.0.0.1:16894}]" -13:21.220 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16894 [Follower]" leader= -13:21.220 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5b4adf07-4c88-a625-9027-edb4f8b6b887.dc1 127.0.0.1 -13:21.221 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5b4adf07-4c88-a625-9027-edb4f8b6b887 127.0.0.1 -2021-10-31T23:13:21.221Z [INFO] agent.router: Initializing LAN area manager -13:21.225 [INFO] TestAgent.server: Adding LAN server: server="Node-5b4adf07-4c88-a625-9027-edb4f8b6b887 (Addr: tcp/127.0.0.1:16894) (DC: dc1)" -13:21.225 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5b4adf07-4c88-a625-9027-edb4f8b6b887.dc1 area=wan -13:21.226 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16868 network=udp -13:21.226 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16868 network=tcp -13:21.227 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16869 network=tcp -13:21.228 [INFO] TestAgent: started state syncer -13:21.228 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16895 network=tcp -13:21.277 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:21.277 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16894 [Candidate]" term=2 -13:21.278 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:21.278 [DEBUG] TestAgent.server.raft: vote granted: from=5b4adf07-4c88-a625-9027-edb4f8b6b887 term=2 tally=1 -13:21.278 [INFO] TestAgent.server.raft: election won: tally=1 -13:21.278 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16894 [Leader]" -13:21.277 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:21.278 [INFO] TestAgent.server: cluster leadership acquired -13:21.278 [INFO] TestAgent.server: initializing acls -13:21.278 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:21.278 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:21.279 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:21.279 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:21.279 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:21.279 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:21.279 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5b4adf07-4c88-a625-9027-edb4f8b6b887 -13:21.279 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5b4adf07-4c88-a625-9027-edb4f8b6b887.dc1 -13:21.280 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:21.281 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:21.281 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:21.281 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:21.281 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:21.281 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:21.281 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5b4adf07-4c88-a625-9027-edb4f8b6b887 -13:21.281 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5b4adf07-4c88-a625-9027-edb4f8b6b887 -13:21.281 [INFO] TestAgent.server: New leader elected: payload=Node-5b4adf07-4c88-a625-9027-edb4f8b6b887 -13:21.281 [INFO] TestAgent.server: Updating LAN server: server="Node-5b4adf07-4c88-a625-9027-edb4f8b6b887 (Addr: tcp/127.0.0.1:16894) (DC: dc1)" -13:21.281 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5b4adf07-4c88-a625-9027-edb4f8b6b887 -13:21.281 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5b4adf07-4c88-a625-9027-edb4f8b6b887.dc1 area=wan -13:21.282 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.405 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5b4adf07-4c88-a625-9027-edb4f8b6b887 -13:21.406 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5b4adf07-4c88-a625-9027-edb4f8b6b887 -13:21.407 [INFO] TestAgent: Updated agent's ACL token: token=acl_token -13:21.534 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -13:21.535 [INFO] TestAgent: Synced node info -13:21.550 [INFO] TestAgent: Requesting shutdown -13:21.550 [INFO] TestAgent.server: shutting down server -13:21.550 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.550 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.550 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:21.550 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:21.550 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.550 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.550 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.550 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.550 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:21.550 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:21.550 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.550 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:21.550 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:21.550 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.550 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:21.550Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.550Z [INFO] agent.router.manager: shutting down -13:21.550 [INFO] TestAgent: consul server down -13:21.550 [INFO] TestAgent: shutdown complete -13:21.550 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16868 network=tcp -13:21.550 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16868 network=udp -13:21.550 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16869 network=tcp -13:22.053 [INFO] TestAgent: Waiting for endpoints to shut down -13:22.063 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_TokenTriggersFullSync/default -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:21.261Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:21.294Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:21.296 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:69adf921-8de5-1b14-e9c8-604b7ad54b85 Address:127.0.0.1:16901}]" -13:21.297 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-69adf921-8de5-1b14-e9c8-604b7ad54b85.dc1 127.0.0.1 -13:21.297 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16901 [Follower]" leader= -13:21.298 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-69adf921-8de5-1b14-e9c8-604b7ad54b85 127.0.0.1 -2021-10-31T23:13:21.298Z [INFO] agent.router: Initializing LAN area manager -13:21.298 [INFO] TestAgent.server: Adding LAN server: server="Node-69adf921-8de5-1b14-e9c8-604b7ad54b85 (Addr: tcp/127.0.0.1:16901) (DC: dc1)" -13:21.298 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-69adf921-8de5-1b14-e9c8-604b7ad54b85.dc1 area=wan -13:21.305 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16896 network=udp -13:21.305 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16896 network=tcp -13:21.306 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16897 network=tcp -13:21.306 [INFO] TestAgent: started state syncer -13:21.306 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16902 network=tcp -13:21.364 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:21.364 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16901 [Candidate]" term=2 -13:21.364 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:21.364 [DEBUG] TestAgent.server.raft: vote granted: from=69adf921-8de5-1b14-e9c8-604b7ad54b85 term=2 tally=1 -13:21.364 [INFO] TestAgent.server.raft: election won: tally=1 -13:21.364 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16901 [Leader]" -13:21.364 [INFO] TestAgent.server: cluster leadership acquired -13:21.364 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16901 -13:21.365 [INFO] TestAgent.server: New leader elected: payload=Node-69adf921-8de5-1b14-e9c8-604b7ad54b85 -13:21.365 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:21.367 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:21.367 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:21.367 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:21.367 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:21.367 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:21.367 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-69adf921-8de5-1b14-e9c8-604b7ad54b85 -13:21.367 [INFO] TestAgent.server: member joined, marking health alive: member=Node-69adf921-8de5-1b14-e9c8-604b7ad54b85 -13:21.368 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.378 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:21.378 [INFO] TestAgent: Synced node info -13:21.611 [INFO] TestAgent: Requesting shutdown -13:21.611 [INFO] TestAgent.server: shutting down server -13:21.611 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.611 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.611 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.611 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.611 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.611 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.611 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:21.611 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.611 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:21.611 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.611 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:21.612Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.612Z [INFO] agent.router.manager: shutting down -13:21.612 [INFO] TestAgent: consul server down -13:21.612 [INFO] TestAgent: shutdown complete -13:21.612 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16896 network=tcp -13:21.612 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16896 network=udp -13:21.612 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16897 network=tcp -13:22.129 [INFO] TestAgent: Waiting for endpoints to shut down -13:22.130 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodes_Filter (0.92s) -=== CONT TestAgent_WarnCheck_ACLDeny -2021-10-31T23:13:21.379Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:21.405Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:21.411 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5bb572e9-ec42-89a0-205a-4f0e9c3361e7 Address:127.0.0.1:16887}]" -13:21.411 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5bb572e9-ec42-89a0-205a-4f0e9c3361e7.dc1 127.0.0.1 -13:21.412 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5bb572e9-ec42-89a0-205a-4f0e9c3361e7 127.0.0.1 -2021-10-31T23:13:21.412Z [INFO] agent.router: Initializing LAN area manager -13:21.412 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16882 network=udp -13:21.412 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16887 [Follower]" leader= -13:21.412 [INFO] TestAgent.server: Adding LAN server: server="Node-5bb572e9-ec42-89a0-205a-4f0e9c3361e7 (Addr: tcp/127.0.0.1:16887) (DC: dc1)" -13:21.412 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5bb572e9-ec42-89a0-205a-4f0e9c3361e7.dc1 area=wan -13:21.412 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16882 network=tcp -13:21.413 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16883 network=tcp -13:21.413 [INFO] TestAgent: started state syncer -13:21.413 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16888 network=tcp -13:21.465 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:21.465 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16887 [Candidate]" term=2 -13:21.466 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:21.466 [DEBUG] TestAgent.server.raft: vote granted: from=5bb572e9-ec42-89a0-205a-4f0e9c3361e7 term=2 tally=1 -13:21.466 [INFO] TestAgent.server.raft: election won: tally=1 -13:21.466 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16887 [Leader]" -13:21.466 [INFO] TestAgent.server: cluster leadership acquired -13:21.466 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16887 -13:21.467 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:21.469 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:21.469 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:21.469 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:21.469 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:21.469 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:21.469 [INFO] TestAgent.server: New leader elected: payload=Node-5bb572e9-ec42-89a0-205a-4f0e9c3361e7 -13:21.470 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.478 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5bb572e9-ec42-89a0-205a-4f0e9c3361e7 -13:21.478 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5bb572e9-ec42-89a0-205a-4f0e9c3361e7 -13:21.670 [INFO] TestAgent: Requesting shutdown -13:21.670 [INFO] TestAgent.server: shutting down server -13:21.670 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.670 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.670 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.670 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.671 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.671 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.671 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:21.671 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:21.672 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.672 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:21.672 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.672 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:21.672Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.672Z [INFO] agent.router.manager: shutting down -13:21.672 [INFO] TestAgent: consul server down -13:21.672 [INFO] TestAgent: shutdown complete -13:21.672 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16882 network=tcp -13:21.672 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16882 network=udp -13:21.672 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16883 network=tcp -13:22.172 [INFO] TestAgent: Waiting for endpoints to shut down -13:22.172 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodes_MetaFilter (0.82s) -=== CONT TestAgent_WarnCheck -2021-10-31T23:13:21.533Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:21.605Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:21.621 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e7258325-e43c-b64a-8455-1882cb0c535e Address:127.0.0.1:16845}]" -13:21.626 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e7258325-e43c-b64a-8455-1882cb0c535e.dc1 127.0.0.1 -13:21.627 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e7258325-e43c-b64a-8455-1882cb0c535e 127.0.0.1 -2021-10-31T23:13:21.627Z [INFO] agent.router: Initializing LAN area manager -13:21.627 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16889 network=udp -13:21.628 [INFO] TestAgent.server: Adding LAN server: server="Node-e7258325-e43c-b64a-8455-1882cb0c535e (Addr: tcp/127.0.0.1:16845) (DC: dc1)" -13:21.628 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e7258325-e43c-b64a-8455-1882cb0c535e.dc1 area=wan -13:21.628 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16889 network=tcp -13:21.629 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16890 network=tcp -13:21.629 [INFO] TestAgent: started state syncer -13:21.629 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16846 network=tcp -13:21.633 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16845 [Follower]" leader= -13:21.681 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:21.681 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16845 [Candidate]" term=2 -13:21.682 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:21.682 [DEBUG] TestAgent.server.raft: vote granted: from=e7258325-e43c-b64a-8455-1882cb0c535e term=2 tally=1 -13:21.682 [INFO] TestAgent.server.raft: election won: tally=1 -13:21.682 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16845 [Leader]" -13:21.682 [INFO] TestAgent.server: cluster leadership acquired -13:21.682 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16845 -13:21.683 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:21.685 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:21.685 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:21.685 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:21.685 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:21.685 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:21.685 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e7258325-e43c-b64a-8455-1882cb0c535e -13:21.685 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e7258325-e43c-b64a-8455-1882cb0c535e -13:21.685 [INFO] TestAgent.server: New leader elected: payload=Node-e7258325-e43c-b64a-8455-1882cb0c535e -13:21.690 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.835 [DEBUG] TestAgent: removed service: service=test -13:21.835 [INFO] TestAgent: Synced node info -13:21.835 [INFO] TestAgent: Deregistered service: service=test -13:21.835 [INFO] TestAgent: Requesting shutdown -13:21.836 [INFO] TestAgent.server: shutting down server -13:21.836 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.836 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.836 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.836 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.836 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.836 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.836 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:21.836 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:21.836 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.836 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:21.836 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.836 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:21.836Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.836Z [INFO] agent.router.manager: shutting down -13:21.836 [INFO] TestAgent: consul server down -13:21.836 [INFO] TestAgent: shutdown complete -13:21.836 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16889 network=tcp -13:21.836 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16889 network=udp -13:21.836 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16890 network=tcp -13:22.337 [INFO] TestAgent: Waiting for endpoints to shut down -13:22.337 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_DeregisterService (0.92s) -=== CONT TestAgent_JoinWAN_viaMeshGateway -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:21.695Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:21.743Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:21.754 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e4782b5e-f6d1-317c-22cd-cbbf05c6bbad Address:127.0.0.1:16954}]" -13:21.755 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e4782b5e-f6d1-317c-22cd-cbbf05c6bbad.dc1 127.0.0.1 -13:21.756 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e4782b5e-f6d1-317c-22cd-cbbf05c6bbad 127.0.0.1 -2021-10-31T23:13:21.756Z [INFO] agent.router: Initializing LAN area manager -13:21.756 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16949 network=udp -13:21.756 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16954 [Follower]" leader= -13:21.757 [INFO] TestAgent.server: Adding LAN server: server="Node-e4782b5e-f6d1-317c-22cd-cbbf05c6bbad (Addr: tcp/127.0.0.1:16954) (DC: dc1)" -13:21.757 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e4782b5e-f6d1-317c-22cd-cbbf05c6bbad.dc1 area=wan -13:21.757 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16949 network=tcp -13:21.759 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16950 network=tcp -13:21.759 [INFO] TestAgent: started state syncer -13:21.759 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16955 network=tcp -13:21.798 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:21.798 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16954 [Candidate]" term=2 -13:21.798 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:21.798 [DEBUG] TestAgent.server.raft: vote granted: from=e4782b5e-f6d1-317c-22cd-cbbf05c6bbad term=2 tally=1 -13:21.798 [INFO] TestAgent.server.raft: election won: tally=1 -13:21.798 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16954 [Leader]" -13:21.798 [INFO] TestAgent.server: cluster leadership acquired -13:21.816 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16954 -13:21.817 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:21.817 [INFO] TestAgent.server: New leader elected: payload=Node-e4782b5e-f6d1-317c-22cd-cbbf05c6bbad -13:21.819 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:21.819 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:21.819 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:21.819 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:21.819 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:21.819 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e4782b5e-f6d1-317c-22cd-cbbf05c6bbad -13:21.819 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e4782b5e-f6d1-317c-22cd-cbbf05c6bbad -13:21.820 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.905 [INFO] TestAgent.server.connect: CA provider config updated -13:21.905 [INFO] TestAgent: Requesting shutdown -13:21.905 [INFO] TestAgent.server: shutting down server -13:21.905 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.905 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.905 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.905 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.905 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.905 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.905 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.905 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.905 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:21.905 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:21.905 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:21.905 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2021-10-31T23:13:21.906Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.906Z [INFO] agent.router.manager: shutting down -13:21.906 [INFO] TestAgent: consul server down -13:21.906 [INFO] TestAgent: shutdown complete -13:21.906 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16949 network=tcp -13:21.906 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16949 network=udp -13:21.906 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16950 network=tcp -13:22.407 [INFO] TestAgent: Waiting for endpoints to shut down -13:22.407 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectCAConfig (5.72s) - --- PASS: TestConnectCAConfig/basic (1.01s) - --- PASS: TestConnectCAConfig/basic_with_IntermediateCertTTL (1.11s) - --- PASS: TestConnectCAConfig/force_without_cross_sign_CamelCase (0.88s) - --- PASS: TestConnectCAConfig/force_without_cross_sign_snake_case (1.06s) - --- PASS: TestConnectCAConfig/setting_state_fails (0.94s) - --- PASS: TestConnectCAConfig/updating_config_with_same_state (0.74s) -=== CONT TestAgent_PassCheck_ACLDeny -2021-10-31T23:13:21.603Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:21.603Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:21.671Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:21.671Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:21.682 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b825f43d-220b-6287-8412-17f893670ef0 Address:127.0.0.1:16908}]" -13:21.683 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b825f43d-220b-6287-8412-17f893670ef0.dc1 127.0.0.1 -13:21.683 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b825f43d-220b-6287-8412-17f893670ef0 127.0.0.1 -2021-10-31T23:13:21.683Z [INFO] agent.router: Initializing LAN area manager -13:21.684 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16931 network=udp -13:21.684 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16908 [Follower]" leader= -13:21.684 [INFO] TestAgent.server: Adding LAN server: server="Node-b825f43d-220b-6287-8412-17f893670ef0 (Addr: tcp/127.0.0.1:16908) (DC: dc1)" -13:21.684 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b825f43d-220b-6287-8412-17f893670ef0.dc1 area=wan -13:21.684 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16931 network=tcp -13:21.685 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16918 network=tcp -13:21.685 [INFO] TestAgent: started state syncer -13:21.685 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16909 network=tcp -13:21.734 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:21.734 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16908 [Candidate]" term=2 -13:21.734 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:21.734 [DEBUG] TestAgent.server.raft: vote granted: from=b825f43d-220b-6287-8412-17f893670ef0 term=2 tally=1 -13:21.734 [INFO] TestAgent.server.raft: election won: tally=1 -13:21.734 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16908 [Leader]" -13:21.734 [INFO] TestAgent.server: cluster leadership acquired -13:21.734 [INFO] TestAgent.server: initializing acls -13:21.735 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:21.735 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:21.736 [INFO] TestAgent.server: New leader elected: payload=Node-b825f43d-220b-6287-8412-17f893670ef0 -13:21.736 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:21.737 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:21.737 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:21.737 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:21.746 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b825f43d-220b-6287-8412-17f893670ef0 -13:21.746 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b825f43d-220b-6287-8412-17f893670ef0.dc1 -13:21.747 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:21.749 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b825f43d-220b-6287-8412-17f893670ef0.dc1 area=wan -13:21.750 [INFO] TestAgent.server: Updating LAN server: server="Node-b825f43d-220b-6287-8412-17f893670ef0 (Addr: tcp/127.0.0.1:16908) (DC: dc1)" -13:21.758 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:21.758 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:21.758 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:21.758 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:21.758 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:21.758 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b825f43d-220b-6287-8412-17f893670ef0 -13:21.758 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b825f43d-220b-6287-8412-17f893670ef0 -13:21.771 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.771 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b825f43d-220b-6287-8412-17f893670ef0 -13:21.775 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:21.776 [INFO] TestAgent: Synced node info -13:21.906 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b825f43d-220b-6287-8412-17f893670ef0 -13:21.906 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b825f43d-220b-6287-8412-17f893670ef0 -13:21.906 [DEBUG] TestAgent: Check status updated: check=test status=passing -13:21.906 [DEBUG] TestAgent: Node info in sync -13:21.906 [WARN] TestAgent: Check registration blocked by ACLs: check=test accessorID=00000000-0000-0000-0000-000000000002 -13:21.906 [INFO] TestAgent: Requesting shutdown -13:21.906 [INFO] TestAgent.server: shutting down server -13:21.906 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.907 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:21.907 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:21.907 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.907 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.907 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.907 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.907 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.907 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:21.907 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:21.907 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.907 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:21.907 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:21.907 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.907 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:21.907Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.907Z [INFO] agent.router.manager: shutting down -13:21.907 [INFO] TestAgent: consul server down -13:21.907 [INFO] TestAgent: shutdown complete -13:21.907 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16931 network=tcp -13:21.907 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16931 network=udp -13:21.907 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16918 network=tcp -13:22.409 [INFO] TestAgent: Waiting for endpoints to shut down -13:22.409 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_UpdateCheck_ACLDeny (0.86s) - --- PASS: TestAgent_UpdateCheck_ACLDeny/no_token (0.00s) - --- PASS: TestAgent_UpdateCheck_ACLDeny/root_token (0.00s) -=== CONT TestAgent_RerouteNewHTTPChecks -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=255)_compressed=false === RUN TestAgent_WarnCheck_ACLDeny/no_token -2021-10-31T23:13:21.813Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:21.838Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:21.840 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ca1cc39b-c528-fc44-4364-f21ce3bafa26 Address:127.0.0.1:16989}]" -13:21.840 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ca1cc39b-c528-fc44-4364-f21ce3bafa26.dc1 127.0.0.1 -13:21.841 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ca1cc39b-c528-fc44-4364-f21ce3bafa26 127.0.0.1 -2021-10-31T23:13:21.841Z [INFO] agent.router: Initializing LAN area manager -13:21.841 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16956 network=udp -13:21.841 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16989 [Follower]" leader= -13:21.841 [INFO] TestAgent.server: Adding LAN server: server="Node-ca1cc39b-c528-fc44-4364-f21ce3bafa26 (Addr: tcp/127.0.0.1:16989) (DC: dc1)" -13:21.841 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ca1cc39b-c528-fc44-4364-f21ce3bafa26.dc1 area=wan -13:21.842 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16956 network=tcp -13:21.842 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16957 network=tcp -13:21.842 [INFO] TestAgent: started state syncer -13:21.842 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16990 network=tcp -13:21.889 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:21.889 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16989 [Candidate]" term=2 -13:21.890 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:21.890 [DEBUG] TestAgent.server.raft: vote granted: from=ca1cc39b-c528-fc44-4364-f21ce3bafa26 term=2 tally=1 -13:21.890 [INFO] TestAgent.server.raft: election won: tally=1 -13:21.890 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16989 [Leader]" -13:21.893 [INFO] TestAgent.server: cluster leadership acquired -13:21.894 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16989 -13:21.895 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:21.897 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:21.897 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:21.897 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:21.897 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:21.897 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:21.897 [INFO] TestAgent.server: New leader elected: payload=Node-ca1cc39b-c528-fc44-4364-f21ce3bafa26 -13:21.897 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ca1cc39b-c528-fc44-4364-f21ce3bafa26 -13:21.897 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ca1cc39b-c528-fc44-4364-f21ce3bafa26 -13:21.898 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.962 [INFO] TestAgent: Requesting shutdown -13:21.962 [INFO] TestAgent.server: shutting down server -13:21.962 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.962 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.962 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.962 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.962 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.962 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.962 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:21.962 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:21.962 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.962 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:21.962 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.962 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:21.962Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.962Z [INFO] agent.router.manager: shutting down -13:21.962 [INFO] TestAgent: consul server down -13:21.962 [INFO] TestAgent: shutdown complete -13:21.962 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16956 network=tcp -13:21.962 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16956 network=udp -13:21.963 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16957 network=tcp -13:22.465 [INFO] TestAgent: Waiting for endpoints to shut down -13:22.465 [INFO] TestAgent: Endpoints down === RUN TestAgent_WarnCheck_ACLDeny/root_token ---- PASS: TestCatalogDatacenters (0.75s) +2022-12-04T05:49:35.761Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:35.782Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:35.785 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:445ba74f-cd22-687d-bed6-f09df239d54c Address:127.0.0.1:24137}]" +49:35.786 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24137 [Follower]" leader= +49:35.786 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-445ba74f-cd22-687d-bed6-f09df239d54c.dc1 127.0.0.1 +49:35.787 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-445ba74f-cd22-687d-bed6-f09df239d54c 127.0.0.1 +2022-12-04T05:49:35.787Z [INFO] agent.router: Initializing LAN area manager +49:35.787 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-445ba74f-cd22-687d-bed6-f09df239d54c.dc1 area=wan +49:35.787 [INFO] TestAgent.server: Adding LAN server: server="Node-445ba74f-cd22-687d-bed6-f09df239d54c (Addr: tcp/127.0.0.1:24137) (DC: dc1)" +49:35.787 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24125 network=udp +49:35.787 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24125 network=tcp +49:35.789 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24133 network=tcp +49:35.789 [INFO] TestAgent: started state syncer +49:35.789 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24180 network=tcp +49:35.828 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.828 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24137 [Candidate]" term=2 +49:35.828 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:35.828 [DEBUG] TestAgent.server.raft: vote granted: from=445ba74f-cd22-687d-bed6-f09df239d54c term=2 tally=1 +49:35.828 [INFO] TestAgent.server.raft: election won: tally=1 +49:35.828 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24137 [Leader]" +49:35.829 [INFO] TestAgent.server: cluster leadership acquired +49:35.829 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24137 +49:35.830 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:35.832 [INFO] TestAgent.server: New leader elected: payload=Node-445ba74f-cd22-687d-bed6-f09df239d54c +49:35.832 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:35.832 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:35.832 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:35.832 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:35.832 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:35.833 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-445ba74f-cd22-687d-bed6-f09df239d54c +49:35.833 [INFO] TestAgent.server: member joined, marking health alive: member=Node-445ba74f-cd22-687d-bed6-f09df239d54c +49:35.834 [INFO] TestAgent.server: federation state anti-entropy synced +49:36.170 [INFO] TestAgent: Service entered maintenance mode: service=test +49:36.171 [INFO] TestAgent: Synced node info +49:36.172 [INFO] TestAgent: Synced service: service=test +49:36.173 [INFO] TestAgent: Synced check: check=_service_maintenance:test +49:36.173 [INFO] TestAgent: Requesting shutdown +49:36.173 [INFO] TestAgent.server: shutting down server +49:36.173 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:36.173 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:36.173 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:36.173 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:36.173 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:36.173 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:36.173 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:36.173 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:36.173 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:36.173 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:36.173 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:36.173 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:36.175Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:36.175Z [INFO] agent.router.manager: shutting down +49:36.175 [INFO] TestAgent: consul server down +49:36.175 [INFO] TestAgent: shutdown complete +49:36.175 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24125 network=tcp +49:36.175 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24125 network=udp +49:36.175 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24133 network=tcp +49:36.676 [INFO] TestAgent: Waiting for endpoints to shut down +49:36.676 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ServiceMaintenance_Enable (0.93s) === CONT TestAgent_PassCheck -2021-10-31T23:13:21.576Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:21.671Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:21.678 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:47b91fe1-6fb6-6257-1f05-096f35716d71 Address:127.0.0.1:16929}]" -13:21.679 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-47b91fe1-6fb6-6257-1f05-096f35716d71.dc1 127.0.0.1 -13:21.679 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-47b91fe1-6fb6-6257-1f05-096f35716d71 127.0.0.1 -2021-10-31T23:13:21.679Z [INFO] agent.router: Initializing LAN area manager -13:21.679 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16847 network=udp -13:21.679 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16929 [Follower]" leader= -13:21.680 [INFO] TestAgent.server: Adding LAN server: server="Node-47b91fe1-6fb6-6257-1f05-096f35716d71 (Addr: tcp/127.0.0.1:16929) (DC: dc1)" -13:21.680 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-47b91fe1-6fb6-6257-1f05-096f35716d71.dc1 area=wan -13:21.680 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16847 network=tcp -13:21.680 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16855 network=tcp -13:21.680 [INFO] TestAgent: started state syncer -13:21.680 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16930 network=tcp -13:21.746 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:21.746 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16929 [Candidate]" term=2 -13:21.746 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:21.746 [DEBUG] TestAgent.server.raft: vote granted: from=47b91fe1-6fb6-6257-1f05-096f35716d71 term=2 tally=1 -13:21.746 [INFO] TestAgent.server.raft: election won: tally=1 -13:21.746 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16929 [Leader]" -13:21.746 [INFO] TestAgent.server: cluster leadership acquired -13:21.746 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16929 -13:21.747 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:21.750 [INFO] TestAgent.server: New leader elected: payload=Node-47b91fe1-6fb6-6257-1f05-096f35716d71 -13:21.753 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:21.753 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:21.753 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:21.753 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:21.753 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:21.753 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-47b91fe1-6fb6-6257-1f05-096f35716d71 -13:21.753 [INFO] TestAgent.server: member joined, marking health alive: member=Node-47b91fe1-6fb6-6257-1f05-096f35716d71 -13:21.754 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.764 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:21.765 [INFO] TestAgent: Synced node info -13:21.765 [DEBUG] TestAgent: Node info in sync -13:21.999 [INFO] TestAgent: Requesting shutdown -13:21.999 [INFO] TestAgent.server: shutting down server -13:21.999 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.999 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.999 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.999 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.999 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.999 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.999 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:21.999 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.999 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:21.999 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.999 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:21.999Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.999Z [INFO] agent.router.manager: shutting down -13:22.000 [INFO] TestAgent: consul server down -13:22.000 [INFO] TestAgent: shutdown complete -13:22.000 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16847 network=tcp -13:22.000 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16847 network=udp -13:22.000 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16855 network=tcp -13:22.501 [INFO] TestAgent: Waiting for endpoints to shut down -13:22.501 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodes (0.97s) -=== CONT TestAgent_RerouteExistingHTTPChecks -2021-10-31T23:13:21.820Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:21.820Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:21.841Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:21.841Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:21.868 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1cf92a17-ad55-91fb-3b26-5ee5fe4f5809 Address:127.0.0.1:16982}]" -13:21.869 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1cf92a17-ad55-91fb-3b26-5ee5fe4f5809.dc1 127.0.0.1 -13:21.869 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16982 [Follower]" leader= -13:21.874 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1cf92a17-ad55-91fb-3b26-5ee5fe4f5809 127.0.0.1 -2021-10-31T23:13:21.874Z [INFO] agent.router: Initializing LAN area manager -13:21.874 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16991 network=udp -13:21.874 [INFO] TestAgent.server: Adding LAN server: server="Node-1cf92a17-ad55-91fb-3b26-5ee5fe4f5809 (Addr: tcp/127.0.0.1:16982) (DC: dc1)" -13:21.874 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1cf92a17-ad55-91fb-3b26-5ee5fe4f5809.dc1 area=wan -13:21.875 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16991 network=tcp -13:21.876 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16992 network=tcp -13:21.876 [INFO] TestAgent: started state syncer -13:21.876 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16983 network=tcp -13:21.909 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:21.909 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16982 [Candidate]" term=2 -13:21.909 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:21.909 [DEBUG] TestAgent.server.raft: vote granted: from=1cf92a17-ad55-91fb-3b26-5ee5fe4f5809 term=2 tally=1 -13:21.909 [INFO] TestAgent.server.raft: election won: tally=1 -13:21.909 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16982 [Leader]" -13:21.909 [INFO] TestAgent.server: cluster leadership acquired -13:21.909 [INFO] TestAgent.server: initializing acls -13:21.909 [INFO] TestAgent.server: New leader elected: payload=Node-1cf92a17-ad55-91fb-3b26-5ee5fe4f5809 -13:21.910 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:21.910 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:21.910 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:21.911 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:21.911 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:21.911 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:21.911 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1cf92a17-ad55-91fb-3b26-5ee5fe4f5809 -13:21.911 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1cf92a17-ad55-91fb-3b26-5ee5fe4f5809.dc1 -13:21.912 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:21.913 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1cf92a17-ad55-91fb-3b26-5ee5fe4f5809.dc1 area=wan -13:21.914 [INFO] TestAgent.server: Updating LAN server: server="Node-1cf92a17-ad55-91fb-3b26-5ee5fe4f5809 (Addr: tcp/127.0.0.1:16982) (DC: dc1)" -13:21.914 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:21.914 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:21.914 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:21.914 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:21.914 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:21.914 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1cf92a17-ad55-91fb-3b26-5ee5fe4f5809 -13:21.914 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1cf92a17-ad55-91fb-3b26-5ee5fe4f5809 -13:21.915 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.915 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1cf92a17-ad55-91fb-3b26-5ee5fe4f5809 -13:22.026 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-1cf92a17-ad55-91fb-3b26-5ee5fe4f5809 -13:22.026 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-1cf92a17-ad55-91fb-3b26-5ee5fe4f5809 -13:22.027 [DEBUG] TestAgent: Check status updated: check=test status=critical -13:22.028 [INFO] TestAgent: Synced node info -13:22.028 [WARN] TestAgent: Check registration blocked by ACLs: check=test accessorID=00000000-0000-0000-0000-000000000002 -13:22.028 [INFO] TestAgent: Requesting shutdown -13:22.028 [INFO] TestAgent.server: shutting down server -13:22.028 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:22.028 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:22.028 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:22.028 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:22.028 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:22.028 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:22.028 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:22.029 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:22.029 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:22.029 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:22.029 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:22.029 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:22.029 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:22.029 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:22.029 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:22.029 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:22.029Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:22.029Z [INFO] agent.router.manager: shutting down -13:22.029 [INFO] TestAgent: consul server down -13:22.029 [INFO] TestAgent: shutdown complete -13:22.029 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16991 network=tcp -13:22.029 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16991 network=udp -13:22.029 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16992 network=tcp -13:22.537 [INFO] TestAgent: Waiting for endpoints to shut down -13:22.537 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_FailCheck_ACLDeny (0.80s) - --- PASS: TestAgent_FailCheck_ACLDeny/no_token (0.00s) - --- PASS: TestAgent_FailCheck_ACLDeny/root_token (0.00s) -=== CONT TestAgent_DeregisterCheckACLDeny -=== RUN TestCatalogRegister_Service_InvalidAddress/addr_0.0.0.0 -=== RUN TestCatalogRegister_Service_InvalidAddress/addr_:: -=== RUN TestCatalogRegister_Service_InvalidAddress/addr_[::] -[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=true#01 === RUN TestAgent_PassCheck_ACLDeny/no_token === RUN TestAgent_PassCheck_ACLDeny/root_token -2021-10-31T23:13:22.077Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:22.102Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:22.105 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ee5464da-d77a-1592-8090-a11b6881bc78 Address:127.0.0.1:16961}]" -13:22.105 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16961 [Follower]" leader= -13:22.126 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ee5464da-d77a-1592-8090-a11b6881bc78.dc1 127.0.0.1 -13:22.127 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ee5464da-d77a-1592-8090-a11b6881bc78 127.0.0.1 -2021-10-31T23:13:22.127Z [INFO] agent.router: Initializing LAN area manager -13:22.127 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16935 network=udp -13:22.128 [INFO] TestAgent.server: Adding LAN server: server="Node-ee5464da-d77a-1592-8090-a11b6881bc78 (Addr: tcp/127.0.0.1:16961) (DC: dc1)" -13:22.128 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ee5464da-d77a-1592-8090-a11b6881bc78.dc1 area=wan -13:22.128 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16935 network=tcp -13:22.133 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16936 network=tcp -13:22.134 [INFO] TestAgent: started state syncer -13:22.134 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16962 network=tcp -13:22.169 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:22.169 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16961 [Candidate]" term=2 -13:22.170 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:22.170 [DEBUG] TestAgent.server.raft: vote granted: from=ee5464da-d77a-1592-8090-a11b6881bc78 term=2 tally=1 -13:22.170 [INFO] TestAgent.server.raft: election won: tally=1 -13:22.170 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16961 [Leader]" -13:22.170 [INFO] TestAgent.server: cluster leadership acquired -13:22.170 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16961 -13:22.171 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:22.173 [INFO] TestAgent.server: New leader elected: payload=Node-ee5464da-d77a-1592-8090-a11b6881bc78 -13:22.178 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:22.178 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:22.178 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:22.178 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:22.178 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:22.178 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ee5464da-d77a-1592-8090-a11b6881bc78 -13:22.178 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ee5464da-d77a-1592-8090-a11b6881bc78 -13:22.180 [INFO] TestAgent.server: federation state anti-entropy synced -13:22.218 [DEBUG] TestAgent: Check status updated: check=test status=critical -13:22.219 [INFO] TestAgent: Synced node info -13:22.219 [INFO] TestAgent: Synced check: check=test -13:22.219 [INFO] TestAgent: Requesting shutdown -13:22.219 [INFO] TestAgent.server: shutting down server -13:22.219 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:22.219 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:22.219 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:22.219 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:22.219 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:22.219 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:22.219 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:22.219 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:22.220 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:22.220 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:22.220 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:22.220 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:22.220Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:22.220Z [INFO] agent.router.manager: shutting down -13:22.220 [INFO] TestAgent: consul server down -13:22.220 [INFO] TestAgent: shutdown complete -13:22.220 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16935 network=tcp -13:22.220 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16935 network=udp -13:22.220 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16936 network=tcp -13:22.720 [INFO] TestAgent: Waiting for endpoints to shut down -13:22.720 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_FailCheck (0.70s) -=== CONT TestAgent_consulConfig_RaftTrailingLogs -=== RUN TestAgent_DeregisterCheckACLDeny/no_token -=== RUN TestAgent_DeregisterCheckACLDeny/root_token -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:22.000Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:22.059Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:22.061 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3f74d684-253f-71a6-99dc-33e81d07f4f0 Address:127.0.0.1:16933}]" -13:22.061 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16933 [Follower]" leader= -13:22.066 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3f74d684-253f-71a6-99dc-33e81d07f4f0.dc1 127.0.0.1 -13:22.067 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3f74d684-253f-71a6-99dc-33e81d07f4f0 127.0.0.1 -2021-10-31T23:13:22.067Z [INFO] agent.router: Initializing LAN area manager -13:22.067 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16984 network=udp -13:22.067 [INFO] TestAgent.server: Adding LAN server: server="Node-3f74d684-253f-71a6-99dc-33e81d07f4f0 (Addr: tcp/127.0.0.1:16933) (DC: dc1)" -13:22.068 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3f74d684-253f-71a6-99dc-33e81d07f4f0.dc1 area=wan -13:22.068 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16984 network=tcp -13:22.068 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16985 network=tcp -13:22.068 [INFO] TestAgent: started state syncer -13:22.068 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16934 network=tcp -13:22.109 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:22.109 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16933 [Candidate]" term=2 -13:22.110 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:22.110 [DEBUG] TestAgent.server.raft: vote granted: from=3f74d684-253f-71a6-99dc-33e81d07f4f0 term=2 tally=1 -13:22.110 [INFO] TestAgent.server.raft: election won: tally=1 -13:22.110 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16933 [Leader]" -13:22.110 [INFO] TestAgent.server: cluster leadership acquired -13:22.110 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16933 -13:22.111 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:22.127 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:22.127 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:22.127 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:22.127 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:22.127 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:22.127 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3f74d684-253f-71a6-99dc-33e81d07f4f0 -13:22.127 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3f74d684-253f-71a6-99dc-33e81d07f4f0 -13:22.128 [INFO] TestAgent.server: federation state anti-entropy synced -13:22.132 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:22.133 [INFO] TestAgent: Synced node info -13:22.133 [DEBUG] TestAgent: Node info in sync -13:22.137 [INFO] TestAgent.server: New leader elected: payload=Node-3f74d684-253f-71a6-99dc-33e81d07f4f0 -13:22.402 [INFO] TestAgent: Requesting shutdown -13:22.402 [INFO] TestAgent.server: shutting down server -13:22.402 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:22.402 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:22.402 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:22.402 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:22.402 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:22.402 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:22.402 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:22.402 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:22.402 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:22.402 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:22.402 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:22.402Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:22.402Z [INFO] agent.router.manager: shutting down -13:22.402 [INFO] TestAgent: consul server down -13:22.402 [INFO] TestAgent: shutdown complete -13:22.402 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16984 network=tcp -13:22.402 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16984 network=udp -13:22.402 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16985 network=tcp -13:22.907 [INFO] TestAgent: Waiting for endpoints to shut down -13:22.907 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogDeregister (0.95s) +2022-12-04T05:49:35.842Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:35.842Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:35.859Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:35.859Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:35.861 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:83a7893e-2551-b614-f7d5-b23fde460c94 Address:127.0.0.1:24214}]" +49:35.861 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24214 [Follower]" leader= +49:35.862 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-83a7893e-2551-b614-f7d5-b23fde460c94.dc1 127.0.0.1 +49:35.862 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-83a7893e-2551-b614-f7d5-b23fde460c94 127.0.0.1 +2022-12-04T05:49:35.862Z [INFO] agent.router: Initializing LAN area manager +49:35.863 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-83a7893e-2551-b614-f7d5-b23fde460c94.dc1 area=wan +49:35.863 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24058 network=udp +49:35.863 [INFO] TestAgent.server: Adding LAN server: server="Node-83a7893e-2551-b614-f7d5-b23fde460c94 (Addr: tcp/127.0.0.1:24214) (DC: dc1)" +49:35.863 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24058 network=tcp +49:35.863 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24210 network=tcp +49:35.864 [INFO] TestAgent: started state syncer +49:35.864 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24103 network=tcp +49:35.913 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:35.924 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.924 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24214 [Candidate]" term=2 +49:35.924 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:35.924 [DEBUG] TestAgent.server.raft: vote granted: from=83a7893e-2551-b614-f7d5-b23fde460c94 term=2 tally=1 +49:35.924 [INFO] TestAgent.server.raft: election won: tally=1 +49:35.924 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24214 [Leader]" +49:35.925 [INFO] TestAgent.server: cluster leadership acquired +49:35.925 [INFO] TestAgent.server: initializing acls +49:35.925 [INFO] TestAgent.server: New leader elected: payload=Node-83a7893e-2551-b614-f7d5-b23fde460c94 +49:35.925 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:35.925 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:35.925 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:35.926 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:35.926 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:35.926 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:35.926 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-83a7893e-2551-b614-f7d5-b23fde460c94 +49:35.926 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-83a7893e-2551-b614-f7d5-b23fde460c94.dc1 +49:35.926 [INFO] TestAgent.server: Updating LAN server: server="Node-83a7893e-2551-b614-f7d5-b23fde460c94 (Addr: tcp/127.0.0.1:24214) (DC: dc1)" +49:35.926 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-83a7893e-2551-b614-f7d5-b23fde460c94.dc1 area=wan +49:35.926 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:35.928 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:35.928 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:35.928 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:35.928 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:35.928 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:35.928 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-83a7893e-2551-b614-f7d5-b23fde460c94 +49:35.928 [INFO] TestAgent.server: member joined, marking health alive: member=Node-83a7893e-2551-b614-f7d5-b23fde460c94 +49:35.928 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-83a7893e-2551-b614-f7d5-b23fde460c94 +49:35.928 [INFO] TestAgent.server: federation state anti-entropy synced +49:36.004 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:36.005 [INFO] TestAgent: Synced node info +49:36.005 [DEBUG] TestAgent: Node info in sync +49:36.220 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-83a7893e-2551-b614-f7d5-b23fde460c94 +49:36.221 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-83a7893e-2551-b614-f7d5-b23fde460c94 +49:36.221 [DEBUG] TestAgent: removed service: service=test +49:36.221 [DEBUG] TestAgent: Node info in sync +49:36.221 [INFO] TestAgent: Deregistered service: service=test +49:36.221 [INFO] TestAgent: Requesting shutdown +49:36.222 [INFO] TestAgent.server: shutting down server +49:36.222 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:36.222 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:36.222 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:36.222 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:36.222 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:36.222 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:36.222 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:36.222 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:36.222 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:36.222 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:36.222 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:36.222 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:36.222 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:36.222 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:36.222 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:36.222Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:36.222Z [INFO] agent.router.manager: shutting down +49:36.223 [INFO] TestAgent: consul server down +49:36.223 [INFO] TestAgent: shutdown complete +49:36.223 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24058 network=tcp +49:36.223 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24058 network=udp +49:36.223 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24210 network=tcp +49:36.723 [INFO] TestAgent: Waiting for endpoints to shut down +49:36.723 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_DeregisterService_ACLDeny (0.91s) + --- PASS: TestAgent_DeregisterService_ACLDeny/no_token (0.00s) + --- PASS: TestAgent_DeregisterService_ACLDeny/root_token (0.00s) +=== CONT TestAgent_DeregisterCheckACLDeny +2022-12-04T05:49:35.858Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:35.869Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:35.876 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:74f3d440-f0b1-2e90-3385-12a21bbfa73e Address:127.0.0.1:24130}]" +49:35.877 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-74f3d440-f0b1-2e90-3385-12a21bbfa73e.dc1 127.0.0.1 +49:35.878 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-74f3d440-f0b1-2e90-3385-12a21bbfa73e 127.0.0.1 +2022-12-04T05:49:35.878Z [INFO] agent.router: Initializing LAN area manager +49:35.878 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24104 network=udp +49:35.878 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24130 [Follower]" leader= +49:35.879 [INFO] TestAgent.server: Adding LAN server: server="Node-74f3d440-f0b1-2e90-3385-12a21bbfa73e (Addr: tcp/127.0.0.1:24130) (DC: dc1)" +49:35.879 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-74f3d440-f0b1-2e90-3385-12a21bbfa73e.dc1 area=wan +49:35.879 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24104 network=tcp +49:35.881 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24126 network=tcp +49:35.881 [INFO] TestAgent: started state syncer +49:35.881 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23833 network=tcp +49:35.929 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.929 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24130 [Candidate]" term=2 +49:35.929 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:35.929 [DEBUG] TestAgent.server.raft: vote granted: from=74f3d440-f0b1-2e90-3385-12a21bbfa73e term=2 tally=1 +49:35.929 [INFO] TestAgent.server.raft: election won: tally=1 +49:35.929 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24130 [Leader]" +49:35.929 [INFO] TestAgent.server: cluster leadership acquired +49:35.929 [INFO] TestAgent.server: New leader elected: payload=Node-74f3d440-f0b1-2e90-3385-12a21bbfa73e +49:35.929 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24130 +49:35.930 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:35.937 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:35.937 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:35.937 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:35.937 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:35.937 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:35.937 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-74f3d440-f0b1-2e90-3385-12a21bbfa73e +49:35.937 [INFO] TestAgent.server: member joined, marking health alive: member=Node-74f3d440-f0b1-2e90-3385-12a21bbfa73e +49:35.938 [INFO] TestAgent.server: federation state anti-entropy synced +49:36.125 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:36.126 [INFO] TestAgent: Synced node info +49:36.221 [DEBUG] TestAgent: removed service: service=test +49:36.221 [DEBUG] TestAgent: Node info in sync +49:36.222 [INFO] TestAgent: Deregistered service: service=test +49:36.222 [INFO] TestAgent: Requesting shutdown +49:36.223 [INFO] TestAgent.server: shutting down server +49:36.223 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:36.223 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:36.223 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:36.223 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:36.223 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:36.223 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:36.223 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:36.223 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:36.223 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:36.223 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:36.223 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:36.223Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:36.223Z [INFO] agent.router.manager: shutting down +49:36.223 [INFO] TestAgent: consul server down +49:36.223 [INFO] TestAgent: shutdown complete +49:36.223 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24104 network=tcp +49:36.223 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24104 network=udp +49:36.223 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24126 network=tcp +49:36.724 [INFO] TestAgent: Waiting for endpoints to shut down +49:36.724 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_DeregisterService (0.88s) === CONT TestAgent_DeregisterCheck -=== CONT TestAgent_JoinWAN_viaMeshGateway - agent_test.go:4395: routing {bob.,}server.dc1.consul => 127.0.0.1:16941 - agent_test.go:4396: routing {betty.,}server.dc2.consul => 127.0.0.1:16971 - agent_test.go:4397: routing {bonnie.,}server.dc3.consul => 127.0.0.1:16919 -2021-10-31T23:13:21.574Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:21.574Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:21.632Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:21.632Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:21.644 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a3bdef6f-481a-e72d-c3d4-54ee72711af3 Address:127.0.0.1:16947}]" -13:21.645 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a3bdef6f-481a-e72d-c3d4-54ee72711af3.dc2 127.0.0.1 -13:21.646 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16947 [Follower]" leader= -13:21.647 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a3bdef6f-481a-e72d-c3d4-54ee72711af3 127.0.0.1 -2021-10-31T23:13:21.647Z [INFO] agent.router: Initializing LAN area manager -13:21.669 [INFO] TestAgent.server: Adding LAN server: server="Node-a3bdef6f-481a-e72d-c3d4-54ee72711af3 (Addr: tcp/127.0.0.1:16947) (DC: dc2)" -13:21.670 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a3bdef6f-481a-e72d-c3d4-54ee72711af3.dc2 area=wan -13:21.671 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16910 network=udp -13:21.671 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16910 network=tcp -13:21.671 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16925 network=tcp -13:21.671 [INFO] TestAgent: started state syncer -13:21.672 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16948 network=tcp -13:21.701 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:21.701 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16947 [Candidate]" term=2 -13:21.702 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:21.702 [DEBUG] TestAgent.server.raft: vote granted: from=a3bdef6f-481a-e72d-c3d4-54ee72711af3 term=2 tally=1 -13:21.702 [INFO] TestAgent.server.raft: election won: tally=1 -13:21.702 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16947 [Leader]" -13:21.702 [INFO] TestAgent.server: cluster leadership acquired -13:21.702 [INFO] TestAgent.server: initializing acls -13:21.702 [INFO] TestAgent.server: New leader elected: payload=Node-a3bdef6f-481a-e72d-c3d4-54ee72711af3 -13:21.702 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:21.703 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:21.703 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:21.703 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:21.703 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a3bdef6f-481a-e72d-c3d4-54ee72711af3 -13:21.703 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a3bdef6f-481a-e72d-c3d4-54ee72711af3.dc2 -13:21.703 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:21.705 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:21.705 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:21.705 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:21.705 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:21.705 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:21.705 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a3bdef6f-481a-e72d-c3d4-54ee72711af3 -13:21.705 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a3bdef6f-481a-e72d-c3d4-54ee72711af3 -13:21.705 [INFO] TestAgent.server: Updating LAN server: server="Node-a3bdef6f-481a-e72d-c3d4-54ee72711af3 (Addr: tcp/127.0.0.1:16947) (DC: dc2)" -13:21.705 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a3bdef6f-481a-e72d-c3d4-54ee72711af3 -13:21.705 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a3bdef6f-481a-e72d-c3d4-54ee72711af3.dc2 area=wan -13:21.706 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.816 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:21.816 [INFO] TestAgent: Synced node info -13:21.946 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:16865] -13:21.949 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:16865 -13:21.966 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b8cfde8a-35b7-9894-e27c-245d315d1eb1.dc1 127.0.0.1 -13:21.966 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -13:21.967 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b8cfde8a-35b7-9894-e27c-245d315d1eb1.dc1 area=wan -13:21.974 [INFO] TestAgent: Requesting shutdown -13:21.974 [INFO] TestAgent.server: shutting down server -13:21.974 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:21.974 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:21.974 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:21.974 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:21.974 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:21.974 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:21.974 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:21.974 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:21.974 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:21.974 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:21.974 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:21.974 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:21.974 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:21.974 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:21.974 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:21.974Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.974Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:21.974Z [INFO] agent.router.manager: shutting down -13:21.974 [INFO] TestAgent: consul server down -13:21.974 [INFO] TestAgent: shutdown complete -13:21.974 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16910 network=tcp -13:21.974 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16910 network=udp -13:21.975 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16925 network=tcp -13:22.475 [INFO] TestAgent: Waiting for endpoints to shut down -13:22.475 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:21.129Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:21.129Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:21.153Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:21.153Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:21.156 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b8cfde8a-35b7-9894-e27c-245d315d1eb1 Address:127.0.0.1:16866}]" -13:21.156 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b8cfde8a-35b7-9894-e27c-245d315d1eb1.dc1 127.0.0.1 -13:21.157 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b8cfde8a-35b7-9894-e27c-245d315d1eb1 127.0.0.1 -2021-10-31T23:13:21.157Z [INFO] agent.router: Initializing LAN area manager -13:21.157 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16833 network=udp -13:21.157 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16866 [Follower]" leader= -13:21.158 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b8cfde8a-35b7-9894-e27c-245d315d1eb1.dc1 area=wan -13:21.158 [INFO] TestAgent.server: Adding LAN server: server="Node-b8cfde8a-35b7-9894-e27c-245d315d1eb1 (Addr: tcp/127.0.0.1:16866) (DC: dc1)" -13:21.158 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16833 network=tcp -13:21.159 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16834 network=tcp -13:21.159 [INFO] TestAgent: started state syncer -13:21.159 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16867 network=tcp -13:21.225 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:21.228 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:21.228 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16866 [Candidate]" term=2 -13:21.228 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:21.228 [DEBUG] TestAgent.server.raft: vote granted: from=b8cfde8a-35b7-9894-e27c-245d315d1eb1 term=2 tally=1 -13:21.228 [INFO] TestAgent.server.raft: election won: tally=1 -13:21.228 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16866 [Leader]" -13:21.228 [INFO] TestAgent.server: cluster leadership acquired -13:21.228 [INFO] TestAgent.server: initializing acls -13:21.229 [INFO] TestAgent.server: New leader elected: payload=Node-b8cfde8a-35b7-9894-e27c-245d315d1eb1 -13:21.229 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:21.229 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:21.232 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:21.232 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:21.232 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b8cfde8a-35b7-9894-e27c-245d315d1eb1 -13:21.232 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b8cfde8a-35b7-9894-e27c-245d315d1eb1.dc1 -13:21.233 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:21.233 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b8cfde8a-35b7-9894-e27c-245d315d1eb1.dc1 area=wan -13:21.233 [INFO] TestAgent.server: Updating LAN server: server="Node-b8cfde8a-35b7-9894-e27c-245d315d1eb1 (Addr: tcp/127.0.0.1:16866) (DC: dc1)" -13:21.235 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:21.235 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:21.235 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:21.235 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:21.235 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:21.235 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b8cfde8a-35b7-9894-e27c-245d315d1eb1 -13:21.235 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b8cfde8a-35b7-9894-e27c-245d315d1eb1 -13:21.236 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b8cfde8a-35b7-9894-e27c-245d315d1eb1 -13:21.237 [INFO] TestAgent.server: federation state anti-entropy synced -13:21.534 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:21.535 [INFO] TestAgent: Synced node info -13:21.950 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:34604 -13:21.954 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a3bdef6f-481a-e72d-c3d4-54ee72711af3.dc2 127.0.0.1 -13:21.954 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a3bdef6f-481a-e72d-c3d4-54ee72711af3.dc2 area=wan -13:22.475 [INFO] TestAgent: Requesting shutdown -13:22.481 [INFO] TestAgent.server: shutting down server -13:22.481 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:22.481 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:22.481 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:22.481 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:22.481 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:22.481 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:22.481 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:22.482 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:22.482 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:22.482 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:22.482 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:22.482 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:22.482 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:22.482 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:22.482 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:22.482Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:22.482Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:22.482Z [INFO] agent.router.manager: shutting down -13:22.482 [INFO] TestAgent: consul server down -13:22.482 [INFO] TestAgent: shutdown complete -13:22.482 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16833 network=tcp -13:22.482 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16833 network=udp -13:22.482 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16834 network=tcp -13:22.994 [INFO] TestAgent: Waiting for endpoints to shut down -13:22.994 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogNodes_WanTranslation (1.88s) -=== CONT TestAgent_consulConfig_AutoEncryptAllowTLS -2021-10-31T23:13:22.170Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:22.170Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:22.229Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:22.229Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:22.248 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:71f26688-6955-7892-ff47-d8187db9805a Address:127.0.0.1:17010}]" -13:22.249 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-71f26688-6955-7892-ff47-d8187db9805a.dc1 127.0.0.1 -13:22.249 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-71f26688-6955-7892-ff47-d8187db9805a 127.0.0.1 -2021-10-31T23:13:22.249Z [INFO] agent.router: Initializing LAN area manager -13:22.249 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17010 [Follower]" leader= -13:22.254 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16826 network=udp -13:22.254 [INFO] TestAgent.server: Adding LAN server: server="Node-71f26688-6955-7892-ff47-d8187db9805a (Addr: tcp/127.0.0.1:17010) (DC: dc1)" -13:22.254 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-71f26688-6955-7892-ff47-d8187db9805a.dc1 area=wan -13:22.254 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16826 network=tcp -13:22.255 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16827 network=tcp -13:22.255 [INFO] TestAgent: started state syncer -13:22.255 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17011 network=tcp -13:22.305 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:22.310 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:22.310 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17010 [Candidate]" term=2 -13:22.310 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:22.310 [DEBUG] TestAgent.server.raft: vote granted: from=71f26688-6955-7892-ff47-d8187db9805a term=2 tally=1 -13:22.310 [INFO] TestAgent.server.raft: election won: tally=1 -13:22.310 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17010 [Leader]" -13:22.310 [INFO] TestAgent.server: cluster leadership acquired -13:22.310 [INFO] TestAgent.server: initializing acls -13:22.311 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:22.311 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:22.311 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:22.312 [INFO] TestAgent.server: New leader elected: payload=Node-71f26688-6955-7892-ff47-d8187db9805a -13:22.312 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:22.312 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:22.312 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:22.312 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-71f26688-6955-7892-ff47-d8187db9805a -13:22.312 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-71f26688-6955-7892-ff47-d8187db9805a.dc1 -13:22.312 [INFO] TestAgent.server: Updating LAN server: server="Node-71f26688-6955-7892-ff47-d8187db9805a (Addr: tcp/127.0.0.1:17010) (DC: dc1)" -13:22.312 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-71f26688-6955-7892-ff47-d8187db9805a.dc1 area=wan -13:22.313 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:22.315 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:22.315 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:22.315 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:22.315 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:22.315 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:22.315 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-71f26688-6955-7892-ff47-d8187db9805a -13:22.315 [INFO] TestAgent.server: member joined, marking health alive: member=Node-71f26688-6955-7892-ff47-d8187db9805a -13:22.316 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-71f26688-6955-7892-ff47-d8187db9805a -13:22.317 [INFO] TestAgent.server: federation state anti-entropy synced -13:22.465 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-71f26688-6955-7892-ff47-d8187db9805a -13:22.465 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-71f26688-6955-7892-ff47-d8187db9805a -13:22.466 [DEBUG] TestAgent: Check status updated: check=test status=warning -13:22.467 [INFO] TestAgent: Synced node info -13:22.467 [WARN] TestAgent: Check registration blocked by ACLs: check=test accessorID=00000000-0000-0000-0000-000000000002 -13:22.467 [INFO] TestAgent: Requesting shutdown -13:22.467 [INFO] TestAgent.server: shutting down server -13:22.467 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:22.467 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:22.467 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:22.467 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:22.467 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:22.467 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:22.467 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:22.468 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:22.468 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:22.468 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:22.468 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:22.468 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:22.468 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:22.468 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:22.468 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:22.468 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:22.468Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:22.468Z [INFO] agent.router.manager: shutting down -13:22.468 [INFO] TestAgent: consul server down -13:22.468 [INFO] TestAgent: shutdown complete -13:22.468 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16826 network=tcp -13:22.468 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16826 network=udp -13:22.468 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16827 network=tcp -13:22.985 [INFO] TestAgent: Waiting for endpoints to shut down -13:22.993 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_WarnCheck_ACLDeny (0.87s) - --- PASS: TestAgent_WarnCheck_ACLDeny/no_token (0.00s) - --- PASS: TestAgent_WarnCheck_ACLDeny/root_token (0.00s) +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=false#01 +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=33)_compressed=true#01 +2022-12-04T05:49:36.205Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:36.224Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:36.227 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ff4b7077-f256-3394-c3ff-37f3dbc42d63 Address:127.0.0.1:24200}]" +49:36.228 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ff4b7077-f256-3394-c3ff-37f3dbc42d63.dc1 127.0.0.1 +49:36.228 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24200 [Follower]" leader= +49:36.229 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ff4b7077-f256-3394-c3ff-37f3dbc42d63 127.0.0.1 +2022-12-04T05:49:36.229Z [INFO] agent.router: Initializing LAN area manager +49:36.229 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24160 network=udp +49:36.229 [INFO] TestAgent.server: Adding LAN server: server="Node-ff4b7077-f256-3394-c3ff-37f3dbc42d63 (Addr: tcp/127.0.0.1:24200) (DC: dc1)" +49:36.229 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ff4b7077-f256-3394-c3ff-37f3dbc42d63.dc1 area=wan +49:36.229 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24160 network=tcp +49:36.230 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24196 network=tcp +49:36.231 [INFO] TestAgent: started state syncer +49:36.231 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24215 network=tcp +49:36.284 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:36.284 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24200 [Candidate]" term=2 +49:36.284 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:36.284 [DEBUG] TestAgent.server.raft: vote granted: from=ff4b7077-f256-3394-c3ff-37f3dbc42d63 term=2 tally=1 +49:36.284 [INFO] TestAgent.server.raft: election won: tally=1 +49:36.284 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24200 [Leader]" +49:36.285 [INFO] TestAgent.server: cluster leadership acquired +49:36.285 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24200 +49:36.286 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:36.287 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:36.287 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:36.287 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:36.287 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:36.287 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:36.288 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ff4b7077-f256-3394-c3ff-37f3dbc42d63 +49:36.288 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ff4b7077-f256-3394-c3ff-37f3dbc42d63 +49:36.288 [INFO] TestAgent.server: New leader elected: payload=Node-ff4b7077-f256-3394-c3ff-37f3dbc42d63 +49:36.289 [INFO] TestAgent.server: federation state anti-entropy synced +49:36.356 [DEBUG] TestAgent: Check status updated: check=test status=critical +49:36.357 [INFO] TestAgent: Synced node info +49:36.358 [INFO] TestAgent: Synced check: check=test +49:36.358 [INFO] TestAgent: Requesting shutdown +49:36.358 [INFO] TestAgent.server: shutting down server +49:36.358 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:36.358 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:36.358 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:36.358 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:36.358 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:36.358 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:36.358 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:36.358 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:36.358 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:36.358 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:36.358 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:36.358 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:36.358Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:36.358Z [INFO] agent.router.manager: shutting down +49:36.359 [INFO] TestAgent: consul server down +49:36.359 [INFO] TestAgent: shutdown complete +49:36.359 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24160 network=tcp +49:36.359 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24160 network=udp +49:36.359 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24196 network=tcp +49:36.859 [INFO] TestAgent: Waiting for endpoints to shut down +49:36.859 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_FailCheck (0.66s) === CONT TestAgent_RegisterCheck_ACLDeny -2021-10-31T23:13:22.091Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:22.091Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:22.135Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:22.135Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:22.147 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b0325f12-bd50-4820-5b3a-fca42f5d4ece Address:127.0.0.1:16824}]" -13:22.148 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b0325f12-bd50-4820-5b3a-fca42f5d4ece.dc1 127.0.0.1 -13:22.149 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16824 [Follower]" leader= -13:22.158 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b0325f12-bd50-4820-5b3a-fca42f5d4ece 127.0.0.1 -2021-10-31T23:13:22.158Z [INFO] agent.router: Initializing LAN area manager -13:22.159 [INFO] TestAgent.server: Adding LAN server: server="Node-b0325f12-bd50-4820-5b3a-fca42f5d4ece (Addr: tcp/127.0.0.1:16824) (DC: dc1)" -13:22.159 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b0325f12-bd50-4820-5b3a-fca42f5d4ece.dc1 area=wan -13:22.159 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16977 network=udp -13:22.159 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16977 network=tcp -13:22.160 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16978 network=tcp -13:22.160 [INFO] TestAgent: started state syncer -13:22.160 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16825 network=tcp -13:22.201 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:22.201 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16824 [Candidate]" term=2 -13:22.202 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:22.202 [DEBUG] TestAgent.server.raft: vote granted: from=b0325f12-bd50-4820-5b3a-fca42f5d4ece term=2 tally=1 -13:22.202 [INFO] TestAgent.server.raft: election won: tally=1 -13:22.202 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16824 [Leader]" -13:22.202 [INFO] TestAgent.server: cluster leadership acquired -13:22.218 [INFO] TestAgent.server: initializing acls -13:22.218 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:22.218 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:22.218 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:22.219 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:22.219 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:22.219 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:22.219 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -13:22.219 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b0325f12-bd50-4820-5b3a-fca42f5d4ece -13:22.219 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b0325f12-bd50-4820-5b3a-fca42f5d4ece.dc1 -13:22.219 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b0325f12-bd50-4820-5b3a-fca42f5d4ece.dc1 area=wan -13:22.219 [INFO] TestAgent.server: Updating LAN server: server="Node-b0325f12-bd50-4820-5b3a-fca42f5d4ece (Addr: tcp/127.0.0.1:16824) (DC: dc1)" -13:22.230 [INFO] TestAgent.server: initializing acls -13:22.230 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:22.231 [INFO] TestAgent.server: New leader elected: payload=Node-b0325f12-bd50-4820-5b3a-fca42f5d4ece -13:22.231 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:22.233 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:22.233 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:22.233 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:22.233 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:22.233 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:22.233 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b0325f12-bd50-4820-5b3a-fca42f5d4ece -13:22.233 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b0325f12-bd50-4820-5b3a-fca42f5d4ece -13:22.234 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b0325f12-bd50-4820-5b3a-fca42f5d4ece -13:22.234 [INFO] TestAgent.server: federation state anti-entropy synced -13:22.317 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b0325f12-bd50-4820-5b3a-fca42f5d4ece -13:22.317 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b0325f12-bd50-4820-5b3a-fca42f5d4ece -13:22.319 [INFO] TestAgent: Updated agent's ACL token: token=default -13:22.551 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -13:22.552 [INFO] TestAgent: Synced node info -13:22.552 [DEBUG] TestAgent: Node info in sync -13:22.573 [INFO] TestAgent: Requesting shutdown -13:22.573 [INFO] TestAgent.server: shutting down server -13:22.573 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:22.573 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:22.573 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:22.573 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:22.573 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:22.573 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:22.573 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:22.574 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:22.574 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:22.574 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:22.574 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:22.574 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:22.574 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:22.574 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:22.574 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -2021-10-31T23:13:22.574Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:22.574Z [INFO] agent.router.manager: shutting down -13:22.574 [INFO] TestAgent: consul server down -13:22.574 [INFO] TestAgent: shutdown complete -13:22.574 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16977 network=tcp -13:22.574 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16977 network=udp -13:22.574 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16978 network=tcp -13:23.090 [INFO] TestAgent: Waiting for endpoints to shut down -13:23.090 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_TokenTriggersFullSync (3.85s) - --- PASS: TestAgent_TokenTriggersFullSync/acl_agent_token (0.87s) - --- PASS: TestAgent_TokenTriggersFullSync/agent (1.03s) - --- PASS: TestAgent_TokenTriggersFullSync/acl_token (0.93s) - --- PASS: TestAgent_TokenTriggersFullSync/default (1.03s) -=== CONT TestAgent_ReloadConfigTLSConfigFailure -2021-10-31T23:13:22.101Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:22.173Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:22.186 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2c083d0b-3d3c-2d8c-3d89-f55c3b9b26b1 Address:127.0.0.1:16975}]" -13:22.187 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2c083d0b-3d3c-2d8c-3d89-f55c3b9b26b1.dc1 127.0.0.1 -13:22.188 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16975 [Follower]" leader= -13:22.189 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2c083d0b-3d3c-2d8c-3d89-f55c3b9b26b1 127.0.0.1 -2021-10-31T23:13:22.189Z [INFO] agent.router: Initializing LAN area manager -13:22.189 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16963 network=udp -13:22.192 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16963 network=tcp -13:22.193 [INFO] TestAgent.server: Adding LAN server: server="Node-2c083d0b-3d3c-2d8c-3d89-f55c3b9b26b1 (Addr: tcp/127.0.0.1:16975) (DC: dc1)" -13:22.222 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2c083d0b-3d3c-2d8c-3d89-f55c3b9b26b1.dc1 area=wan -13:22.230 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16964 network=tcp -13:22.230 [INFO] TestAgent: started state syncer -13:22.230 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16976 network=tcp -13:22.237 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:22.237 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16975 [Candidate]" term=2 -13:22.238 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:22.238 [DEBUG] TestAgent.server.raft: vote granted: from=2c083d0b-3d3c-2d8c-3d89-f55c3b9b26b1 term=2 tally=1 -13:22.238 [INFO] TestAgent.server.raft: election won: tally=1 -13:22.238 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16975 [Leader]" -13:22.238 [INFO] TestAgent.server: cluster leadership acquired -13:22.238 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16975 -13:22.239 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:22.240 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:22.240 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:22.240 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:22.240 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:22.240 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:22.240 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2c083d0b-3d3c-2d8c-3d89-f55c3b9b26b1 -13:22.240 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2c083d0b-3d3c-2d8c-3d89-f55c3b9b26b1 -13:22.241 [INFO] TestAgent.server: New leader elected: payload=Node-2c083d0b-3d3c-2d8c-3d89-f55c3b9b26b1 -13:22.241 [INFO] TestAgent.server: federation state anti-entropy synced -13:22.450 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:22.451 [INFO] TestAgent: Synced node info -13:22.600 [INFO] TestAgent: Requesting shutdown -13:22.600 [INFO] TestAgent.server: shutting down server -13:22.600 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:22.600 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:22.600 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:22.600 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:22.600 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:22.600 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:22.600 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:22.600 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:22.600 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:22.600 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:22.601 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:22.601Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:22.601Z [INFO] agent.router.manager: shutting down -13:22.601 [INFO] TestAgent: consul server down -13:22.601 [INFO] TestAgent: shutdown complete -13:22.601 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16963 network=tcp -13:22.601 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16963 network=udp -13:22.601 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16964 network=tcp -13:23.101 [INFO] TestAgent: Waiting for endpoints to shut down -13:23.101 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogRegister_Service_InvalidAddress (1.06s) - --- PASS: TestCatalogRegister_Service_InvalidAddress/addr_0.0.0.0 (0.00s) - --- PASS: TestCatalogRegister_Service_InvalidAddress/addr_:: (0.00s) - --- PASS: TestCatalogRegister_Service_InvalidAddress/addr_[::] (0.00s) +2022-12-04T05:49:36.328Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:36.346Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:36.349 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c716b5cf-8342-2c87-46f9-ddd73996a761 Address:127.0.0.1:24270}]" +49:36.350 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c716b5cf-8342-2c87-46f9-ddd73996a761.dc1 127.0.0.1 +49:36.351 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c716b5cf-8342-2c87-46f9-ddd73996a761 127.0.0.1 +2022-12-04T05:49:36.351Z [INFO] agent.router: Initializing LAN area manager +49:36.351 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24258 network=udp +49:36.351 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24270 [Follower]" leader= +49:36.352 [INFO] TestAgent.server: Adding LAN server: server="Node-c716b5cf-8342-2c87-46f9-ddd73996a761 (Addr: tcp/127.0.0.1:24270) (DC: dc1)" +49:36.352 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c716b5cf-8342-2c87-46f9-ddd73996a761.dc1 area=wan +49:36.352 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24258 network=tcp +49:36.353 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24266 network=tcp +49:36.353 [INFO] TestAgent: started state syncer +49:36.354 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24236 network=tcp +49:36.399 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:36.399 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24270 [Candidate]" term=2 +49:36.399 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:36.399 [DEBUG] TestAgent.server.raft: vote granted: from=c716b5cf-8342-2c87-46f9-ddd73996a761 term=2 tally=1 +49:36.399 [INFO] TestAgent.server.raft: election won: tally=1 +49:36.399 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24270 [Leader]" +49:36.399 [INFO] TestAgent.server: cluster leadership acquired +49:36.399 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24270 +49:36.400 [INFO] TestAgent.server: New leader elected: payload=Node-c716b5cf-8342-2c87-46f9-ddd73996a761 +49:36.400 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:36.401 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:36.401 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:36.401 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:36.401 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:36.401 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:36.401 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c716b5cf-8342-2c87-46f9-ddd73996a761 +49:36.401 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c716b5cf-8342-2c87-46f9-ddd73996a761 +49:36.402 [INFO] TestAgent.server: federation state anti-entropy synced +49:36.424 [INFO] TestAgent: Requesting shutdown +49:36.432 [INFO] TestAgent.server: shutting down server +49:36.432 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:36.432 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:36.432 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:36.432 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:36.432 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:36.432 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:36.432 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:36.432 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:36.432 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:36.432 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:36.432 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:36.433 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:36.433Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:36.433Z [INFO] agent.router.manager: shutting down +49:36.433 [INFO] TestAgent: consul server down +49:36.433 [INFO] TestAgent: shutdown complete +49:36.433 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24258 network=tcp +49:36.433 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24258 network=udp +49:36.433 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24266 network=tcp +49:36.933 [INFO] TestAgent: Waiting for endpoints to shut down +49:36.933 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Self (1.34s) + --- PASS: TestAgent_Self/no_grpc (0.72s) + --- PASS: TestAgent_Self/normal (0.62s) === CONT TestAgent_RegisterCheck_BadStatus -[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:22.467Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:22.468Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:22.511Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:22.511Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:22.514 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:82a280c7-e448-bdf8-71c4-00b0f758fea4 Address:127.0.0.1:17006}]" -13:22.515 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-82a280c7-e448-bdf8-71c4-00b0f758fea4.dc1 127.0.0.1 -13:22.515 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-82a280c7-e448-bdf8-71c4-00b0f758fea4 127.0.0.1 -2021-10-31T23:13:22.515Z [INFO] agent.router: Initializing LAN area manager -13:22.516 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17001 network=udp -13:22.516 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17006 [Follower]" leader= -13:22.516 [INFO] TestAgent.server: Adding LAN server: server="Node-82a280c7-e448-bdf8-71c4-00b0f758fea4 (Addr: tcp/127.0.0.1:17006) (DC: dc1)" -13:22.516 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-82a280c7-e448-bdf8-71c4-00b0f758fea4.dc1 area=wan -13:22.517 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17001 network=tcp -13:22.538 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17002 network=tcp -13:22.538 [INFO] TestAgent: started state syncer -13:22.538 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17028 network=tcp -13:22.569 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:22.573 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:22.573 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17006 [Candidate]" term=2 -13:22.573 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:22.573 [DEBUG] TestAgent.server.raft: vote granted: from=82a280c7-e448-bdf8-71c4-00b0f758fea4 term=2 tally=1 -13:22.573 [INFO] TestAgent.server.raft: election won: tally=1 -13:22.573 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17006 [Leader]" -13:22.573 [INFO] TestAgent.server: cluster leadership acquired -13:22.573 [INFO] TestAgent.server: New leader elected: payload=Node-82a280c7-e448-bdf8-71c4-00b0f758fea4 -13:22.577 [INFO] TestAgent.server: initializing acls -13:22.578 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:22.578 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:22.578 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:22.579 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:22.579 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:22.579 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:22.579 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-82a280c7-e448-bdf8-71c4-00b0f758fea4 -13:22.579 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-82a280c7-e448-bdf8-71c4-00b0f758fea4.dc1 -13:22.580 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:22.581 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:22.581 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:22.581 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:22.581 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:22.581 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:22.581 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-82a280c7-e448-bdf8-71c4-00b0f758fea4 -13:22.581 [INFO] TestAgent.server: member joined, marking health alive: member=Node-82a280c7-e448-bdf8-71c4-00b0f758fea4 -13:22.581 [INFO] TestAgent.server: Updating LAN server: server="Node-82a280c7-e448-bdf8-71c4-00b0f758fea4 (Addr: tcp/127.0.0.1:17006) (DC: dc1)" -13:22.582 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-82a280c7-e448-bdf8-71c4-00b0f758fea4.dc1 area=wan -13:22.582 [INFO] TestAgent.server: federation state anti-entropy synced -13:22.582 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-82a280c7-e448-bdf8-71c4-00b0f758fea4 -13:22.646 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-82a280c7-e448-bdf8-71c4-00b0f758fea4 -13:22.646 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-82a280c7-e448-bdf8-71c4-00b0f758fea4 -13:22.646 [DEBUG] TestAgent: Check status updated: check=test status=passing -13:22.647 [INFO] TestAgent: Synced node info -13:22.647 [WARN] TestAgent: Check registration blocked by ACLs: check=test accessorID=00000000-0000-0000-0000-000000000002 -13:22.647 [INFO] TestAgent: Requesting shutdown -13:22.647 [INFO] TestAgent.server: shutting down server -13:22.647 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:22.647 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:22.647 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:22.647 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:22.647 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:22.647 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:22.647 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:22.648 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:22.648 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:22.648 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:22.648 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:22.648 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:22.648 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:22.648 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:22.648 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:22.648 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:22.648Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:22.648Z [INFO] agent.router.manager: shutting down -13:22.648 [INFO] TestAgent: consul server down -13:22.648 [INFO] TestAgent: shutdown complete -13:22.648 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17001 network=tcp -13:22.648 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17001 network=udp -13:22.648 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17002 network=tcp -13:23.149 [INFO] TestAgent: Waiting for endpoints to shut down -13:23.149 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_PassCheck_ACLDeny (0.74s) - --- PASS: TestAgent_PassCheck_ACLDeny/no_token (0.00s) - --- PASS: TestAgent_PassCheck_ACLDeny/root_token (0.00s) +2022-12-04T05:49:36.033Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:36.033Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:36.046Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:36.046Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:36.047 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2140edcc-5b9e-8c2b-bbb7-586273593b12 Address:127.0.0.1:23839}]" +49:36.047 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23839 [Follower]" leader= +49:36.047 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2140edcc-5b9e-8c2b-bbb7-586273593b12.dc1 127.0.0.1 +49:36.048 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2140edcc-5b9e-8c2b-bbb7-586273593b12 127.0.0.1 +2022-12-04T05:49:36.048Z [INFO] agent.router: Initializing LAN area manager +49:36.048 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23834 network=udp +49:36.048 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2140edcc-5b9e-8c2b-bbb7-586273593b12.dc1 area=wan +49:36.048 [INFO] TestAgent.server: Adding LAN server: server="Node-2140edcc-5b9e-8c2b-bbb7-586273593b12 (Addr: tcp/127.0.0.1:23839) (DC: dc1)" +49:36.048 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23834 network=tcp +49:36.049 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23835 network=tcp +49:36.049 [INFO] TestAgent: started state syncer +49:36.049 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24208 network=tcp +49:36.098 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:36.104 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:36.104 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23839 [Candidate]" term=2 +49:36.104 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:36.104 [DEBUG] TestAgent.server.raft: vote granted: from=2140edcc-5b9e-8c2b-bbb7-586273593b12 term=2 tally=1 +49:36.104 [INFO] TestAgent.server.raft: election won: tally=1 +49:36.104 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23839 [Leader]" +49:36.104 [INFO] TestAgent.server: cluster leadership acquired +49:36.104 [INFO] TestAgent.server: initializing acls +49:36.105 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:36.105 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:36.105 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:36.106 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:36.106 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:36.106 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:36.106 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-2140edcc-5b9e-8c2b-bbb7-586273593b12 +49:36.106 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-2140edcc-5b9e-8c2b-bbb7-586273593b12.dc1 +49:36.106 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:36.108 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:36.108 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:36.108 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:36.108 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:36.108 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:36.108 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2140edcc-5b9e-8c2b-bbb7-586273593b12 +49:36.108 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2140edcc-5b9e-8c2b-bbb7-586273593b12 +49:36.108 [INFO] TestAgent.server: New leader elected: payload=Node-2140edcc-5b9e-8c2b-bbb7-586273593b12 +49:36.108 [INFO] TestAgent.server: Updating LAN server: server="Node-2140edcc-5b9e-8c2b-bbb7-586273593b12 (Addr: tcp/127.0.0.1:23839) (DC: dc1)" +49:36.108 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-2140edcc-5b9e-8c2b-bbb7-586273593b12.dc1 area=wan +49:36.108 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2140edcc-5b9e-8c2b-bbb7-586273593b12 +49:36.108 [INFO] TestAgent.server: federation state anti-entropy synced +49:36.387 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:36.387 [INFO] TestAgent: Synced node info +49:36.463 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-2140edcc-5b9e-8c2b-bbb7-586273593b12 +49:36.463 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-2140edcc-5b9e-8c2b-bbb7-586273593b12 +49:36.465 [DEBUG] TestAgent: Check status updated: check=test status=passing +49:36.465 [DEBUG] TestAgent: Node info in sync +49:36.465 [WARN] TestAgent: Check registration blocked by ACLs: check=test accessorID=00000000-0000-0000-0000-000000000002 +49:36.465 [INFO] TestAgent: Requesting shutdown +49:36.465 [INFO] TestAgent.server: shutting down server +49:36.465 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:36.465 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:36.465 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:36.465 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:36.465 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:36.465 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:36.465 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:36.466 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:36.466 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:36.466 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:36.466 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:36.466 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:36.466 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:36.466 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:36.466 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:36.466Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:36.466Z [INFO] agent.router.manager: shutting down +49:36.466 [INFO] TestAgent: consul server down +49:36.466 [INFO] TestAgent: shutdown complete +49:36.466 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23834 network=tcp +49:36.466 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23834 network=udp +49:36.466 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23835 network=tcp +49:36.967 [INFO] TestAgent: Waiting for endpoints to shut down +49:36.967 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_UpdateCheck_ACLDeny (0.95s) + --- PASS: TestAgent_UpdateCheck_ACLDeny/no_token (0.00s) + --- PASS: TestAgent_UpdateCheck_ACLDeny/root_token (0.00s) === CONT TestAgent_RegisterCheck_Passing -2021-10-31T23:13:22.233Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:22.260Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:22.267 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6b2e54a7-ca19-0389-e5df-1da9fce0ad13 Address:127.0.0.1:16968}]" -13:22.267 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16968 [Follower]" leader= -13:22.268 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6b2e54a7-ca19-0389-e5df-1da9fce0ad13.dc1 127.0.0.1 -13:22.269 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6b2e54a7-ca19-0389-e5df-1da9fce0ad13 127.0.0.1 -2021-10-31T23:13:22.269Z [INFO] agent.router: Initializing LAN area manager -13:22.269 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17012 network=udp -13:22.269 [INFO] TestAgent.server: Adding LAN server: server="Node-6b2e54a7-ca19-0389-e5df-1da9fce0ad13 (Addr: tcp/127.0.0.1:16968) (DC: dc1)" -13:22.269 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6b2e54a7-ca19-0389-e5df-1da9fce0ad13.dc1 area=wan -13:22.270 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17012 network=tcp -13:22.271 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17013 network=tcp -13:22.271 [INFO] TestAgent: started state syncer -13:22.271 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16969 network=tcp -13:22.305 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:22.305 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16968 [Candidate]" term=2 -13:22.309 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:22.309 [DEBUG] TestAgent.server.raft: vote granted: from=6b2e54a7-ca19-0389-e5df-1da9fce0ad13 term=2 tally=1 -13:22.309 [INFO] TestAgent.server.raft: election won: tally=1 -13:22.309 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16968 [Leader]" -13:22.312 [INFO] TestAgent.server: cluster leadership acquired -13:22.312 [INFO] TestAgent.server: New leader elected: payload=Node-6b2e54a7-ca19-0389-e5df-1da9fce0ad13 -13:22.314 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16968 -13:22.315 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:22.318 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:22.318 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:22.318 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:22.318 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:22.318 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:22.318 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6b2e54a7-ca19-0389-e5df-1da9fce0ad13 -13:22.318 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6b2e54a7-ca19-0389-e5df-1da9fce0ad13 -13:22.319 [INFO] TestAgent.server: federation state anti-entropy synced -13:22.702 [DEBUG] TestAgent: Check status updated: check=test status=warning -13:22.703 [INFO] TestAgent: Synced node info -13:22.703 [INFO] TestAgent: Synced check: check=test -13:22.703 [INFO] TestAgent: Requesting shutdown -13:22.703 [INFO] TestAgent.server: shutting down server -13:22.703 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:22.703 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:22.703 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:22.703 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:22.703 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:22.703 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:22.703 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:22.703 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:22.703 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:22.703 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:22.703 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:22.703 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:22.703Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:22.703Z [INFO] agent.router.manager: shutting down -13:22.704 [INFO] TestAgent: consul server down -13:22.704 [INFO] TestAgent: shutdown complete -13:22.704 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17012 network=tcp -13:22.704 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17012 network=udp -13:22.704 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17013 network=tcp -13:23.206 [INFO] TestAgent: Waiting for endpoints to shut down -13:23.206 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_WarnCheck (1.03s) +=== RUN TestAgent_DeregisterCheckACLDeny/no_token +=== RUN TestAgent_DeregisterCheckACLDeny/root_token +2022-12-04T05:49:36.380Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:36.389Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:36.391 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f550764b-174e-b23f-77d9-fd7901a74323 Address:127.0.0.1:24256}]" +49:36.391 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24256 [Follower]" leader= +49:36.391 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f550764b-174e-b23f-77d9-fd7901a74323.dc1 127.0.0.1 +49:36.392 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f550764b-174e-b23f-77d9-fd7901a74323 127.0.0.1 +2022-12-04T05:49:36.392Z [INFO] agent.router: Initializing LAN area manager +49:36.392 [INFO] TestAgent.server: Adding LAN server: server="Node-f550764b-174e-b23f-77d9-fd7901a74323 (Addr: tcp/127.0.0.1:24256) (DC: dc1)" +49:36.392 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f550764b-174e-b23f-77d9-fd7901a74323.dc1 area=wan +49:36.392 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24237 network=udp +49:36.392 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24237 network=tcp +49:36.393 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24252 network=tcp +49:36.393 [INFO] TestAgent: started state syncer +49:36.393 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24292 network=tcp +49:36.444 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:36.444 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24256 [Candidate]" term=2 +49:36.444 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:36.444 [DEBUG] TestAgent.server.raft: vote granted: from=f550764b-174e-b23f-77d9-fd7901a74323 term=2 tally=1 +49:36.444 [INFO] TestAgent.server.raft: election won: tally=1 +49:36.444 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24256 [Leader]" +49:36.444 [INFO] TestAgent.server: cluster leadership acquired +49:36.445 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24256 +49:36.446 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:36.447 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:36.447 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:36.447 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:36.447 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:36.447 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:36.448 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f550764b-174e-b23f-77d9-fd7901a74323 +49:36.448 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f550764b-174e-b23f-77d9-fd7901a74323 +49:36.448 [INFO] TestAgent.server: New leader elected: payload=Node-f550764b-174e-b23f-77d9-fd7901a74323 +49:36.450 [INFO] TestAgent.server: federation state anti-entropy synced +49:36.508 [DEBUG] TestAgent: Check status updated: check=test status=warning +49:36.509 [INFO] TestAgent: Synced node info +49:36.510 [INFO] TestAgent: Synced check: check=test +49:36.510 [INFO] TestAgent: Requesting shutdown +49:36.510 [INFO] TestAgent.server: shutting down server +49:36.510 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:36.510 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:36.510 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:36.510 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:36.510 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:36.510 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:36.510 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:36.510 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:36.510 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:36.510 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:36.510 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:36.510 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:36.510Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:36.510Z [INFO] agent.router.manager: shutting down +49:36.516 [INFO] TestAgent: consul server down +49:36.516 [INFO] TestAgent: shutdown complete +49:36.516 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24237 network=tcp +49:36.516 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24237 network=udp +49:36.516 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24252 network=tcp +49:37.016 [INFO] TestAgent: Waiting for endpoints to shut down +49:37.016 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_WarnCheck (0.65s) === CONT TestAgent_RegisterCheckScriptsExecRemoteDisable +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestAgent_RegisterCheck_ACLDeny/no_token_-_node_check === RUN TestAgent_RegisterCheck_ACLDeny/svc_token_-_node_check === RUN TestAgent_RegisterCheck_ACLDeny/node_token_-_node_check === RUN TestAgent_RegisterCheck_ACLDeny/no_token_-_svc_check === RUN TestAgent_RegisterCheck_ACLDeny/node_token_-_svc_check === RUN TestAgent_RegisterCheck_ACLDeny/svc_token_-_svc_check -2021-10-31T23:13:22.572Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:22.572Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:22.614Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:22.614Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:22.616 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f527f84a-8996-a082-6e50-e0e5753fad76 Address:127.0.0.1:17027}]" -13:22.617 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f527f84a-8996-a082-6e50-e0e5753fad76.dc1 127.0.0.1 -13:22.617 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17027 [Follower]" leader= -13:22.618 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f527f84a-8996-a082-6e50-e0e5753fad76 127.0.0.1 -2021-10-31T23:13:22.618Z [INFO] agent.router: Initializing LAN area manager -13:22.618 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17022 network=udp -13:22.618 [INFO] TestAgent.server: Adding LAN server: server="Node-f527f84a-8996-a082-6e50-e0e5753fad76 (Addr: tcp/127.0.0.1:17027) (DC: dc1)" -13:22.618 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f527f84a-8996-a082-6e50-e0e5753fad76.dc1 area=wan -13:22.618 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17022 network=tcp -13:22.619 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17023 network=tcp -13:22.619 [INFO] TestAgent: started state syncer -13:22.619 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17042 network=tcp -13:22.665 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:22.665 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17027 [Candidate]" term=2 -13:22.666 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:22.666 [DEBUG] TestAgent.server.raft: vote granted: from=f527f84a-8996-a082-6e50-e0e5753fad76 term=2 tally=1 -13:22.666 [INFO] TestAgent.server.raft: election won: tally=1 -13:22.666 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17027 [Leader]" -13:22.666 [INFO] TestAgent.server: cluster leadership acquired -13:22.666 [INFO] TestAgent.server: initializing acls -13:22.666 [INFO] TestAgent.server: New leader elected: payload=Node-f527f84a-8996-a082-6e50-e0e5753fad76 -13:22.666 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:22.666 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:22.667 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:22.667 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:22.667 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:22.667 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:22.667 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f527f84a-8996-a082-6e50-e0e5753fad76 -13:22.667 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f527f84a-8996-a082-6e50-e0e5753fad76.dc1 -13:22.668 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:22.669 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:22.669 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:22.669 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:22.669 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:22.669 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:22.669 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f527f84a-8996-a082-6e50-e0e5753fad76 -13:22.669 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f527f84a-8996-a082-6e50-e0e5753fad76 -13:22.670 [INFO] TestAgent.server: Updating LAN server: server="Node-f527f84a-8996-a082-6e50-e0e5753fad76 (Addr: tcp/127.0.0.1:17027) (DC: dc1)" -13:22.670 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f527f84a-8996-a082-6e50-e0e5753fad76.dc1 area=wan -13:22.670 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f527f84a-8996-a082-6e50-e0e5753fad76 -13:22.671 [INFO] TestAgent.server: federation state anti-entropy synced -13:22.785 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-f527f84a-8996-a082-6e50-e0e5753fad76 -13:22.786 [DEBUG] TestAgent: removed check: check=test -13:22.787 [INFO] TestAgent: Synced node info -13:22.787 [INFO] TestAgent: Deregistered check: check=test -13:22.787 [INFO] TestAgent: Requesting shutdown -13:22.787 [INFO] TestAgent.server: shutting down server -13:22.787 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:22.787 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:22.787 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:22.787 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:22.787 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:22.787 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:22.787 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:22.787 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:22.787 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:22.787 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:22.787 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:22.787 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:22.787 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:22.787 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:22.787 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:22.787 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:22.787Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:22.787Z [INFO] agent.router.manager: shutting down -13:22.787 [INFO] TestAgent: consul server down -13:22.787 [INFO] TestAgent: shutdown complete -13:22.787 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17022 network=tcp -13:22.787 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17022 network=udp -13:22.788 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17023 network=tcp -13:23.289 [INFO] TestAgent: Waiting for endpoints to shut down -13:23.289 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_DeregisterCheckACLDeny (0.75s) - --- PASS: TestAgent_DeregisterCheckACLDeny/no_token (0.00s) - --- PASS: TestAgent_DeregisterCheckACLDeny/root_token (0.00s) -=== CONT TestAgent_ReloadConfigIncomingRPCConfig -2021-10-31T23:13:22.539Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:22.564Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:22.570 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:165c4134-cbe4-ceef-26a6-0d358ab7a533 Address:127.0.0.1:17020}]" -13:22.571 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-165c4134-cbe4-ceef-26a6-0d358ab7a533.dc1 127.0.0.1 -13:22.571 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-165c4134-cbe4-ceef-26a6-0d358ab7a533 127.0.0.1 -2021-10-31T23:13:22.571Z [INFO] agent.router: Initializing LAN area manager -13:22.571 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17015 network=udp -13:22.571 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17020 [Follower]" leader= -13:22.572 [INFO] TestAgent.server: Adding LAN server: server="Node-165c4134-cbe4-ceef-26a6-0d358ab7a533 (Addr: tcp/127.0.0.1:17020) (DC: dc1)" -13:22.572 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-165c4134-cbe4-ceef-26a6-0d358ab7a533.dc1 area=wan -13:22.572 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17015 network=tcp -13:22.573 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17016 network=tcp -13:22.573 [INFO] TestAgent: started state syncer -13:22.573 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17021 network=tcp -13:22.629 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:22.629 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17020 [Candidate]" term=2 -13:22.630 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:22.630 [DEBUG] TestAgent.server.raft: vote granted: from=165c4134-cbe4-ceef-26a6-0d358ab7a533 term=2 tally=1 -13:22.630 [INFO] TestAgent.server.raft: election won: tally=1 -13:22.630 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17020 [Leader]" -13:22.630 [INFO] TestAgent.server: cluster leadership acquired -13:22.630 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17020 -13:22.631 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:22.632 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:22.632 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:22.632 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:22.632 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:22.632 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:22.633 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-165c4134-cbe4-ceef-26a6-0d358ab7a533 -13:22.633 [INFO] TestAgent.server: member joined, marking health alive: member=Node-165c4134-cbe4-ceef-26a6-0d358ab7a533 -13:22.633 [INFO] TestAgent.server: New leader elected: payload=Node-165c4134-cbe4-ceef-26a6-0d358ab7a533 -13:22.638 [INFO] TestAgent.server: federation state anti-entropy synced -13:22.802 [WARN] TestAgent: check has interval below minimum: check=http minimum_interval=1s -13:22.802 [WARN] TestAgent: check has interval below minimum: check=grpc minimum_interval=1s -13:22.802 [INFO] TestAgent: Requesting shutdown -13:22.802 [INFO] TestAgent.server: shutting down server -13:22.802 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:22.802 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:22.802 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:22.802 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:22.802 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:22.802 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:22.802 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:22.802 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:22.802 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:22.802 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:22.802 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:22.802 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:22.802Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:22.802Z [INFO] agent.router.manager: shutting down -13:22.803 [INFO] TestAgent: consul server down -13:22.803 [INFO] TestAgent: shutdown complete -13:22.803 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17015 network=tcp -13:22.803 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17015 network=udp -13:22.803 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17016 network=tcp -13:23.305 [INFO] TestAgent: Waiting for endpoints to shut down -13:23.305 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RerouteExistingHTTPChecks (0.80s) +2022-12-04T05:49:36.125Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:36.125Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:36.135Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:36.135Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:36.136 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c3f76e63-9750-f3ce-f54d-c21e87926b37 Address:127.0.0.1:24042}]" +49:36.137 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24042 [Follower]" leader= +49:36.137 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c3f76e63-9750-f3ce-f54d-c21e87926b37.dc1 127.0.0.1 +49:36.137 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c3f76e63-9750-f3ce-f54d-c21e87926b37 127.0.0.1 +2022-12-04T05:49:36.138Z [INFO] agent.router: Initializing LAN area manager +49:36.138 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c3f76e63-9750-f3ce-f54d-c21e87926b37.dc1 area=wan +49:36.138 [INFO] TestAgent.server: Adding LAN server: server="Node-c3f76e63-9750-f3ce-f54d-c21e87926b37 (Addr: tcp/127.0.0.1:24042) (DC: dc1)" +49:36.138 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24209 network=udp +49:36.138 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24209 network=tcp +49:36.139 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24038 network=tcp +49:36.139 [INFO] TestAgent: started state syncer +49:36.139 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24159 network=tcp +49:36.185 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:36.186 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24042 [Candidate]" term=2 +49:36.186 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:36.186 [DEBUG] TestAgent.server.raft: vote granted: from=c3f76e63-9750-f3ce-f54d-c21e87926b37 term=2 tally=1 +49:36.186 [INFO] TestAgent.server.raft: election won: tally=1 +49:36.186 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24042 [Leader]" +49:36.186 [INFO] TestAgent.server: cluster leadership acquired +49:36.186 [INFO] TestAgent.server: New leader elected: payload=Node-c3f76e63-9750-f3ce-f54d-c21e87926b37 +49:36.186 [INFO] TestAgent.server: initializing acls +49:36.186 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:36.186 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:36.187 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:36.187 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:36.187 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:36.187 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:36.187 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c3f76e63-9750-f3ce-f54d-c21e87926b37 +49:36.187 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c3f76e63-9750-f3ce-f54d-c21e87926b37.dc1 +49:36.188 [INFO] TestAgent.server: Updating LAN server: server="Node-c3f76e63-9750-f3ce-f54d-c21e87926b37 (Addr: tcp/127.0.0.1:24042) (DC: dc1)" +49:36.188 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:36.188 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c3f76e63-9750-f3ce-f54d-c21e87926b37.dc1 area=wan +49:36.189 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:36.189 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:36.189 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:36.189 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:36.189 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:36.189 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c3f76e63-9750-f3ce-f54d-c21e87926b37 +49:36.189 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c3f76e63-9750-f3ce-f54d-c21e87926b37 +49:36.190 [INFO] TestAgent.server: federation state anti-entropy synced +49:36.190 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c3f76e63-9750-f3ce-f54d-c21e87926b37 +49:36.248 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:36.249 [INFO] TestAgent: Synced node info +49:36.568 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-c3f76e63-9750-f3ce-f54d-c21e87926b37 +49:36.568 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-c3f76e63-9750-f3ce-f54d-c21e87926b37 +49:36.569 [DEBUG] TestAgent: Check status updated: check=test status=critical +49:36.569 [DEBUG] TestAgent: Node info in sync +49:36.569 [WARN] TestAgent: Check registration blocked by ACLs: check=test accessorID=00000000-0000-0000-0000-000000000002 +49:36.569 [INFO] TestAgent: Requesting shutdown +49:36.569 [INFO] TestAgent.server: shutting down server +49:36.569 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:36.569 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:36.569 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:36.569 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:36.569 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:36.569 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:36.569 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:36.569 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:36.569 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:36.569 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:36.569 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:36.569 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:36.569 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:36.569 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:36.570 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:36.570Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:36.570Z [INFO] agent.router.manager: shutting down +49:36.570 [INFO] TestAgent: consul server down +49:36.570 [INFO] TestAgent: shutdown complete +49:36.570 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24209 network=tcp +49:36.570 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24209 network=udp +49:36.570 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24038 network=tcp +49:37.070 [INFO] TestAgent: Waiting for endpoints to shut down +49:37.070 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_FailCheck_ACLDeny (0.96s) + --- PASS: TestAgent_FailCheck_ACLDeny/no_token (0.00s) + --- PASS: TestAgent_FailCheck_ACLDeny/root_token (0.00s) === CONT TestAgent_RegisterCheckScriptsExecDisable -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:22.495Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:22.544Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:22.554 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6b5cc706-07bc-e9dd-b8e8-9ea8991f7349 Address:127.0.0.1:17041}]" -13:22.555 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6b5cc706-07bc-e9dd-b8e8-9ea8991f7349.dc1 127.0.0.1 -13:22.556 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6b5cc706-07bc-e9dd-b8e8-9ea8991f7349 127.0.0.1 -2021-10-31T23:13:22.556Z [INFO] agent.router: Initializing LAN area manager -13:22.556 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17036 network=udp -13:22.556 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17041 [Follower]" leader= -13:22.556 [INFO] TestAgent.server: Adding LAN server: server="Node-6b5cc706-07bc-e9dd-b8e8-9ea8991f7349 (Addr: tcp/127.0.0.1:17041) (DC: dc1)" -13:22.556 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6b5cc706-07bc-e9dd-b8e8-9ea8991f7349.dc1 area=wan -13:22.556 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17036 network=tcp -13:22.557 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17037 network=tcp -13:22.557 [INFO] TestAgent: started state syncer -13:22.557 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17014 network=tcp -13:22.629 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:22.630 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17041 [Candidate]" term=2 -13:22.630 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:22.630 [DEBUG] TestAgent.server.raft: vote granted: from=6b5cc706-07bc-e9dd-b8e8-9ea8991f7349 term=2 tally=1 -13:22.630 [INFO] TestAgent.server.raft: election won: tally=1 -13:22.630 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17041 [Leader]" -13:22.630 [INFO] TestAgent.server: cluster leadership acquired -13:22.630 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17041 -13:22.631 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:22.632 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:22.632 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:22.632 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:22.632 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:22.632 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:22.633 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6b5cc706-07bc-e9dd-b8e8-9ea8991f7349 -13:22.633 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6b5cc706-07bc-e9dd-b8e8-9ea8991f7349 -13:22.633 [INFO] TestAgent.server: New leader elected: payload=Node-6b5cc706-07bc-e9dd-b8e8-9ea8991f7349 -13:22.638 [INFO] TestAgent.server: federation state anti-entropy synced -13:22.797 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:22.797 [INFO] TestAgent: Synced node info -13:22.871 [DEBUG] TestAgent: Check status updated: check=test status=passing -13:22.871 [DEBUG] TestAgent: Node info in sync -13:22.872 [INFO] TestAgent: Synced check: check=test -13:22.872 [INFO] TestAgent: Requesting shutdown -13:22.872 [INFO] TestAgent.server: shutting down server -13:22.872 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:22.872 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:22.872 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:22.872 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:22.872 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:22.873 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:22.873 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:22.873 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:22.874 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:22.878 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" -13:22.878 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -2021-10-31T23:13:22.878Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:22.878Z [INFO] agent.router.manager: shutting down -13:22.878 [INFO] TestAgent: consul server down -13:22.878 [INFO] TestAgent: shutdown complete -13:22.878 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17036 network=tcp -13:22.878 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17036 network=udp -13:22.878 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17037 network=tcp -13:23.381 [INFO] TestAgent: Waiting for endpoints to shut down -13:23.381 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_PassCheck (0.92s) -=== CONT TestAgent_ReloadConfigOutgoingRPCConfig -2021-10-31T23:13:22.504Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:22.554Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:22.556 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:880ae763-dd63-d5e2-e150-b77fad896e7c Address:127.0.0.1:17034}]" -13:22.556 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-880ae763-dd63-d5e2-e150-b77fad896e7c.dc1 127.0.0.1 -13:22.557 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-880ae763-dd63-d5e2-e150-b77fad896e7c 127.0.0.1 -2021-10-31T23:13:22.557Z [INFO] agent.router: Initializing LAN area manager -13:22.557 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17034 [Follower]" leader= -13:22.557 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-880ae763-dd63-d5e2-e150-b77fad896e7c.dc1 area=wan -13:22.557 [INFO] TestAgent.server: Adding LAN server: server="Node-880ae763-dd63-d5e2-e150-b77fad896e7c (Addr: tcp/127.0.0.1:17034) (DC: dc1)" -13:22.557 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17029 network=udp -13:22.557 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17029 network=tcp -13:22.558 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17030 network=tcp -13:22.558 [INFO] TestAgent: started state syncer -13:22.558 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17035 network=tcp -13:22.597 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:22.597 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17034 [Candidate]" term=2 -13:22.598 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:22.598 [DEBUG] TestAgent.server.raft: vote granted: from=880ae763-dd63-d5e2-e150-b77fad896e7c term=2 tally=1 -13:22.598 [INFO] TestAgent.server.raft: election won: tally=1 -13:22.598 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17034 [Leader]" -13:22.598 [INFO] TestAgent.server: cluster leadership acquired -13:22.599 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17034 -13:22.599 [INFO] TestAgent.server: New leader elected: payload=Node-880ae763-dd63-d5e2-e150-b77fad896e7c -13:22.600 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:22.602 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:22.602 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:22.602 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:22.602 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:22.602 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:22.602 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-880ae763-dd63-d5e2-e150-b77fad896e7c -13:22.602 [INFO] TestAgent.server: member joined, marking health alive: member=Node-880ae763-dd63-d5e2-e150-b77fad896e7c -13:22.603 [INFO] TestAgent.server: federation state anti-entropy synced -13:22.917 [WARN] TestAgent: check has interval below minimum: check=http minimum_interval=1s -13:22.917 [WARN] TestAgent: check has interval below minimum: check=grpc minimum_interval=1s -13:22.917 [INFO] TestAgent: Requesting shutdown -13:22.918 [INFO] TestAgent.server: shutting down server -13:22.918 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:22.918 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:22.918 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:22.918 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:22.918 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:22.918 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:22.918 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:22.918 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:22.918 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:22.918 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:22.918 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:22.918 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:22.918Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:22.918Z [INFO] agent.router.manager: shutting down -13:22.918 [INFO] TestAgent: consul server down -13:22.918 [INFO] TestAgent: shutdown complete -13:22.918 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17029 network=tcp -13:22.918 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17029 network=udp -13:22.918 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17030 network=tcp -13:23.421 [INFO] TestAgent: Waiting for endpoints to shut down -13:23.454 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RerouteNewHTTPChecks (1.04s) +2022-12-04T05:49:36.322Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:36.322Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:36.333Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:36.333Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:36.334 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4e3ad8ee-2f1d-2465-e0c5-d23c7c9a7666 Address:127.0.0.1:24158}]" +49:36.334 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24158 [Follower]" leader= +49:36.337 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4e3ad8ee-2f1d-2465-e0c5-d23c7c9a7666.dc1 127.0.0.1 +49:36.338 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4e3ad8ee-2f1d-2465-e0c5-d23c7c9a7666 127.0.0.1 +2022-12-04T05:49:36.338Z [INFO] agent.router: Initializing LAN area manager +49:36.339 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24216 network=udp +49:36.340 [INFO] TestAgent.server: Adding LAN server: server="Node-4e3ad8ee-2f1d-2465-e0c5-d23c7c9a7666 (Addr: tcp/127.0.0.1:24158) (DC: dc1)" +49:36.342 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4e3ad8ee-2f1d-2465-e0c5-d23c7c9a7666.dc1 area=wan +49:36.342 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24216 network=tcp +49:36.343 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24154 network=tcp +49:36.343 [INFO] TestAgent: started state syncer +49:36.343 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24257 network=tcp +49:36.391 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:36.401 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:36.401 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24158 [Candidate]" term=2 +49:36.401 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:36.401 [DEBUG] TestAgent.server.raft: vote granted: from=4e3ad8ee-2f1d-2465-e0c5-d23c7c9a7666 term=2 tally=1 +49:36.401 [INFO] TestAgent.server.raft: election won: tally=1 +49:36.401 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24158 [Leader]" +49:36.402 [INFO] TestAgent.server: cluster leadership acquired +49:36.402 [INFO] TestAgent.server: initializing acls +49:36.402 [INFO] TestAgent.server: New leader elected: payload=Node-4e3ad8ee-2f1d-2465-e0c5-d23c7c9a7666 +49:36.402 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:36.402 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:36.403 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:36.403 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:36.403 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:36.403 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:36.403 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4e3ad8ee-2f1d-2465-e0c5-d23c7c9a7666 +49:36.404 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4e3ad8ee-2f1d-2465-e0c5-d23c7c9a7666.dc1 +49:36.404 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4e3ad8ee-2f1d-2465-e0c5-d23c7c9a7666.dc1 area=wan +49:36.404 [INFO] TestAgent.server: Updating LAN server: server="Node-4e3ad8ee-2f1d-2465-e0c5-d23c7c9a7666 (Addr: tcp/127.0.0.1:24158) (DC: dc1)" +49:36.405 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:36.407 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:36.407 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:36.407 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:36.407 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:36.407 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:36.407 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4e3ad8ee-2f1d-2465-e0c5-d23c7c9a7666 +49:36.407 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4e3ad8ee-2f1d-2465-e0c5-d23c7c9a7666 +49:36.408 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4e3ad8ee-2f1d-2465-e0c5-d23c7c9a7666 +49:36.408 [INFO] TestAgent.server: federation state anti-entropy synced +49:36.561 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:36.561 [INFO] TestAgent: Synced node info +49:36.644 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4e3ad8ee-2f1d-2465-e0c5-d23c7c9a7666 +49:36.644 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4e3ad8ee-2f1d-2465-e0c5-d23c7c9a7666 +49:36.644 [DEBUG] TestAgent: Check status updated: check=test status=warning +49:36.645 [DEBUG] TestAgent: Node info in sync +49:36.645 [WARN] TestAgent: Check registration blocked by ACLs: check=test accessorID=00000000-0000-0000-0000-000000000002 +49:36.645 [INFO] TestAgent: Requesting shutdown +49:36.645 [INFO] TestAgent.server: shutting down server +49:36.645 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:36.645 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:36.645 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:36.645 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:36.645 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:36.645 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:36.645 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:36.645 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:36.645 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:36.645 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:36.645 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:36.645 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:36.645 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:36.645 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:36.645 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:36.645Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:36.645Z [INFO] agent.router.manager: shutting down +49:36.645 [INFO] TestAgent: consul server down +49:36.645 [INFO] TestAgent: shutdown complete +49:36.645 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24216 network=tcp +49:36.645 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24216 network=udp +49:36.645 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24154 network=tcp +49:37.146 [INFO] TestAgent: Waiting for endpoints to shut down +49:37.146 [INFO] TestAgent: Endpoints down === CONT TestAgent_RegisterCheck_Scripts -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:22.749Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:22.786Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:22.819 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:febe2910-6f28-cf0e-0038-e5de3c84a68c Address:127.0.0.1:17048}]" -13:22.822 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-febe2910-6f28-cf0e-0038-e5de3c84a68c.dc1 127.0.0.1 -13:22.824 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-febe2910-6f28-cf0e-0038-e5de3c84a68c 127.0.0.1 -2021-10-31T23:13:22.824Z [INFO] agent.router: Initializing LAN area manager -13:22.824 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17043 network=udp -13:22.824 [INFO] TestAgent.server: Adding LAN server: server="Node-febe2910-6f28-cf0e-0038-e5de3c84a68c (Addr: tcp/127.0.0.1:17048) (DC: dc1)" -13:22.824 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-febe2910-6f28-cf0e-0038-e5de3c84a68c.dc1 area=wan -13:22.824 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17043 network=tcp -13:22.825 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17048 [Follower]" leader= -13:22.834 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17044 network=tcp -13:22.834 [INFO] TestAgent: started state syncer -13:22.834 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17063 network=tcp -13:22.870 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:22.870 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17048 [Candidate]" term=2 -13:22.870 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:22.870 [DEBUG] TestAgent.server.raft: vote granted: from=febe2910-6f28-cf0e-0038-e5de3c84a68c term=2 tally=1 -13:22.870 [INFO] TestAgent.server.raft: election won: tally=1 -13:22.870 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17048 [Leader]" -13:22.870 [INFO] TestAgent.server: cluster leadership acquired -13:22.870 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17048 -13:22.871 [INFO] TestAgent.server: New leader elected: payload=Node-febe2910-6f28-cf0e-0038-e5de3c84a68c -13:22.871 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:22.911 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:22.911 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:22.911 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:22.911 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:22.911 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:22.911 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-febe2910-6f28-cf0e-0038-e5de3c84a68c -13:22.911 [INFO] TestAgent.server: member joined, marking health alive: member=Node-febe2910-6f28-cf0e-0038-e5de3c84a68c -13:22.912 [INFO] TestAgent.server: federation state anti-entropy synced -13:23.206 [INFO] TestAgent: Requesting shutdown -13:23.206 [INFO] TestAgent.server: shutting down server -13:23.206 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:23.206 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:23.206 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:23.206 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:23.206 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:23.206 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:23.206 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:23.206 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:23.206 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:23.206 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:23.206 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:23.206 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:23.207Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:23.207Z [INFO] agent.router.manager: shutting down -13:23.207 [INFO] TestAgent: consul server down -13:23.207 [INFO] TestAgent: shutdown complete -13:23.207 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17043 network=tcp -13:23.207 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17043 network=udp -13:23.207 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17044 network=tcp -13:23.709 [INFO] TestAgent: Waiting for endpoints to shut down -13:23.709 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_consulConfig_RaftTrailingLogs (0.99s) -=== CONT TestAgent_SecurityChecks -2021-10-31T23:13:23.047Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.134Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:23.136 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4a6d6ff8-4a59-91bb-98b1-c9b2c410e63f Address:127.0.0.1:17076}]" -13:23.137 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17076 [Follower]" leader= -13:23.150 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4a6d6ff8-4a59-91bb-98b1-c9b2c410e63f.dc1 127.0.0.1 -13:23.170 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4a6d6ff8-4a59-91bb-98b1-c9b2c410e63f 127.0.0.1 -2021-10-31T23:13:23.170Z [INFO] agent.router: Initializing LAN area manager -13:23.171 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17071 network=udp -13:23.171 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4a6d6ff8-4a59-91bb-98b1-c9b2c410e63f.dc1 area=wan -13:23.171 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17071 network=tcp -13:23.173 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17072 network=tcp -13:23.173 [INFO] TestAgent: started state syncer -13:23.173 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17084 network=tcp -13:23.174 [INFO] TestAgent.server: Adding LAN server: server="Node-4a6d6ff8-4a59-91bb-98b1-c9b2c410e63f (Addr: tcp/127.0.0.1:17076) (DC: dc1)" -13:23.185 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:23.186 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17076 [Candidate]" term=2 -13:23.203 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:23.203 [DEBUG] TestAgent.server.raft: vote granted: from=4a6d6ff8-4a59-91bb-98b1-c9b2c410e63f term=2 tally=1 -13:23.203 [INFO] TestAgent.server.raft: election won: tally=1 -13:23.203 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17076 [Leader]" -13:23.203 [INFO] TestAgent.server: cluster leadership acquired -13:23.203 [INFO] TestAgent.server: initializing acls -13:23.204 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:23.204 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:23.204 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:23.205 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:23.205 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:23.205 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:23.205 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4a6d6ff8-4a59-91bb-98b1-c9b2c410e63f -13:23.205 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4a6d6ff8-4a59-91bb-98b1-c9b2c410e63f.dc1 -13:23.205 [INFO] TestAgent.server: New leader elected: payload=Node-4a6d6ff8-4a59-91bb-98b1-c9b2c410e63f -13:23.206 [INFO] TestAgent.server: Updating LAN server: server="Node-4a6d6ff8-4a59-91bb-98b1-c9b2c410e63f (Addr: tcp/127.0.0.1:17076) (DC: dc1)" -13:23.206 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4a6d6ff8-4a59-91bb-98b1-c9b2c410e63f.dc1 area=wan -13:23.206 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:23.208 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:23.208 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:23.208 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:23.208 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:23.208 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:23.208 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4a6d6ff8-4a59-91bb-98b1-c9b2c410e63f -13:23.208 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4a6d6ff8-4a59-91bb-98b1-c9b2c410e63f -13:23.209 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4a6d6ff8-4a59-91bb-98b1-c9b2c410e63f -13:23.218 [INFO] TestAgent.server: federation state anti-entropy synced -13:23.264 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4a6d6ff8-4a59-91bb-98b1-c9b2c410e63f -13:23.264 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4a6d6ff8-4a59-91bb-98b1-c9b2c410e63f -13:23.275 [INFO] TestAgent: Synced node info -13:23.275 [INFO] TestAgent: Synced service: service=foo:1234 -13:23.279 [DEBUG] TestAgent: Node info in sync -13:23.279 [DEBUG] TestAgent: Service in sync: service=foo:1234 -13:23.279 [INFO] TestAgent: Synced check: check=test -13:23.280 [DEBUG] TestAgent: Node info in sync -13:23.280 [DEBUG] TestAgent: Service in sync: service=foo:1234 -13:23.280 [DEBUG] TestAgent: Check in sync: check=test -13:23.281 [INFO] TestAgent: Synced check: check=test2 -13:23.281 [INFO] TestAgent: Requesting shutdown -13:23.281 [INFO] TestAgent.server: shutting down server -13:23.281 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:23.281 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:23.281 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:23.281 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:23.281 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:23.281 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:23.281 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:23.281 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:23.281 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:23.281 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:23.281 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:23.281 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:23.281 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:23.281 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:23.281 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:23.281 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:23.281Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:23.281Z [INFO] agent.router.manager: shutting down -13:23.281 [INFO] TestAgent: consul server down -13:23.281 [INFO] TestAgent: shutdown complete -13:23.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17071 network=tcp -13:23.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17071 network=udp -13:23.281 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17072 network=tcp -13:23.785 [INFO] TestAgent: Waiting for endpoints to shut down -13:23.786 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RegisterCheck_ACLDeny (0.78s) - --- PASS: TestAgent_RegisterCheck_ACLDeny/no_token_-_node_check (0.00s) - --- PASS: TestAgent_RegisterCheck_ACLDeny/svc_token_-_node_check (0.00s) - --- PASS: TestAgent_RegisterCheck_ACLDeny/node_token_-_node_check (0.00s) - --- PASS: TestAgent_RegisterCheck_ACLDeny/no_token_-_svc_check (0.00s) - --- PASS: TestAgent_RegisterCheck_ACLDeny/node_token_-_svc_check (0.00s) - --- PASS: TestAgent_RegisterCheck_ACLDeny/svc_token_-_svc_check (0.00s) -=== CONT TestAgent_ForceLeavePrune -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=255)_compressed=true -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:22.960Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.021Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:23.028 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2de4542b-f0fe-6a9b-a9f4-907eeefaa218 Address:127.0.0.1:17069}]" -13:23.030 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17069 [Follower]" leader= -13:23.090 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2de4542b-f0fe-6a9b-a9f4-907eeefaa218.dc1 127.0.0.1 -13:23.091 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2de4542b-f0fe-6a9b-a9f4-907eeefaa218 127.0.0.1 -2021-10-31T23:13:23.091Z [INFO] agent.router: Initializing LAN area manager -13:23.091 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17064 network=udp -13:23.091 [INFO] TestAgent.server: Adding LAN server: server="Node-2de4542b-f0fe-6a9b-a9f4-907eeefaa218 (Addr: tcp/127.0.0.1:17069) (DC: dc1)" -13:23.091 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2de4542b-f0fe-6a9b-a9f4-907eeefaa218.dc1 area=wan -13:23.091 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17064 network=tcp -13:23.092 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17065 network=tcp -13:23.092 [INFO] TestAgent: started state syncer -13:23.092 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17049 network=tcp -13:23.092 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:23.092 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17069 [Candidate]" term=2 -13:23.092 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:23.092 [DEBUG] TestAgent.server.raft: vote granted: from=2de4542b-f0fe-6a9b-a9f4-907eeefaa218 term=2 tally=1 -13:23.092 [INFO] TestAgent.server.raft: election won: tally=1 -13:23.092 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17069 [Leader]" -13:23.093 [INFO] TestAgent.server: cluster leadership acquired -13:23.093 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17069 -13:23.093 [INFO] TestAgent.server: New leader elected: payload=Node-2de4542b-f0fe-6a9b-a9f4-907eeefaa218 -13:23.097 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:23.099 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:23.100 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:23.100 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:23.100 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:23.100 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:23.100 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2de4542b-f0fe-6a9b-a9f4-907eeefaa218 -13:23.100 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2de4542b-f0fe-6a9b-a9f4-907eeefaa218 -13:23.100 [INFO] TestAgent.server: federation state anti-entropy synced -13:23.109 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:23.110 [INFO] TestAgent: Synced node info -13:23.110 [DEBUG] TestAgent: Node info in sync -13:23.366 [DEBUG] TestAgent: removed check: check=test -13:23.366 [DEBUG] TestAgent: Node info in sync -13:23.367 [INFO] TestAgent: Deregistered check: check=test -13:23.367 [INFO] TestAgent: Requesting shutdown -13:23.367 [INFO] TestAgent.server: shutting down server -13:23.367 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:23.367 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:23.367 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:23.367 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:23.367 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:23.367 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:23.367 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:23.367 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:23.367 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:23.367 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:23.367 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:23.367Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:23.367Z [INFO] agent.router.manager: shutting down -13:23.367 [INFO] TestAgent: consul server down -13:23.367 [INFO] TestAgent: shutdown complete -13:23.367 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17064 network=tcp -13:23.367 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17064 network=udp -13:23.367 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17065 network=tcp -13:23.870 [INFO] TestAgent: Waiting for endpoints to shut down -13:23.870 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_DeregisterCheck (0.96s) -=== CONT TestAgent_loadTokens -2021-10-31T23:13:23.090Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.147Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:23.170 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:51f9350c-8aa4-db14-94e2-444ee641d56f Address:127.0.0.1:17055}]" -13:23.171 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-51f9350c-8aa4-db14-94e2-444ee641d56f.dc1 127.0.0.1 -13:23.172 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-51f9350c-8aa4-db14-94e2-444ee641d56f 127.0.0.1 -2021-10-31T23:13:23.172Z [INFO] agent.router: Initializing LAN area manager -13:23.172 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17050 network=udp -13:23.172 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17055 [Follower]" leader= -13:23.176 [INFO] TestAgent.server: Adding LAN server: server="Node-51f9350c-8aa4-db14-94e2-444ee641d56f (Addr: tcp/127.0.0.1:17055) (DC: dc1)" -13:23.176 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-51f9350c-8aa4-db14-94e2-444ee641d56f.dc1 area=wan -13:23.176 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17050 network=tcp -13:23.194 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17051 network=tcp -13:23.194 [INFO] TestAgent: started state syncer -13:23.194 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17070 network=tcp -13:23.229 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:23.239 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17055 [Candidate]" term=2 -13:23.239 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:23.239 [DEBUG] TestAgent.server.raft: vote granted: from=51f9350c-8aa4-db14-94e2-444ee641d56f term=2 tally=1 -13:23.239 [INFO] TestAgent.server.raft: election won: tally=1 -13:23.239 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17055 [Leader]" -13:23.240 [INFO] TestAgent.server: cluster leadership acquired -13:23.258 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17055 -13:23.259 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:23.260 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:23.260 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:23.260 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:23.260 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:23.260 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:23.260 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-51f9350c-8aa4-db14-94e2-444ee641d56f -13:23.260 [INFO] TestAgent.server: member joined, marking health alive: member=Node-51f9350c-8aa4-db14-94e2-444ee641d56f -13:23.261 [INFO] TestAgent.server: New leader elected: payload=Node-51f9350c-8aa4-db14-94e2-444ee641d56f -13:23.262 [INFO] TestAgent.server: federation state anti-entropy synced -13:23.347 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:23.348 [INFO] TestAgent: Synced node info -13:23.348 [DEBUG] TestAgent: Node info in sync -13:23.366 [INFO] TestAgent: Requesting shutdown -13:23.366 [INFO] TestAgent.server: shutting down server -13:23.366 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:23.366 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:23.366 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:23.366 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:23.366 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:23.366 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:23.366 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:23.366 [INFO] TestAgent.server: shutting down Connect CA roots monitor -13:23.366 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:23.366 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:23.366 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:23.366 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:23.366Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:23.366Z [INFO] agent.router.manager: shutting down -13:23.367 [INFO] TestAgent: consul server down -13:23.367 [INFO] TestAgent: shutdown complete -13:23.367 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17050 network=tcp -13:23.367 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17050 network=udp -13:23.367 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17051 network=tcp -13:23.873 [INFO] TestAgent: Waiting for endpoints to shut down -13:23.873 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_consulConfig_AutoEncryptAllowTLS (0.88s) -=== CONT TestAgent_ForceLeave_ACLDeny +--- PASS: TestAgent_WarnCheck_ACLDeny (0.84s) + --- PASS: TestAgent_WarnCheck_ACLDeny/no_token (0.00s) + --- PASS: TestAgent_WarnCheck_ACLDeny/root_token (0.00s) +2022-12-04T05:49:36.458Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:36.459Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:36.472Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:36.472Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:36.474 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e765a016-156c-3315-8e4b-2d25668a99bf Address:127.0.0.1:24263}]" +49:36.474 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24263 [Follower]" leader= +49:36.475 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e765a016-156c-3315-8e4b-2d25668a99bf.dc1 127.0.0.1 +49:36.475 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e765a016-156c-3315-8e4b-2d25668a99bf 127.0.0.1 +2022-12-04T05:49:36.475Z [INFO] agent.router: Initializing LAN area manager +49:36.475 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24293 network=udp +49:36.475 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e765a016-156c-3315-8e4b-2d25668a99bf.dc1 area=wan +49:36.476 [INFO] TestAgent.server: Adding LAN server: server="Node-e765a016-156c-3315-8e4b-2d25668a99bf (Addr: tcp/127.0.0.1:24263) (DC: dc1)" +49:36.476 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24293 network=tcp +49:36.476 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24259 network=tcp +49:36.476 [INFO] TestAgent: started state syncer +49:36.476 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24173 network=tcp +49:36.526 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:36.540 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:36.540 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24263 [Candidate]" term=2 +49:36.540 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:36.540 [DEBUG] TestAgent.server.raft: vote granted: from=e765a016-156c-3315-8e4b-2d25668a99bf term=2 tally=1 +49:36.540 [INFO] TestAgent.server.raft: election won: tally=1 +49:36.540 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24263 [Leader]" +49:36.540 [INFO] TestAgent.server: cluster leadership acquired +49:36.541 [INFO] TestAgent.server: New leader elected: payload=Node-e765a016-156c-3315-8e4b-2d25668a99bf +49:36.541 [INFO] TestAgent.server: initializing acls +49:36.541 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:36.541 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:36.542 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:36.542 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:36.542 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:36.542 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:36.542 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e765a016-156c-3315-8e4b-2d25668a99bf +49:36.542 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e765a016-156c-3315-8e4b-2d25668a99bf.dc1 +49:36.542 [INFO] TestAgent.server: Updating LAN server: server="Node-e765a016-156c-3315-8e4b-2d25668a99bf (Addr: tcp/127.0.0.1:24263) (DC: dc1)" +49:36.542 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e765a016-156c-3315-8e4b-2d25668a99bf.dc1 area=wan +49:36.543 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:36.544 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:36.544 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:36.544 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:36.544 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:36.544 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:36.544 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e765a016-156c-3315-8e4b-2d25668a99bf +49:36.544 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e765a016-156c-3315-8e4b-2d25668a99bf +49:36.544 [INFO] TestAgent.server: federation state anti-entropy synced +49:36.545 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e765a016-156c-3315-8e4b-2d25668a99bf +49:36.701 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e765a016-156c-3315-8e4b-2d25668a99bf +49:36.701 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e765a016-156c-3315-8e4b-2d25668a99bf +49:36.701 [DEBUG] TestAgent: Check status updated: check=test status=passing +49:36.702 [INFO] TestAgent: Synced node info +49:36.702 [WARN] TestAgent: Check registration blocked by ACLs: check=test accessorID=00000000-0000-0000-0000-000000000002 +49:36.702 [INFO] TestAgent: Requesting shutdown +49:36.702 [INFO] TestAgent.server: shutting down server +49:36.702 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:36.702 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:36.702 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:36.702 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:36.702 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:36.702 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:36.702 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:36.702 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:36.702 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:36.702 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:36.702 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:36.702 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:36.702 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:36.702 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:36.702 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:36.703 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:36.703Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:36.703Z [INFO] agent.router.manager: shutting down +49:36.703 [INFO] TestAgent: consul server down +49:36.703 [INFO] TestAgent: shutdown complete +49:36.703 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24293 network=tcp +49:36.703 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24293 network=udp +49:36.703 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24259 network=tcp +49:37.203 [INFO] TestAgent: Waiting for endpoints to shut down +49:37.203 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_PassCheck_ACLDeny (0.76s) + --- PASS: TestAgent_PassCheck_ACLDeny/no_token (0.00s) + --- PASS: TestAgent_PassCheck_ACLDeny/root_token (0.00s) +=== CONT TestAgent_RegisterCheck +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=33)_compressed=false#01 +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestAgent_RegisterCheck_Scripts/==_Consul_1.0.0_as_node_check === RUN TestAgent_RegisterCheck_Scripts/==_Consul_1.0.0_as_top-level_service_check === RUN TestAgent_RegisterCheck_Scripts/==_Consul_1.0.0_as_slice-based_service_check @@ -39301,5062 +48203,2641 @@ === RUN TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_as_node_check === RUN TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_as_top-level_service_check === RUN TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_as_slice-based_service_check -2021-10-31T23:13:23.280Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.358Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:23.362 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6f575759-5da7-a83c-b4f1-9e8cdb92082e Address:127.0.0.1:17125}]" -13:23.363 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6f575759-5da7-a83c-b4f1-9e8cdb92082e.dc1 127.0.0.1 -13:23.364 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6f575759-5da7-a83c-b4f1-9e8cdb92082e 127.0.0.1 -2021-10-31T23:13:23.364Z [INFO] agent.router: Initializing LAN area manager -13:23.365 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17120 network=udp -13:23.365 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17125 [Follower]" leader= -13:23.365 [INFO] TestAgent.server: Adding LAN server: server="Node-6f575759-5da7-a83c-b4f1-9e8cdb92082e (Addr: tcp/127.0.0.1:17125) (DC: dc1)" -13:23.366 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17120 network=tcp -13:23.366 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6f575759-5da7-a83c-b4f1-9e8cdb92082e.dc1 area=wan -13:23.367 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17121 network=tcp -13:23.367 [INFO] TestAgent: started state syncer -13:23.367 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17126 network=tcp -13:23.437 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:23.437 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17125 [Candidate]" term=2 -13:23.438 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:23.438 [DEBUG] TestAgent.server.raft: vote granted: from=6f575759-5da7-a83c-b4f1-9e8cdb92082e term=2 tally=1 -13:23.438 [INFO] TestAgent.server.raft: election won: tally=1 -13:23.438 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17125 [Leader]" -13:23.438 [INFO] TestAgent.server: cluster leadership acquired -13:23.438 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17125 -13:23.439 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:23.440 [INFO] TestAgent.server: New leader elected: payload=Node-6f575759-5da7-a83c-b4f1-9e8cdb92082e -13:23.440 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:23.440 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:23.440 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:23.440 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:23.440 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:23.440 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6f575759-5da7-a83c-b4f1-9e8cdb92082e -13:23.440 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6f575759-5da7-a83c-b4f1-9e8cdb92082e -13:23.450 [INFO] TestAgent.server: federation state anti-entropy synced -13:23.466 [INFO] TestAgent: Requesting shutdown -13:23.466 [INFO] TestAgent.server: shutting down server -13:23.466 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:23.466 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:23.466 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:23.466 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:23.466 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:23.466 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:23.466 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:23.466 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:23.466 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:23.466 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:23.466 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:23.466 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:23.467Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:23.467Z [INFO] agent.router.manager: shutting down -13:23.467 [INFO] TestAgent: consul server down -13:23.467 [INFO] TestAgent: shutdown complete -13:23.467 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17120 network=tcp -13:23.467 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17120 network=udp -13:23.467 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17121 network=tcp -13:23.969 [INFO] TestAgent: Waiting for endpoints to shut down -13:23.969 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RegisterCheckScriptsExecRemoteDisable (0.76s) +2022-12-04T05:49:36.734Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:36.744Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:36.746 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d0481a1b-65ca-efbc-62de-09550d79e631 Address:127.0.0.1:24172}]" +49:36.746 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24172 [Follower]" leader= +49:36.746 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d0481a1b-65ca-efbc-62de-09550d79e631.dc1 127.0.0.1 +49:36.747 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d0481a1b-65ca-efbc-62de-09550d79e631 127.0.0.1 +2022-12-04T05:49:36.747Z [INFO] agent.router: Initializing LAN area manager +49:36.747 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d0481a1b-65ca-efbc-62de-09550d79e631.dc1 area=wan +49:36.747 [INFO] TestAgent.server: Adding LAN server: server="Node-d0481a1b-65ca-efbc-62de-09550d79e631 (Addr: tcp/127.0.0.1:24172) (DC: dc1)" +49:36.747 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24230 network=tcp +49:36.747 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24230 network=udp +49:36.748 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24168 network=tcp +49:36.748 [INFO] TestAgent: started state syncer +49:36.748 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24250 network=tcp +49:36.799 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:36.799 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24172 [Candidate]" term=2 +49:36.799 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:36.799 [DEBUG] TestAgent.server.raft: vote granted: from=d0481a1b-65ca-efbc-62de-09550d79e631 term=2 tally=1 +49:36.799 [INFO] TestAgent.server.raft: election won: tally=1 +49:36.799 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24172 [Leader]" +49:36.800 [INFO] TestAgent.server: cluster leadership acquired +49:36.800 [INFO] TestAgent.server: New leader elected: payload=Node-d0481a1b-65ca-efbc-62de-09550d79e631 +49:36.800 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24172 +49:36.801 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:36.802 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:36.802 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:36.802 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:36.802 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:36.802 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:36.802 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d0481a1b-65ca-efbc-62de-09550d79e631 +49:36.802 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d0481a1b-65ca-efbc-62de-09550d79e631 +49:36.803 [INFO] TestAgent.server: federation state anti-entropy synced +49:36.837 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:36.837 [INFO] TestAgent: Synced node info +49:36.837 [DEBUG] TestAgent: Node info in sync +49:36.892 [DEBUG] TestAgent: removed check: check=test +49:36.892 [DEBUG] TestAgent: Node info in sync +49:36.892 [INFO] TestAgent: Deregistered check: check=test +49:36.892 [INFO] TestAgent: Requesting shutdown +49:36.892 [INFO] TestAgent.server: shutting down server +49:36.892 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:36.892 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:36.892 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:36.892 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:36.892 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:36.892 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:36.892 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:36.892 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:36.892 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:36.892 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:36.892 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:36.892Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:36.892Z [INFO] agent.router.manager: shutting down +49:36.892 [INFO] TestAgent: consul server down +49:36.892 [INFO] TestAgent: shutdown complete +49:36.892 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24230 network=tcp +49:36.892 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24230 network=udp +49:36.892 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24168 network=tcp +49:37.396 [INFO] TestAgent: Waiting for endpoints to shut down +49:37.396 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_DeregisterCheck (0.67s) +=== CONT TestAgent_ForceLeavePrune +=== RUN TestAgent_JoinWAN_viaMeshGateway/dc1_to_dc2 +=== RUN TestAgent_JoinWAN_viaMeshGateway/dc1_to_dc3 +=== RUN TestAgent_JoinWAN_viaMeshGateway/dc2_to_dc1 +=== RUN TestAgent_JoinWAN_viaMeshGateway/dc2_to_dc3 +=== RUN TestAgent_JoinWAN_viaMeshGateway/dc3_to_dc1 +=== RUN TestAgent_JoinWAN_viaMeshGateway/dc3_to_dc2 +2022-12-04T05:49:36.536Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:36.536Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:36.549Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:36.549Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:36.551 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:30fef304-bc45-9898-1883-08a05aaabef6 Address:127.0.0.1:23292}]" +49:36.551 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23292 [Follower]" leader= +49:36.552 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-30fef304-bc45-9898-1883-08a05aaabef6.dc1 127.0.0.1 +49:36.553 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-30fef304-bc45-9898-1883-08a05aaabef6 127.0.0.1 +2022-12-04T05:49:36.553Z [INFO] agent.router: Initializing LAN area manager +49:36.553 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-30fef304-bc45-9898-1883-08a05aaabef6.dc1 area=wan +49:36.553 [INFO] TestAgent.server: Adding LAN server: server="Node-30fef304-bc45-9898-1883-08a05aaabef6 (Addr: tcp/127.0.0.1:23292) (DC: dc1)" +49:36.553 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24174 network=udp +49:36.554 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24174 network=tcp +49:36.560 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23288 network=tcp +49:36.560 [INFO] TestAgent: started state syncer +49:36.561 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24201 network=tcp +49:36.598 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:36.598 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23292 [Candidate]" term=2 +49:36.598 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:36.598 [DEBUG] TestAgent.server.raft: vote granted: from=30fef304-bc45-9898-1883-08a05aaabef6 term=2 tally=1 +49:36.598 [INFO] TestAgent.server.raft: election won: tally=1 +49:36.598 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23292 [Leader]" +49:36.599 [INFO] TestAgent.server: cluster leadership acquired +49:36.599 [INFO] TestAgent.server: New leader elected: payload=Node-30fef304-bc45-9898-1883-08a05aaabef6 +49:36.599 [INFO] TestAgent.server: initializing acls +49:36.599 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:36.599 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:36.600 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:36.600 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:36.600 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:36.600 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:36.600 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-30fef304-bc45-9898-1883-08a05aaabef6 +49:36.600 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-30fef304-bc45-9898-1883-08a05aaabef6.dc1 +49:36.601 [INFO] TestAgent.server: Updating LAN server: server="Node-30fef304-bc45-9898-1883-08a05aaabef6 (Addr: tcp/127.0.0.1:23292) (DC: dc1)" +49:36.601 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-30fef304-bc45-9898-1883-08a05aaabef6.dc1 area=wan +49:36.601 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:36.602 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:36.602 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:36.602 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:36.602 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:36.602 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:36.602 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-30fef304-bc45-9898-1883-08a05aaabef6 +49:36.602 [INFO] TestAgent.server: member joined, marking health alive: member=Node-30fef304-bc45-9898-1883-08a05aaabef6 +49:36.603 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-30fef304-bc45-9898-1883-08a05aaabef6 +49:36.603 [INFO] TestAgent.server: federation state anti-entropy synced +49:36.960 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-30fef304-bc45-9898-1883-08a05aaabef6 +49:36.960 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-30fef304-bc45-9898-1883-08a05aaabef6 +49:36.962 [INFO] TestAgent: Updated agent's ACL token: token=acl_token +49:36.984 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +49:36.986 [INFO] TestAgent: Synced node info +49:36.988 [INFO] TestAgent: Requesting shutdown +49:36.988 [INFO] TestAgent.server: shutting down server +49:36.988 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:36.988 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:36.988 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:36.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:36.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:36.988 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:36.988 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:36.989 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:36.989 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:36.989 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:36.989 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:36.989 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:36.989 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:36.989 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:36.989 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:36.989Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:36.989Z [INFO] agent.router.manager: shutting down +49:36.989 [INFO] TestAgent: consul server down +49:36.989 [INFO] TestAgent: shutdown complete +49:36.989 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24174 network=tcp +49:36.989 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24174 network=udp +49:36.989 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23288 network=tcp +49:37.490 [INFO] TestAgent: Waiting for endpoints to shut down +49:37.490 [INFO] TestAgent: Endpoints down +=== RUN TestAgent_TokenTriggersFullSync/default +2022-12-04T05:49:36.735Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:36.736Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:36.746Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:36.746Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:36.748 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:765167cc-aa3c-4493-d7e8-9a22f4b87cdf Address:127.0.0.1:24277}]" +49:36.748 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24277 [Follower]" leader= +49:36.748 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-765167cc-aa3c-4493-d7e8-9a22f4b87cdf.dc1 127.0.0.1 +49:36.749 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-765167cc-aa3c-4493-d7e8-9a22f4b87cdf 127.0.0.1 +2022-12-04T05:49:36.749Z [INFO] agent.router: Initializing LAN area manager +49:36.749 [INFO] TestAgent.server: Adding LAN server: server="Node-765167cc-aa3c-4493-d7e8-9a22f4b87cdf (Addr: tcp/127.0.0.1:24277) (DC: dc1)" +49:36.749 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-765167cc-aa3c-4493-d7e8-9a22f4b87cdf.dc1 area=wan +49:36.749 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23294 network=tcp +49:36.749 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23294 network=udp +49:36.749 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24273 network=tcp +49:36.749 [INFO] TestAgent: started state syncer +49:36.750 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24229 network=tcp +49:36.795 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:36.795 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24277 [Candidate]" term=2 +49:36.795 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:36.795 [DEBUG] TestAgent.server.raft: vote granted: from=765167cc-aa3c-4493-d7e8-9a22f4b87cdf term=2 tally=1 +49:36.795 [INFO] TestAgent.server.raft: election won: tally=1 +49:36.795 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24277 [Leader]" +49:36.795 [INFO] TestAgent.server: cluster leadership acquired +49:36.796 [INFO] TestAgent.server: New leader elected: payload=Node-765167cc-aa3c-4493-d7e8-9a22f4b87cdf +49:36.796 [INFO] TestAgent.server: initializing acls +49:36.796 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:36.796 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:36.796 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:36.797 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:36.797 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:36.797 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:36.797 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-765167cc-aa3c-4493-d7e8-9a22f4b87cdf +49:36.797 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-765167cc-aa3c-4493-d7e8-9a22f4b87cdf.dc1 +49:36.797 [INFO] TestAgent.server: Updating LAN server: server="Node-765167cc-aa3c-4493-d7e8-9a22f4b87cdf (Addr: tcp/127.0.0.1:24277) (DC: dc1)" +49:36.797 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-765167cc-aa3c-4493-d7e8-9a22f4b87cdf.dc1 area=wan +49:36.798 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:36.799 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:36.799 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:36.799 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:36.799 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:36.799 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:36.799 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-765167cc-aa3c-4493-d7e8-9a22f4b87cdf +49:36.799 [INFO] TestAgent.server: member joined, marking health alive: member=Node-765167cc-aa3c-4493-d7e8-9a22f4b87cdf +49:36.799 [INFO] TestAgent.server: federation state anti-entropy synced +49:36.800 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-765167cc-aa3c-4493-d7e8-9a22f4b87cdf +49:36.878 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:36.878 [INFO] TestAgent: Synced node info +49:36.896 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:36.896 [DEBUG] TestAgent: Node info in sync +49:36.896 [DEBUG] TestAgent: Node info in sync +49:36.978 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-765167cc-aa3c-4493-d7e8-9a22f4b87cdf +49:36.979 [DEBUG] TestAgent: removed check: check=test +49:36.979 [DEBUG] TestAgent: Node info in sync +49:36.979 [INFO] TestAgent: Deregistered check: check=test +49:36.979 [INFO] TestAgent: Requesting shutdown +49:36.979 [INFO] TestAgent.server: shutting down server +49:36.979 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:36.979 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:36.979 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:36.979 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:36.979 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:36.979 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:36.979 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:36.979 [DEBUG] TestAgent: Node info in sync +49:36.979 [DEBUG] TestAgent: Node info in sync +49:36.979 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:36.979 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:36.979 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:36.979 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:36.979 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:36.979 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:36.979 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:36.979 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:36.980Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:36.980Z [INFO] agent.router.manager: shutting down +49:36.980 [INFO] TestAgent: consul server down +49:36.980 [INFO] TestAgent: shutdown complete +49:36.980 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23294 network=tcp +49:36.980 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23294 network=udp +49:36.992 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24273 network=tcp +49:37.492 [INFO] TestAgent: Waiting for endpoints to shut down +49:37.493 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_DeregisterCheckACLDeny (0.77s) + --- PASS: TestAgent_DeregisterCheckACLDeny/no_token (0.00s) + --- PASS: TestAgent_DeregisterCheckACLDeny/root_token (0.00s) +=== CONT TestAgent_ForceLeave_ACLDeny +2022-12-04T05:49:36.695Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:36.706Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:36.707 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:29fb0c63-c3d0-9218-a1aa-7cd1510aa462 Address:127.0.0.1:24193}]" +49:36.707 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24193 [Follower]" leader= +49:36.708 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-29fb0c63-c3d0-9218-a1aa-7cd1510aa462.dc1 127.0.0.1 +49:36.708 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-29fb0c63-c3d0-9218-a1aa-7cd1510aa462 127.0.0.1 +2022-12-04T05:49:36.708Z [INFO] agent.router: Initializing LAN area manager +49:36.708 [INFO] TestAgent.server: Adding LAN server: server="Node-29fb0c63-c3d0-9218-a1aa-7cd1510aa462 (Addr: tcp/127.0.0.1:24193) (DC: dc1)" +49:36.708 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-29fb0c63-c3d0-9218-a1aa-7cd1510aa462.dc1 area=wan +49:36.709 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24202 network=udp +49:36.709 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24202 network=tcp +49:36.709 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24189 network=tcp +49:36.709 [INFO] TestAgent: started state syncer +49:36.709 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23293 network=tcp +49:36.773 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:36.773 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24193 [Candidate]" term=2 +49:36.773 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:36.773 [DEBUG] TestAgent.server.raft: vote granted: from=29fb0c63-c3d0-9218-a1aa-7cd1510aa462 term=2 tally=1 +49:36.773 [INFO] TestAgent.server.raft: election won: tally=1 +49:36.773 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24193 [Leader]" +49:36.773 [INFO] TestAgent.server: cluster leadership acquired +49:36.774 [INFO] TestAgent.server: New leader elected: payload=Node-29fb0c63-c3d0-9218-a1aa-7cd1510aa462 +49:36.774 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24193 +49:36.774 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:36.776 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:36.776 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:36.776 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:36.776 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:36.776 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:36.776 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-29fb0c63-c3d0-9218-a1aa-7cd1510aa462 +49:36.776 [INFO] TestAgent.server: member joined, marking health alive: member=Node-29fb0c63-c3d0-9218-a1aa-7cd1510aa462 +49:36.777 [INFO] TestAgent.server: federation state anti-entropy synced +49:37.023 [DEBUG] TestAgent: Check status updated: check=test status=passing +49:37.025 [INFO] TestAgent: Synced node info +49:37.026 [INFO] TestAgent: Synced check: check=test +49:37.026 [INFO] TestAgent: Requesting shutdown +49:37.026 [INFO] TestAgent.server: shutting down server +49:37.026 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:37.026 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:37.026 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:37.026 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:37.026 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:37.026 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:37.026 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:37.026 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:37.026 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:37.026 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:37.026 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:37.026 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:37.026Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.026Z [INFO] agent.router.manager: shutting down +49:37.027 [INFO] TestAgent: consul server down +49:37.027 [INFO] TestAgent: shutdown complete +49:37.027 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24202 network=tcp +49:37.027 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24202 network=udp +49:37.027 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24189 network=tcp +49:37.527 [INFO] TestAgent: Waiting for endpoints to shut down +49:37.527 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_PassCheck (0.85s) === CONT TestOpenMetricsMimeTypeHeaders --- PASS: TestOpenMetricsMimeTypeHeaders (0.00s) -=== CONT TestAgent_reloadWatchesHTTPS -2021-10-31T23:13:23.264Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.290Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:23.292 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b8d40bd3-68c7-8b3e-d09d-a95f9dfaf0a3 Address:127.0.0.1:17118}]" -13:23.293 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b8d40bd3-68c7-8b3e-d09d-a95f9dfaf0a3.dc1 127.0.0.1 -13:23.293 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17118 [Follower]" leader= -13:23.298 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b8d40bd3-68c7-8b3e-d09d-a95f9dfaf0a3 127.0.0.1 -2021-10-31T23:13:23.298Z [INFO] agent.router: Initializing LAN area manager -13:23.298 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17113 network=udp -13:23.299 [INFO] TestAgent.server: Adding LAN server: server="Node-b8d40bd3-68c7-8b3e-d09d-a95f9dfaf0a3 (Addr: tcp/127.0.0.1:17118) (DC: dc1)" -13:23.299 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b8d40bd3-68c7-8b3e-d09d-a95f9dfaf0a3.dc1 area=wan -13:23.299 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17113 network=tcp -13:23.299 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17114 network=tcp -13:23.299 [INFO] TestAgent: started state syncer -13:23.299 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17119 network=tcp -13:23.346 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:23.346 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17118 [Candidate]" term=2 -13:23.346 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:23.346 [DEBUG] TestAgent.server.raft: vote granted: from=b8d40bd3-68c7-8b3e-d09d-a95f9dfaf0a3 term=2 tally=1 -13:23.346 [INFO] TestAgent.server.raft: election won: tally=1 -13:23.346 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17118 [Leader]" -13:23.346 [INFO] TestAgent.server: cluster leadership acquired -13:23.347 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17118 -13:23.347 [INFO] TestAgent.server: New leader elected: payload=Node-b8d40bd3-68c7-8b3e-d09d-a95f9dfaf0a3 -13:23.348 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:23.359 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:23.359 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:23.359 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:23.359 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:23.359 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:23.359 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b8d40bd3-68c7-8b3e-d09d-a95f9dfaf0a3 -13:23.359 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b8d40bd3-68c7-8b3e-d09d-a95f9dfaf0a3 -13:23.370 [INFO] TestAgent.server: federation state anti-entropy synced -13:23.498 [INFO] TestAgent: Synced node info -13:23.499 [INFO] TestAgent: Synced check: check=test -13:23.499 [INFO] TestAgent: Requesting shutdown -13:23.499 [INFO] TestAgent.server: shutting down server -13:23.499 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:23.499 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:23.499 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:23.499 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:23.499 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:23.499 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:23.499 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:23.499 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:23.499 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:23.499 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:23.500 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:23.500 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:23.500Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:23.500Z [INFO] agent.router.manager: shutting down -13:23.500 [INFO] TestAgent: consul server down -13:23.500 [INFO] TestAgent: shutdown complete -13:23.500 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17113 network=tcp -13:23.500 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17113 network=udp -13:23.500 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17114 network=tcp -13:24.001 [INFO] TestAgent: Waiting for endpoints to shut down -13:24.001 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RegisterCheck_Passing (0.85s) === CONT TestAgent_ForceLeave -2021-10-31T23:13:23.495Z [WARN] TestAgent_ReloadConfigTLSConfigFailure: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.112Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.193Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:23.213 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4803ab56-96f4-583b-d6d1-abdbe076e516 Address:127.0.0.1:17090}]" -13:23.213 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17090 [Follower]" leader= -13:23.226 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4803ab56-96f4-583b-d6d1-abdbe076e516.dc1 127.0.0.1 -13:23.227 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4803ab56-96f4-583b-d6d1-abdbe076e516 127.0.0.1 -2021-10-31T23:13:23.227Z [INFO] agent.router: Initializing LAN area manager -13:23.227 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17085 network=udp -13:23.227 [INFO] TestAgent.server: Adding LAN server: server="Node-4803ab56-96f4-583b-d6d1-abdbe076e516 (Addr: tcp/127.0.0.1:17090) (DC: dc1)" -13:23.227 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4803ab56-96f4-583b-d6d1-abdbe076e516.dc1 area=wan -13:23.228 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17085 network=tcp -13:23.229 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17086 network=tcp -13:23.229 [INFO] TestAgent: started state syncer -13:23.229 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17098 network=tcp -13:23.269 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:23.269 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17090 [Candidate]" term=2 -13:23.270 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:23.270 [DEBUG] TestAgent.server.raft: vote granted: from=4803ab56-96f4-583b-d6d1-abdbe076e516 term=2 tally=1 -13:23.270 [INFO] TestAgent.server.raft: election won: tally=1 -13:23.270 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17090 [Leader]" -13:23.270 [INFO] TestAgent.server: cluster leadership acquired -13:23.270 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17090 -13:23.271 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:23.272 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:23.272 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:23.272 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:23.272 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:23.272 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:23.272 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4803ab56-96f4-583b-d6d1-abdbe076e516 -13:23.272 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4803ab56-96f4-583b-d6d1-abdbe076e516 -13:23.273 [INFO] TestAgent.server: New leader elected: payload=Node-4803ab56-96f4-583b-d6d1-abdbe076e516 -13:23.273 [INFO] TestAgent.server: federation state anti-entropy synced -13:23.346 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:23.348 [INFO] TestAgent: Synced node info -13:23.495 [INFO] TestAgent: Requesting shutdown -13:23.495 [INFO] TestAgent.server: shutting down server -13:23.495 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:23.502 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:23.502 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:23.502Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:23.502Z [INFO] agent.router.manager: shutting down -13:23.503 [INFO] TestAgent: consul server down -13:23.503 [INFO] TestAgent: shutdown complete -13:23.503 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17085 network=tcp -13:23.503 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17085 network=udp -13:23.503 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17086 network=tcp -13:24.005 [INFO] TestAgent: Waiting for endpoints to shut down -13:24.005 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ReloadConfigTLSConfigFailure (0.92s) -=== CONT TestAgent_reloadWatches -2021-10-31T23:13:23.150Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.242Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:23.244 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:048938a8-4198-8fbb-97e0-b7155f7c7d9c Address:127.0.0.1:17104}]" -13:23.261 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-048938a8-4198-8fbb-97e0-b7155f7c7d9c.dc1 127.0.0.1 -13:23.261 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17104 [Follower]" leader= -13:23.262 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-048938a8-4198-8fbb-97e0-b7155f7c7d9c 127.0.0.1 -2021-10-31T23:13:23.262Z [INFO] agent.router: Initializing LAN area manager -13:23.262 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17099 network=udp -13:23.262 [INFO] TestAgent.server: Adding LAN server: server="Node-048938a8-4198-8fbb-97e0-b7155f7c7d9c (Addr: tcp/127.0.0.1:17104) (DC: dc1)" -13:23.262 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-048938a8-4198-8fbb-97e0-b7155f7c7d9c.dc1 area=wan -13:23.263 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17099 network=tcp -13:23.263 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17100 network=tcp -13:23.263 [INFO] TestAgent: started state syncer -13:23.263 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17112 network=tcp -13:23.334 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:23.334 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17104 [Candidate]" term=2 -13:23.334 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:23.334 [DEBUG] TestAgent.server.raft: vote granted: from=048938a8-4198-8fbb-97e0-b7155f7c7d9c term=2 tally=1 -13:23.334 [INFO] TestAgent.server.raft: election won: tally=1 -13:23.334 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17104 [Leader]" -13:23.334 [INFO] TestAgent.server: cluster leadership acquired -13:23.335 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17104 -13:23.336 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:23.341 [INFO] TestAgent.server: New leader elected: payload=Node-048938a8-4198-8fbb-97e0-b7155f7c7d9c -13:23.346 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:23.346 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:23.346 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:23.346 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:23.346 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:23.346 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-048938a8-4198-8fbb-97e0-b7155f7c7d9c -13:23.346 [INFO] TestAgent.server: member joined, marking health alive: member=Node-048938a8-4198-8fbb-97e0-b7155f7c7d9c -13:23.347 [INFO] TestAgent.server: federation state anti-entropy synced -13:23.384 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:23.385 [INFO] TestAgent: Synced node info -13:23.550 [INFO] TestAgent: Requesting shutdown -13:23.551 [INFO] TestAgent.server: shutting down server -13:23.551 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:23.551 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:23.551 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:23.551 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:23.551 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:23.551 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:23.551 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:23.551 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:23.551 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:23.551 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:23.551 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:23.552Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:23.552Z [INFO] agent.router.manager: shutting down -13:23.552 [INFO] TestAgent: consul server down -13:23.552 [INFO] TestAgent: shutdown complete -13:23.552 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17099 network=tcp -13:23.552 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17099 network=udp -13:23.552 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17100 network=tcp -13:24.054 [INFO] TestAgent: Waiting for endpoints to shut down -13:24.054 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RegisterCheck_BadStatus (0.95s) +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:36.870Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:36.880Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:36.881 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0fc90784-9ff9-7663-0591-5eefd04d58bd Address:127.0.0.1:24291}]" +49:36.881 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24291 [Follower]" leader= +49:36.882 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0fc90784-9ff9-7663-0591-5eefd04d58bd.dc1 127.0.0.1 +49:36.882 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0fc90784-9ff9-7663-0591-5eefd04d58bd 127.0.0.1 +2022-12-04T05:49:36.882Z [INFO] agent.router: Initializing LAN area manager +49:36.882 [INFO] TestAgent.server: Adding LAN server: server="Node-0fc90784-9ff9-7663-0591-5eefd04d58bd (Addr: tcp/127.0.0.1:24291) (DC: dc1)" +49:36.882 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0fc90784-9ff9-7663-0591-5eefd04d58bd.dc1 area=wan +49:36.882 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24251 network=udp +49:36.882 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24251 network=tcp +49:36.883 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24287 network=tcp +49:36.883 [INFO] TestAgent: started state syncer +49:36.883 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24278 network=tcp +49:36.932 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:36.932 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24291 [Candidate]" term=2 +49:36.933 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:36.933 [DEBUG] TestAgent.server.raft: vote granted: from=0fc90784-9ff9-7663-0591-5eefd04d58bd term=2 tally=1 +49:36.933 [INFO] TestAgent.server.raft: election won: tally=1 +49:36.933 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24291 [Leader]" +49:36.933 [INFO] TestAgent.server: cluster leadership acquired +49:36.933 [INFO] TestAgent.server: initializing acls +49:36.933 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:36.933 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:36.934 [INFO] TestAgent.server: New leader elected: payload=Node-0fc90784-9ff9-7663-0591-5eefd04d58bd +49:36.934 [INFO] TestAgent.server: initializing acls +49:36.934 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:36.934 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:36.935 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:36.935 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:36.935 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:36.936 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0fc90784-9ff9-7663-0591-5eefd04d58bd +49:36.936 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0fc90784-9ff9-7663-0591-5eefd04d58bd.dc1 +49:36.936 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:36.936 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +49:36.936 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0fc90784-9ff9-7663-0591-5eefd04d58bd +49:36.937 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0fc90784-9ff9-7663-0591-5eefd04d58bd.dc1 +49:36.937 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0fc90784-9ff9-7663-0591-5eefd04d58bd.dc1 area=wan +49:36.937 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0fc90784-9ff9-7663-0591-5eefd04d58bd.dc1 area=wan +49:36.937 [INFO] TestAgent.server: Updating LAN server: server="Node-0fc90784-9ff9-7663-0591-5eefd04d58bd (Addr: tcp/127.0.0.1:24291) (DC: dc1)" +49:36.937 [INFO] TestAgent.server: Updating LAN server: server="Node-0fc90784-9ff9-7663-0591-5eefd04d58bd (Addr: tcp/127.0.0.1:24291) (DC: dc1)" +49:36.937 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:36.939 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:36.939 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:36.939 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:36.939 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:36.939 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:36.939 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0fc90784-9ff9-7663-0591-5eefd04d58bd +49:36.939 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0fc90784-9ff9-7663-0591-5eefd04d58bd +49:36.940 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0fc90784-9ff9-7663-0591-5eefd04d58bd +49:36.940 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0fc90784-9ff9-7663-0591-5eefd04d58bd +49:36.941 [INFO] TestAgent.server: federation state anti-entropy synced +49:37.050 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0fc90784-9ff9-7663-0591-5eefd04d58bd +49:37.051 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0fc90784-9ff9-7663-0591-5eefd04d58bd +49:37.053 [INFO] TestAgent: Synced node info +49:37.053 [INFO] TestAgent: Synced service: service=foo:1234 +49:37.055 [DEBUG] TestAgent: Node info in sync +49:37.055 [DEBUG] TestAgent: Service in sync: service=foo:1234 +49:37.056 [INFO] TestAgent: Synced check: check=test +49:37.056 [DEBUG] TestAgent: Node info in sync +49:37.056 [DEBUG] TestAgent: Service in sync: service=foo:1234 +49:37.056 [DEBUG] TestAgent: Check in sync: check=test +49:37.057 [INFO] TestAgent: Synced check: check=test2 +49:37.057 [INFO] TestAgent: Requesting shutdown +49:37.057 [INFO] TestAgent.server: shutting down server +49:37.057 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:37.057 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:37.057 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:37.057 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:37.057 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:37.057 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:37.057 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:37.057 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:37.057 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:37.057 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:37.057 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:37.057 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:37.057 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:37.057 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:37.057 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:37.057 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:37.057Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.057Z [INFO] agent.router.manager: shutting down +49:37.057 [INFO] TestAgent: consul server down +49:37.057 [INFO] TestAgent: shutdown complete +49:37.057 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24251 network=tcp +49:37.057 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24251 network=udp +49:37.057 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24287 network=tcp +49:37.559 [INFO] TestAgent: Waiting for endpoints to shut down +49:37.560 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RegisterCheck_ACLDeny (0.70s) + --- PASS: TestAgent_RegisterCheck_ACLDeny/no_token_-_node_check (0.00s) + --- PASS: TestAgent_RegisterCheck_ACLDeny/svc_token_-_node_check (0.00s) + --- PASS: TestAgent_RegisterCheck_ACLDeny/node_token_-_node_check (0.00s) + --- PASS: TestAgent_RegisterCheck_ACLDeny/no_token_-_svc_check (0.00s) + --- PASS: TestAgent_RegisterCheck_ACLDeny/node_token_-_svc_check (0.00s) + --- PASS: TestAgent_RegisterCheck_ACLDeny/svc_token_-_svc_check (0.00s) === CONT TestAgent_Leave_ACLDeny -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=255)_compressed=false -[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestAgent_loadTokens/original-configuration -=== RUN TestAgent_loadTokens/updated-configuration -=== RUN TestAgent_loadTokens/persisted-tokens -=== RUN TestAgent_loadTokens/persisted-tokens-override -=== RUN TestAgent_loadTokens/partial-persisted -=== RUN TestAgent_loadTokens/persistence-error-not-json -=== RUN TestAgent_loadTokens/persistence-error-wrong-top-level -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=true -2021-10-31T23:13:23.352Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.462Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:23.464 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d6d2ce8b-f2a3-f448-4dd2-ccce53f56f7d Address:127.0.0.1:17111}]" -13:23.465 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d6d2ce8b-f2a3-f448-4dd2-ccce53f56f7d.dc1 127.0.0.1 -13:23.465 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17111 [Follower]" leader= -13:23.487 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d6d2ce8b-f2a3-f448-4dd2-ccce53f56f7d 127.0.0.1 -2021-10-31T23:13:23.487Z [INFO] agent.router: Initializing LAN area manager -13:23.491 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17106 network=udp -13:23.492 [INFO] TestAgent.server: Adding LAN server: server="Node-d6d2ce8b-f2a3-f448-4dd2-ccce53f56f7d (Addr: tcp/127.0.0.1:17111) (DC: dc1)" -13:23.492 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d6d2ce8b-f2a3-f448-4dd2-ccce53f56f7d.dc1 area=wan -13:23.493 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17106 network=tcp -13:23.510 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17107 network=tcp -13:23.510 [INFO] TestAgent: started state syncer -13:23.510 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17133 network=tcp -13:23.531 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:23.531 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17111 [Candidate]" term=2 -13:23.531 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:23.531 [DEBUG] TestAgent.server.raft: vote granted: from=d6d2ce8b-f2a3-f448-4dd2-ccce53f56f7d term=2 tally=1 -13:23.531 [INFO] TestAgent.server.raft: election won: tally=1 -13:23.531 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17111 [Leader]" -13:23.531 [INFO] TestAgent.server: cluster leadership acquired -13:23.532 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17111 -13:23.532 [INFO] TestAgent.server: New leader elected: payload=Node-d6d2ce8b-f2a3-f448-4dd2-ccce53f56f7d -13:23.533 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:23.535 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:23.535 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:23.535 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:23.535 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:23.535 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:23.535 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d6d2ce8b-f2a3-f448-4dd2-ccce53f56f7d -13:23.535 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d6d2ce8b-f2a3-f448-4dd2-ccce53f56f7d -13:23.536 [INFO] TestAgent.server: federation state anti-entropy synced -13:23.837 [INFO] TestAgent: Requesting shutdown -13:23.837 [INFO] TestAgent.server: shutting down server -13:23.837 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:23.837 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:23.837 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:23.837 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:23.837 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:23.837 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:23.837 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:23.837 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:23.837 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:23.837 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:23.837 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:23.837 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:23.838Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:23.838Z [INFO] agent.router.manager: shutting down -13:23.838 [INFO] TestAgent: consul server down -13:23.838 [INFO] TestAgent: shutdown complete -13:23.838 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17106 network=tcp -13:23.838 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17106 network=udp -13:23.838 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17107 network=tcp -13:24.340 [INFO] TestAgent: Waiting for endpoints to shut down -13:24.340 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RegisterCheckScriptsExecDisable (1.03s) -=== CONT TestAgent_GetCoordinate -=== RUN TestAgent_ForceLeave_ACLDeny/no_token -=== RUN TestAgent_ForceLeave_ACLDeny/agent_master_token -=== RUN TestAgent_ForceLeave_ACLDeny/read-only_token -=== RUN TestAgent_ForceLeave_ACLDeny/operator_write_token -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=33)_compressed=true#01 +=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=33)_compressed=false#01 +2022-12-04T05:49:37.053Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.067Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.069 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f9e238a6-0f60-5469-943b-350227f40d3e Address:127.0.0.1:23362}]" +49:37.069 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23362 [Follower]" leader= +49:37.070 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f9e238a6-0f60-5469-943b-350227f40d3e.dc1 127.0.0.1 +49:37.071 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f9e238a6-0f60-5469-943b-350227f40d3e 127.0.0.1 +2022-12-04T05:49:37.071Z [INFO] agent.router: Initializing LAN area manager +49:37.071 [INFO] TestAgent.server: Adding LAN server: server="Node-f9e238a6-0f60-5469-943b-350227f40d3e (Addr: tcp/127.0.0.1:23362) (DC: dc1)" +49:37.071 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f9e238a6-0f60-5469-943b-350227f40d3e.dc1 area=wan +49:37.071 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23315 network=udp +49:37.071 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23315 network=tcp +49:37.073 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23386 network=tcp +49:37.073 [INFO] TestAgent: started state syncer +49:37.073 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24285 network=tcp +49:37.128 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:37.128 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23362 [Candidate]" term=2 +49:37.128 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:37.128 [DEBUG] TestAgent.server.raft: vote granted: from=f9e238a6-0f60-5469-943b-350227f40d3e term=2 tally=1 +49:37.128 [INFO] TestAgent.server.raft: election won: tally=1 +49:37.128 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23362 [Leader]" +49:37.129 [INFO] TestAgent.server: cluster leadership acquired +49:37.129 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23362 +49:37.131 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:37.132 [INFO] TestAgent.server: New leader elected: payload=Node-f9e238a6-0f60-5469-943b-350227f40d3e +49:37.135 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:37.135 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:37.135 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:37.135 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:37.135 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:37.135 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f9e238a6-0f60-5469-943b-350227f40d3e +49:37.135 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f9e238a6-0f60-5469-943b-350227f40d3e +49:37.145 [INFO] TestAgent.server: federation state anti-entropy synced +49:37.249 [INFO] TestAgent: Requesting shutdown +49:37.249 [INFO] TestAgent.server: shutting down server +49:37.249 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:37.249 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:37.249 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:37.249 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:37.249 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:37.249 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:37.249 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:37.249 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:37.249 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:37.249 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:37.249 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:37.249 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:37.249Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.249Z [INFO] agent.router.manager: shutting down +49:37.249 [INFO] TestAgent: consul server down +49:37.249 [INFO] TestAgent: shutdown complete +49:37.249 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23315 network=tcp +49:37.249 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23315 network=udp +49:37.249 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23386 network=tcp +49:37.750 [INFO] TestAgent: Waiting for endpoints to shut down +49:37.750 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RegisterCheckScriptsExecRemoteDisable (0.73s) +=== CONT TestAgent_Leave +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestAgent_Leave_ACLDeny/no_token === RUN TestAgent_Leave_ACLDeny/read-only_token === RUN TestAgent_Leave_ACLDeny/agent_master_token -2021-10-31T23:13:23.528Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.528Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2021-10-31T23:13:23.611Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.611Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -13:23.622 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:11293a23-6a31-8391-f17c-6d39f0c16ed7 Address:127.0.0.1:17097}]" -13:23.623 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-11293a23-6a31-8391-f17c-6d39f0c16ed7.dc1 127.0.0.1 -13:23.624 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-11293a23-6a31-8391-f17c-6d39f0c16ed7 127.0.0.1 -2021-10-31T23:13:23.624Z [INFO] agent.router: Initializing LAN area manager -13:23.624 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17092 network=udp -13:23.624 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17097 [Follower]" leader= -13:23.625 [INFO] TestAgent.server: Adding LAN server: server="Node-11293a23-6a31-8391-f17c-6d39f0c16ed7 (Addr: tcp/127.0.0.1:17097) (DC: dc1)" -13:23.625 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-11293a23-6a31-8391-f17c-6d39f0c16ed7.dc1 area=wan -13:23.625 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17092 network=tcp -13:23.630 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17093 network=tcp -13:23.630 [INFO] TestAgent: started state syncer -13:23.630 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17056 network=tcp -13:23.669 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:23.669 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17097 [Candidate]" term=2 -13:23.669 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:23.669 [DEBUG] TestAgent.server.raft: vote granted: from=11293a23-6a31-8391-f17c-6d39f0c16ed7 term=2 tally=1 -13:23.669 [INFO] TestAgent.server.raft: election won: tally=1 -13:23.669 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17097 [Leader]" -13:23.669 [INFO] TestAgent.server: cluster leadership acquired -13:23.674 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17097 -13:23.674 [INFO] TestAgent.server: New leader elected: payload=Node-11293a23-6a31-8391-f17c-6d39f0c16ed7 -13:23.675 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:23.677 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:23.677 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:23.677 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:23.677 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:23.677 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:23.678 [INFO] TestAgent.server: federation state anti-entropy synced -13:23.682 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-11293a23-6a31-8391-f17c-6d39f0c16ed7 -13:23.682 [INFO] TestAgent.server: member joined, marking health alive: member=Node-11293a23-6a31-8391-f17c-6d39f0c16ed7 -13:23.762 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:23.762 [INFO] TestAgent: Synced node info -13:23.762 [DEBUG] TestAgent: Node info in sync -13:23.941 [DEBUG] TestAgent: Node info in sync -13:23.942 [INFO] TestAgent: Synced check: check=test -13:23.942 [DEBUG] TestAgent: Node info in sync -13:23.943 [INFO] TestAgent: Synced service: service=a -13:23.943 [DEBUG] TestAgent: Check in sync: check=test -13:23.943 [DEBUG] TestAgent: Check in sync: check=service:a -13:23.943 [DEBUG] TestAgent: Node info in sync -13:23.944 [INFO] TestAgent: Synced service: service=a -13:23.944 [DEBUG] TestAgent: Check in sync: check=test -13:23.944 [DEBUG] TestAgent: Check in sync: check=service:a -13:23.944 [DEBUG] TestAgent: Node info in sync -13:23.944 [DEBUG] TestAgent: Service in sync: service=a -13:23.944 [DEBUG] TestAgent: Check in sync: check=service:a -13:23.944 [INFO] TestAgent: Synced check: check=test -13:23.945 [DEBUG] TestAgent: Node info in sync -13:23.945 [INFO] TestAgent: Synced service: service=a -13:23.945 [DEBUG] TestAgent: Check in sync: check=test -13:23.945 [DEBUG] TestAgent: Check in sync: check=service:a -13:23.951 [DEBUG] TestAgent: Node info in sync -13:23.952 [INFO] TestAgent: Synced service: service=a -13:23.952 [DEBUG] TestAgent: Check in sync: check=test -13:23.952 [DEBUG] TestAgent: Check in sync: check=service:a -13:23.952 [DEBUG] TestAgent: Node info in sync -13:23.952 [DEBUG] TestAgent: Service in sync: service=a -13:23.953 [INFO] TestAgent: Synced check: check=test -13:23.953 [DEBUG] TestAgent: Check in sync: check=service:a -13:23.953 [DEBUG] TestAgent: Node info in sync -13:23.954 [INFO] TestAgent: Synced service: service=a -13:23.954 [DEBUG] TestAgent: Check in sync: check=service:a -13:23.954 [DEBUG] TestAgent: Check in sync: check=test -13:23.954 [DEBUG] TestAgent: Node info in sync -13:23.955 [INFO] TestAgent: Synced service: service=a -13:23.955 [DEBUG] TestAgent: Check in sync: check=test -13:23.955 [DEBUG] TestAgent: Check in sync: check=service:a -13:23.955 [INFO] TestAgent: Requesting shutdown -13:23.955 [INFO] TestAgent.server: shutting down server -13:23.955 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:23.955 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:23.955 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:23.955 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:23.955 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:23.955 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:23.955 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:23.955 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:23.955 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:23.955 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:23.955 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:23.955Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:23.955Z [INFO] agent.router.manager: shutting down -13:23.955 [INFO] TestAgent: consul server down -13:23.955 [INFO] TestAgent: shutdown complete -13:23.955 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17092 network=tcp -13:23.955 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17092 network=udp -13:23.955 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17093 network=tcp -13:24.457 [INFO] TestAgent: Waiting for endpoints to shut down -13:24.457 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:23.946Z [WARN] TestAgent_ReloadConfigIncomingRPCConfig: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.348Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.458Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:23.460 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6d6faed9-9fc5-70e2-ee5e-b416a6dd5272 Address:127.0.0.1:17132}]" -13:23.461 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6d6faed9-9fc5-70e2-ee5e-b416a6dd5272.dc1 127.0.0.1 -13:23.461 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17132 [Follower]" leader= -13:23.470 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6d6faed9-9fc5-70e2-ee5e-b416a6dd5272 127.0.0.1 -2021-10-31T23:13:23.470Z [INFO] agent.router: Initializing LAN area manager -13:23.470 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17127 network=udp -13:23.470 [INFO] TestAgent.server: Adding LAN server: server="Node-6d6faed9-9fc5-70e2-ee5e-b416a6dd5272 (Addr: tcp/127.0.0.1:17132) (DC: dc1)" -13:23.470 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6d6faed9-9fc5-70e2-ee5e-b416a6dd5272.dc1 area=wan -13:23.471 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17127 network=tcp -13:23.472 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17128 network=tcp -13:23.502 [INFO] TestAgent: started state syncer -13:23.503 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17105 network=tcp -13:23.517 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:23.517 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17132 [Candidate]" term=2 -13:23.517 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:23.517 [DEBUG] TestAgent.server.raft: vote granted: from=6d6faed9-9fc5-70e2-ee5e-b416a6dd5272 term=2 tally=1 -13:23.517 [INFO] TestAgent.server.raft: election won: tally=1 -13:23.517 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17132 [Leader]" -13:23.522 [INFO] TestAgent.server: New leader elected: payload=Node-6d6faed9-9fc5-70e2-ee5e-b416a6dd5272 -13:23.522 [INFO] TestAgent.server: cluster leadership acquired -13:23.522 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17132 -13:23.523 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:23.528 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:23.528 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:23.528 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:23.528 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:23.528 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:23.528 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6d6faed9-9fc5-70e2-ee5e-b416a6dd5272 -13:23.528 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6d6faed9-9fc5-70e2-ee5e-b416a6dd5272 -13:23.530 [INFO] TestAgent.server: federation state anti-entropy synced -13:23.552 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:23.554 [INFO] TestAgent: Synced node info -13:23.554 [DEBUG] TestAgent: Node info in sync -13:23.957 [INFO] TestAgent: Requesting shutdown -13:23.957 [INFO] TestAgent.server: shutting down server -13:23.957 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:23.957 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:23.957 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:23.957Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:23.957Z [INFO] agent.router.manager: shutting down -13:23.957 [INFO] TestAgent: consul server down -13:23.957 [INFO] TestAgent: shutdown complete -13:23.957 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17127 network=tcp -13:23.957 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17127 network=udp -13:23.957 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17128 network=tcp -13:24.457 [INFO] TestAgent: Waiting for endpoints to shut down -13:24.457 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ReloadConfigIncomingRPCConfig (1.17s) -=== CONT TestAgent_Leave ---- PASS: TestAgent_RegisterCheck_Scripts (1.00s) +2022-12-04T05:49:36.955Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:36.968Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:36.974 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:beb192c3-1b58-568d-79e4-80c160f5e95c Address:127.0.0.1:23313}]" +49:36.974 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23313 [Follower]" leader= +49:36.975 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-beb192c3-1b58-568d-79e4-80c160f5e95c.dc1 127.0.0.1 +49:36.976 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-beb192c3-1b58-568d-79e4-80c160f5e95c 127.0.0.1 +2022-12-04T05:49:36.976Z [INFO] agent.router: Initializing LAN area manager +49:36.976 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24279 network=udp +49:36.976 [INFO] TestAgent.server: Adding LAN server: server="Node-beb192c3-1b58-568d-79e4-80c160f5e95c (Addr: tcp/127.0.0.1:23313) (DC: dc1)" +49:36.977 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-beb192c3-1b58-568d-79e4-80c160f5e95c.dc1 area=wan +49:36.977 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24279 network=tcp +49:36.978 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23281 network=tcp +49:36.978 [INFO] TestAgent: started state syncer +49:36.978 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24264 network=tcp +49:37.032 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:37.032 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23313 [Candidate]" term=2 +49:37.032 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:37.032 [DEBUG] TestAgent.server.raft: vote granted: from=beb192c3-1b58-568d-79e4-80c160f5e95c term=2 tally=1 +49:37.032 [INFO] TestAgent.server.raft: election won: tally=1 +49:37.032 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23313 [Leader]" +49:37.033 [INFO] TestAgent.server: cluster leadership acquired +49:37.033 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23313 +49:37.034 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:37.040 [INFO] TestAgent.server: New leader elected: payload=Node-beb192c3-1b58-568d-79e4-80c160f5e95c +49:37.040 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:37.040 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:37.040 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:37.040 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:37.040 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:37.040 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-beb192c3-1b58-568d-79e4-80c160f5e95c +49:37.040 [INFO] TestAgent.server: member joined, marking health alive: member=Node-beb192c3-1b58-568d-79e4-80c160f5e95c +49:37.041 [INFO] TestAgent.server: federation state anti-entropy synced +49:37.341 [INFO] TestAgent: Requesting shutdown +49:37.341 [INFO] TestAgent.server: shutting down server +49:37.341 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:37.341 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:37.341 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:37.341 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:37.341 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:37.341 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:37.341 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:37.341 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:37.341 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:37.341 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:37.341 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:37.341 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:49:37.341Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.341Z [INFO] agent.router.manager: shutting down +49:37.341 [INFO] TestAgent: consul server down +49:37.341 [INFO] TestAgent: shutdown complete +49:37.341 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24279 network=tcp +49:37.341 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24279 network=udp +49:37.341 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23281 network=tcp +49:37.842 [INFO] TestAgent: Waiting for endpoints to shut down +49:37.842 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RegisterCheck_BadStatus (0.91s) +=== CONT TestAgent_JoinLANNotify +2022-12-04T05:49:37.160Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.160Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +2022-12-04T05:49:37.177Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.177Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ +49:37.179 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:27aad5a4-ae84-b690-de7a-e96d71e006e8 Address:127.0.0.1:23306}]" +49:37.180 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-27aad5a4-ae84-b690-de7a-e96d71e006e8.dc1 127.0.0.1 +49:37.181 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-27aad5a4-ae84-b690-de7a-e96d71e006e8 127.0.0.1 +2022-12-04T05:49:37.181Z [INFO] agent.router: Initializing LAN area manager +49:37.181 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23350 network=udp +49:37.181 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23306 [Follower]" leader= +49:37.183 [INFO] TestAgent.server: Adding LAN server: server="Node-27aad5a4-ae84-b690-de7a-e96d71e006e8 (Addr: tcp/127.0.0.1:23306) (DC: dc1)" +49:37.183 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-27aad5a4-ae84-b690-de7a-e96d71e006e8.dc1 area=wan +49:37.183 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23350 network=tcp +49:37.185 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23316 network=tcp +49:37.185 [INFO] TestAgent: started state syncer +49:37.185 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23384 network=tcp +49:37.237 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:37.237 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23306 [Candidate]" term=2 +49:37.237 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:37.237 [DEBUG] TestAgent.server.raft: vote granted: from=27aad5a4-ae84-b690-de7a-e96d71e006e8 term=2 tally=1 +49:37.237 [INFO] TestAgent.server.raft: election won: tally=1 +49:37.237 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23306 [Leader]" +49:37.237 [INFO] TestAgent.server: cluster leadership acquired +49:37.237 [INFO] TestAgent.server: New leader elected: payload=Node-27aad5a4-ae84-b690-de7a-e96d71e006e8 +49:37.237 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23306 +49:37.238 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:37.239 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:37.239 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:37.239 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:37.239 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:37.239 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:37.239 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-27aad5a4-ae84-b690-de7a-e96d71e006e8 +49:37.239 [INFO] TestAgent.server: member joined, marking health alive: member=Node-27aad5a4-ae84-b690-de7a-e96d71e006e8 +49:37.240 [INFO] TestAgent.server: federation state anti-entropy synced +49:37.291 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:37.292 [INFO] TestAgent: Synced node info +49:37.292 [DEBUG] TestAgent: Node info in sync +49:37.336 [DEBUG] TestAgent: Node info in sync +49:37.337 [INFO] TestAgent: Synced check: check=test +49:37.337 [DEBUG] TestAgent: Node info in sync +49:37.338 [INFO] TestAgent: Synced service: service=a +49:37.338 [DEBUG] TestAgent: Check in sync: check=test +49:37.338 [DEBUG] TestAgent: Check in sync: check=service:a +49:37.338 [DEBUG] TestAgent: Node info in sync +49:37.338 [INFO] TestAgent: Synced service: service=a +49:37.338 [DEBUG] TestAgent: Check in sync: check=test +49:37.338 [DEBUG] TestAgent: Check in sync: check=service:a +49:37.339 [DEBUG] TestAgent: Node info in sync +49:37.339 [DEBUG] TestAgent: Service in sync: service=a +49:37.339 [INFO] TestAgent: Synced check: check=test +49:37.339 [DEBUG] TestAgent: Check in sync: check=service:a +49:37.339 [DEBUG] TestAgent: Node info in sync +49:37.340 [INFO] TestAgent: Synced service: service=a +49:37.340 [DEBUG] TestAgent: Check in sync: check=test +49:37.340 [DEBUG] TestAgent: Check in sync: check=service:a +49:37.340 [DEBUG] TestAgent: Node info in sync +49:37.341 [INFO] TestAgent: Synced service: service=a +49:37.341 [DEBUG] TestAgent: Check in sync: check=service:a +49:37.341 [DEBUG] TestAgent: Check in sync: check=test +49:37.341 [DEBUG] TestAgent: Node info in sync +49:37.341 [DEBUG] TestAgent: Service in sync: service=a +49:37.341 [INFO] TestAgent: Synced check: check=test +49:37.341 [DEBUG] TestAgent: Check in sync: check=service:a +49:37.342 [DEBUG] TestAgent: Node info in sync +49:37.342 [INFO] TestAgent: Synced service: service=a +49:37.342 [DEBUG] TestAgent: Check in sync: check=test +49:37.342 [DEBUG] TestAgent: Check in sync: check=service:a +49:37.343 [DEBUG] TestAgent: Node info in sync +49:37.343 [INFO] TestAgent: Synced service: service=a +49:37.343 [DEBUG] TestAgent: Check in sync: check=test +49:37.343 [DEBUG] TestAgent: Check in sync: check=service:a +49:37.343 [INFO] TestAgent: Requesting shutdown +49:37.343 [INFO] TestAgent.server: shutting down server +49:37.343 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:37.343 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:37.343 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:37.343 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:37.343 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:37.343 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:37.343 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:37.343 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:37.343 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:37.343 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:37.344Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.344Z [INFO] agent.router.manager: shutting down +49:37.344 [INFO] TestAgent: consul server down +49:37.344 [INFO] TestAgent: shutdown complete +49:37.344 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23350 network=tcp +49:37.344 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23350 network=udp +49:37.344 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23316 network=tcp +49:37.844 [INFO] TestAgent: Waiting for endpoints to shut down +49:37.844 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RegisterCheck_Scripts (0.70s) --- PASS: TestAgent_RegisterCheck_Scripts/==_Consul_1.0.0_as_node_check (0.00s) --- PASS: TestAgent_RegisterCheck_Scripts/==_Consul_1.0.0_as_top-level_service_check (0.00s) --- PASS: TestAgent_RegisterCheck_Scripts/==_Consul_1.0.0_as_slice-based_service_check (0.00s) --- PASS: TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_(fixup)_as_node_check (0.00s) --- PASS: TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_(fixup)_as_top-level_service_check (0.00s) - --- PASS: TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_(fixup)_as_slice-based_service_check (0.01s) + --- PASS: TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_(fixup)_as_slice-based_service_check (0.00s) --- PASS: TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_as_node_check (0.00s) --- PASS: TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_as_top-level_service_check (0.00s) --- PASS: TestAgent_RegisterCheck_Scripts/>_Consul_1.0.0_as_slice-based_service_check (0.00s) -=== CONT TestAgent_JoinLANNotify -2021-10-31T23:13:24.005Z [WARN] TestAgent_ReloadConfigOutgoingRPCConfig: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.468Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.584Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:23.595 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2944dd4e-c5af-bdde-79a3-d7ceaffd2fb3 Address:127.0.0.1:17139}]" -13:23.595 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2944dd4e-c5af-bdde-79a3-d7ceaffd2fb3.dc1 127.0.0.1 -13:23.596 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2944dd4e-c5af-bdde-79a3-d7ceaffd2fb3 127.0.0.1 -2021-10-31T23:13:23.596Z [INFO] agent.router: Initializing LAN area manager -13:23.596 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17134 network=udp -13:23.596 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17139 [Follower]" leader= -13:23.596 [INFO] TestAgent.server: Adding LAN server: server="Node-2944dd4e-c5af-bdde-79a3-d7ceaffd2fb3 (Addr: tcp/127.0.0.1:17139) (DC: dc1)" -13:23.596 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2944dd4e-c5af-bdde-79a3-d7ceaffd2fb3.dc1 area=wan -13:23.596 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17134 network=tcp -13:23.597 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17135 network=tcp -13:23.597 [INFO] TestAgent: started state syncer -13:23.597 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17091 network=tcp -13:23.646 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:23.646 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17139 [Candidate]" term=2 -13:23.647 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:23.647 [DEBUG] TestAgent.server.raft: vote granted: from=2944dd4e-c5af-bdde-79a3-d7ceaffd2fb3 term=2 tally=1 -13:23.647 [INFO] TestAgent.server.raft: election won: tally=1 -13:23.647 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17139 [Leader]" -13:23.647 [INFO] TestAgent.server: cluster leadership acquired -13:23.648 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17139 -13:23.649 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:23.649 [INFO] TestAgent.server: New leader elected: payload=Node-2944dd4e-c5af-bdde-79a3-d7ceaffd2fb3 -13:23.656 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:23.656 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:23.656 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:23.656 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:23.656 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:23.656 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2944dd4e-c5af-bdde-79a3-d7ceaffd2fb3 -13:23.656 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2944dd4e-c5af-bdde-79a3-d7ceaffd2fb3 -13:23.657 [INFO] TestAgent.server: federation state anti-entropy synced -13:23.786 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:23.794 [INFO] TestAgent: Synced node info -13:24.014 [INFO] TestAgent: Requesting shutdown -13:24.014 [INFO] TestAgent.server: shutting down server -13:24.015 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:24.015 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:24.015 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:24.015Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:24.015Z [INFO] agent.router.manager: shutting down -13:24.015 [INFO] TestAgent: consul server down -13:24.015 [INFO] TestAgent: shutdown complete -13:24.015 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17134 network=tcp -13:24.015 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17134 network=udp -13:24.015 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17135 network=tcp -13:24.517 [INFO] TestAgent: Waiting for endpoints to shut down -13:24.517 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ReloadConfigOutgoingRPCConfig (1.14s) -=== CONT TestAgent_purgeCheckState -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:24.001Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:24.029Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:24.039 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3a3a9242-357e-8c08-b707-177cab9511e3 Address:127.0.0.1:16140}]" -13:24.040 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3a3a9242-357e-8c08-b707-177cab9511e3.dc1 127.0.0.1 -13:24.040 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3a3a9242-357e-8c08-b707-177cab9511e3 127.0.0.1 -2021-10-31T23:13:24.040Z [INFO] agent.router: Initializing LAN area manager -13:24.041 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16140 [Follower]" leader= -13:24.041 [INFO] TestAgent.server: Adding LAN server: server="Node-3a3a9242-357e-8c08-b707-177cab9511e3 (Addr: tcp/127.0.0.1:16140) (DC: dc1)" -13:24.041 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3a3a9242-357e-8c08-b707-177cab9511e3.dc1 area=wan -13:24.041 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16135 network=tcp -13:24.050 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16135 network=udp -13:24.050 [INFO] TestAgent: Started HTTPS server: address=127.0.0.1:16137 network=tcp -13:24.051 [INFO] TestAgent: started state syncer -13:24.051 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16141 network=tcp -13:24.093 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:24.093 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16140 [Candidate]" term=2 -13:24.093 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:24.093 [DEBUG] TestAgent.server.raft: vote granted: from=3a3a9242-357e-8c08-b707-177cab9511e3 term=2 tally=1 -13:24.093 [INFO] TestAgent.server.raft: election won: tally=1 -13:24.093 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16140 [Leader]" -13:24.093 [INFO] TestAgent.server: cluster leadership acquired -13:24.093 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16140 -13:24.093 [INFO] TestAgent.server: New leader elected: payload=Node-3a3a9242-357e-8c08-b707-177cab9511e3 -13:24.094 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:24.095 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:24.095 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:24.095 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:24.095 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:24.095 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:24.095 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3a3a9242-357e-8c08-b707-177cab9511e3 -13:24.095 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3a3a9242-357e-8c08-b707-177cab9511e3 -13:24.096 [INFO] TestAgent.server: federation state anti-entropy synced -13:24.145 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:24.146 [INFO] TestAgent: Synced node info -13:24.214 [INFO] TestAgent: Requesting shutdown -13:24.214 [INFO] TestAgent.server: shutting down server -13:24.214 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:24.214 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:24.214 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:24.214 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:24.214 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:24.214 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:24.214 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:24.214 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:24.214 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:24.214 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:24.214 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:24.214Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:24.214Z [INFO] agent.router.manager: shutting down -13:24.214 [INFO] TestAgent: consul server down -13:24.214 [INFO] TestAgent: shutdown complete -13:24.214 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16135 network=tcp -13:24.214 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16135 network=udp -13:24.214 [INFO] TestAgent: Stopping server: protocol=HTTPS address=127.0.0.1:16137 network=tcp -13:24.721 [INFO] TestAgent: Waiting for endpoints to shut down -13:24.753 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_reloadWatchesHTTPS (0.78s) -=== CONT TestAgent_loadCheckState ---- PASS: TestAgent_SecurityChecks (1.08s) === CONT TestAgent_Join_ACLDeny -2021-10-31T23:13:23.914Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.957Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:23.967 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0a795c7c-bda7-ef76-d1c1-f9d790083215 Address:127.0.0.1:16245}]" -13:23.967 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0a795c7c-bda7-ef76-d1c1-f9d790083215.dc1 127.0.0.1 -13:23.968 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0a795c7c-bda7-ef76-d1c1-f9d790083215 127.0.0.1 -2021-10-31T23:13:23.968Z [INFO] agent.router: Initializing LAN area manager -13:23.968 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16226 network=udp -13:23.968 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16245 [Follower]" leader= -13:23.968 [INFO] TestAgent.server: Adding LAN server: server="Node-0a795c7c-bda7-ef76-d1c1-f9d790083215 (Addr: tcp/127.0.0.1:16245) (DC: dc1)" -13:23.969 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0a795c7c-bda7-ef76-d1c1-f9d790083215.dc1 area=wan -13:23.969 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16226 network=tcp -13:23.969 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16241 network=tcp -13:23.978 [INFO] TestAgent: started state syncer -13:23.978 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16127 network=tcp -13:24.017 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:24.017 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16245 [Candidate]" term=2 -13:24.018 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:24.018 [DEBUG] TestAgent.server.raft: vote granted: from=0a795c7c-bda7-ef76-d1c1-f9d790083215 term=2 tally=1 -13:24.022 [INFO] TestAgent.server.raft: election won: tally=1 -13:24.022 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16245 [Leader]" -13:24.021 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:24.022 [INFO] TestAgent.server: cluster leadership acquired -13:24.023 [INFO] TestAgent.server: initializing acls -13:24.023 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:24.023 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:24.023 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:24.023 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:24.023 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0a795c7c-bda7-ef76-d1c1-f9d790083215 -13:24.024 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0a795c7c-bda7-ef76-d1c1-f9d790083215.dc1 -13:24.024 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:24.025 [INFO] TestAgent.server: New leader elected: payload=Node-0a795c7c-bda7-ef76-d1c1-f9d790083215 -13:24.026 [INFO] TestAgent.server: Updating LAN server: server="Node-0a795c7c-bda7-ef76-d1c1-f9d790083215 (Addr: tcp/127.0.0.1:16245) (DC: dc1)" -13:24.026 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0a795c7c-bda7-ef76-d1c1-f9d790083215.dc1 area=wan -13:24.026 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:24.026 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:24.026 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:24.026 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:24.026 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:24.026 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a795c7c-bda7-ef76-d1c1-f9d790083215 -13:24.026 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0a795c7c-bda7-ef76-d1c1-f9d790083215 -13:24.026 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a795c7c-bda7-ef76-d1c1-f9d790083215 -13:24.027 [INFO] TestAgent.server: federation state anti-entropy synced -13:24.061 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" -13:24.290 [WARN] TestAgent: "default" token present in both the configuration and persisted token store, using the persisted token -13:24.290 [WARN] TestAgent: "agent" token present in both the configuration and persisted token store, using the persisted token -13:24.290 [WARN] TestAgent: "agent_master" token present in both the configuration and persisted token store, using the persisted token -13:24.290 [WARN] TestAgent: "replication" token present in both the configuration and persisted token store, using the persisted token -13:24.290 [WARN] TestAgent: "default" token present in both the configuration and persisted token store, using the persisted token -13:24.290 [WARN] TestAgent: "agent" token present in both the configuration and persisted token store, using the persisted token -13:24.290 [WARN] TestAgent: "agent_master" token present in both the configuration and persisted token store, using the persisted token -13:24.290 [WARN] TestAgent: "replication" token present in both the configuration and persisted token store, using the persisted token -13:24.290 [WARN] TestAgent: "agent" token present in both the configuration and persisted token store, using the persisted token -13:24.290 [WARN] TestAgent: "agent_master" token present in both the configuration and persisted token store, using the persisted token -13:24.291 [WARN] TestAgent: unable to load persisted tokens: error="failed to decode tokens file "/tmp/consul-test/TestAgent_loadTokens-TestAgent-agent091988730/acl-tokens.json": invalid character '\x01' looking for beginning of value" -13:24.291 [WARN] TestAgent: unable to load persisted tokens: error="failed to decode tokens file "/tmp/consul-test/TestAgent_loadTokens-TestAgent-agent091988730/acl-tokens.json": json: cannot unmarshal array into Go value of type agent.persistedTokens" -13:24.291 [INFO] TestAgent: Requesting shutdown -13:24.291 [INFO] TestAgent.server: shutting down server -13:24.291 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:24.291 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:24.291 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:24.291 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:24.291 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:24.291 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:24.291 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:24.291 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:24.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:24.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:24.291 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:24.291 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:24.291 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:24.291 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:24.291 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:24.291Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:24.291Z [INFO] agent.router.manager: shutting down -13:24.291 [INFO] TestAgent: consul server down -13:24.291 [INFO] TestAgent: shutdown complete -13:24.291 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16226 network=tcp -13:24.291 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16226 network=udp -13:24.291 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16241 network=tcp -13:24.792 [INFO] TestAgent: Waiting for endpoints to shut down -13:24.792 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_loadTokens (0.92s) - --- PASS: TestAgent_loadTokens/original-configuration (0.00s) - --- PASS: TestAgent_loadTokens/updated-configuration (0.00s) - --- PASS: TestAgent_loadTokens/persisted-tokens (0.00s) - --- PASS: TestAgent_loadTokens/persisted-tokens-override (0.00s) - --- PASS: TestAgent_loadTokens/partial-persisted (0.00s) - --- PASS: TestAgent_loadTokens/persistence-error-not-json (0.00s) - --- PASS: TestAgent_loadTokens/persistence-error-wrong-top-level (0.00s) +=== RUN TestAgent_ForceLeave_ACLDeny/no_token +=== RUN TestAgent_ForceLeave_ACLDeny/agent_master_token +=== RUN TestAgent_ForceLeave_ACLDeny/read-only_token +=== RUN TestAgent_ForceLeave_ACLDeny/operator_write_token +=== RUN TestAgent_Monitor/stream_JSON_logs +2022-12-04T05:49:37.215Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.230Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.232 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6b06e110-ca62-e7e5-0103-3a3711e30948 Address:127.0.0.1:23348}]" +49:37.232 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23348 [Follower]" leader= +49:37.233 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6b06e110-ca62-e7e5-0103-3a3711e30948.dc1 127.0.0.1 +49:37.233 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6b06e110-ca62-e7e5-0103-3a3711e30948 127.0.0.1 +2022-12-04T05:49:37.233Z [INFO] agent.router: Initializing LAN area manager +49:37.233 [INFO] TestAgent.server: Adding LAN server: server="Node-6b06e110-ca62-e7e5-0103-3a3711e30948 (Addr: tcp/127.0.0.1:23348) (DC: dc1)" +49:37.233 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6b06e110-ca62-e7e5-0103-3a3711e30948.dc1 area=wan +49:37.234 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23385 network=udp +49:37.234 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23385 network=tcp +49:37.234 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23379 network=tcp +49:37.234 [INFO] TestAgent: started state syncer +49:37.234 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23279 network=tcp +49:37.281 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:37.281 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23348 [Candidate]" term=2 +49:37.281 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:37.281 [DEBUG] TestAgent.server.raft: vote granted: from=6b06e110-ca62-e7e5-0103-3a3711e30948 term=2 tally=1 +49:37.281 [INFO] TestAgent.server.raft: election won: tally=1 +49:37.281 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23348 [Leader]" +49:37.281 [INFO] TestAgent.server: cluster leadership acquired +49:37.282 [INFO] TestAgent.server: New leader elected: payload=Node-6b06e110-ca62-e7e5-0103-3a3711e30948 +49:37.282 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23348 +49:37.283 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:37.285 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:37.285 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:37.285 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:37.285 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:37.285 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:37.285 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6b06e110-ca62-e7e5-0103-3a3711e30948 +49:37.285 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6b06e110-ca62-e7e5-0103-3a3711e30948 +49:37.286 [INFO] TestAgent.server: federation state anti-entropy synced +49:37.398 [INFO] TestAgent: Synced node info +49:37.398 [INFO] TestAgent: Synced check: check=test +49:37.398 [INFO] TestAgent: Requesting shutdown +49:37.398 [INFO] TestAgent.server: shutting down server +49:37.398 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:37.398 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:37.398 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:37.399 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:37.399 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:37.399 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:37.399 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:37.399 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:37.399 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:37.399 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:37.399 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:37.399 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:37.399Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.399Z [INFO] agent.router.manager: shutting down +49:37.399 [INFO] TestAgent: consul server down +49:37.399 [INFO] TestAgent: shutdown complete +49:37.399 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23385 network=tcp +49:37.399 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23385 network=udp +49:37.399 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23379 network=tcp +49:37.900 [INFO] TestAgent: Waiting for endpoints to shut down +49:37.900 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RegisterCheck (0.70s) === CONT TestAgent_Join_WAN -2021-10-31T23:13:23.919Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:23.919Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.969Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:23.969Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:23.979 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:feadbb5e-3b8a-d1f4-21b1-b50a1770de89 Address:127.0.0.1:16133}]" -13:23.979 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-feadbb5e-3b8a-d1f4-21b1-b50a1770de89.dc1 127.0.0.1 -13:23.980 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-feadbb5e-3b8a-d1f4-21b1-b50a1770de89 127.0.0.1 -2021-10-31T23:13:23.980Z [INFO] agent.router: Initializing LAN area manager -13:23.980 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16128 network=udp -13:23.980 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16133 [Follower]" leader= -13:23.980 [INFO] TestAgent.server: Adding LAN server: server="Node-feadbb5e-3b8a-d1f4-21b1-b50a1770de89 (Addr: tcp/127.0.0.1:16133) (DC: dc1)" -13:23.980 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-feadbb5e-3b8a-d1f4-21b1-b50a1770de89.dc1 area=wan -13:23.981 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16128 network=tcp -13:23.981 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16129 network=tcp -13:23.981 [INFO] TestAgent: started state syncer -13:23.981 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16134 network=tcp -13:24.033 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:24.049 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:24.049 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16133 [Candidate]" term=2 -13:24.050 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:24.050 [DEBUG] TestAgent.server.raft: vote granted: from=feadbb5e-3b8a-d1f4-21b1-b50a1770de89 term=2 tally=1 -13:24.050 [INFO] TestAgent.server.raft: election won: tally=1 -13:24.050 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16133 [Leader]" -13:24.050 [INFO] TestAgent.server: cluster leadership acquired -13:24.050 [INFO] TestAgent.server: initializing acls -13:24.050 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:24.050 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:24.050 [INFO] TestAgent.server: New leader elected: payload=Node-feadbb5e-3b8a-d1f4-21b1-b50a1770de89 -13:24.051 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:24.051 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:24.051 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:24.051 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:24.051 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-feadbb5e-3b8a-d1f4-21b1-b50a1770de89 -13:24.051 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-feadbb5e-3b8a-d1f4-21b1-b50a1770de89.dc1 -13:24.052 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:24.053 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:24.053 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:24.053 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:24.053 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:24.053 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:24.053 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-feadbb5e-3b8a-d1f4-21b1-b50a1770de89 -13:24.053 [INFO] TestAgent.server: member joined, marking health alive: member=Node-feadbb5e-3b8a-d1f4-21b1-b50a1770de89 -13:24.053 [INFO] TestAgent.server: Updating LAN server: server="Node-feadbb5e-3b8a-d1f4-21b1-b50a1770de89 (Addr: tcp/127.0.0.1:16133) (DC: dc1)" -13:24.053 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-feadbb5e-3b8a-d1f4-21b1-b50a1770de89.dc1 area=wan -13:24.054 [INFO] TestAgent.server: federation state anti-entropy synced -13:24.062 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-feadbb5e-3b8a-d1f4-21b1-b50a1770de89 -13:24.284 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:24.284 [INFO] TestAgent: Synced node info -13:24.351 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-feadbb5e-3b8a-d1f4-21b1-b50a1770de89 -13:24.352 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-feadbb5e-3b8a-d1f4-21b1-b50a1770de89 -13:24.353 [INFO] TestAgent: Force leaving node: node=Node-feadbb5e-3b8a-d1f4-21b1-b50a1770de89 -13:24.353 [DEBUG] TestAgent.server.serf.lan: serf: Refuting an older leave intent -13:24.353 [DEBUG] TestAgent.server.serf.wan: serf: Refuting an older leave intent -13:24.353 [INFO] TestAgent: Requesting shutdown -13:24.353 [INFO] TestAgent.server: shutting down server -13:24.353 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:24.353 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:24.353 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:24.353 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:24.353 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:24.353 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:24.353 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:24.353 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:24.353 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:24.353 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:24.353 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:24.353 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:24.353 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:24.353 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:24.353 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:24.354 [INFO] TestAgent: consul server down -13:24.354 [INFO] TestAgent: shutdown complete -13:24.354 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16128 network=tcp -13:24.354 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16128 network=udp -2021-10-31T23:13:24.354Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:24.354Z [INFO] agent.router.manager: shutting down -13:24.354 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16129 network=tcp -13:24.858 [INFO] TestAgent: Waiting for endpoints to shut down -13:24.858 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ForceLeave_ACLDeny (0.99s) +2022-12-04T05:49:37.094Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.108Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.110 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:29d9395b-ac07-e7c8-672f-9831e5e327c7 Address:127.0.0.1:23334}]" +49:37.110 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23334 [Follower]" leader= +49:37.110 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-29d9395b-ac07-e7c8-672f-9831e5e327c7.dc1 127.0.0.1 +49:37.111 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-29d9395b-ac07-e7c8-672f-9831e5e327c7 127.0.0.1 +2022-12-04T05:49:37.111Z [INFO] agent.router: Initializing LAN area manager +49:37.111 [INFO] TestAgent.server: Adding LAN server: server="Node-29d9395b-ac07-e7c8-672f-9831e5e327c7 (Addr: tcp/127.0.0.1:23334) (DC: dc1)" +49:37.111 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-29d9395b-ac07-e7c8-672f-9831e5e327c7.dc1 area=wan +49:37.111 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24286 network=udp +49:37.111 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24286 network=tcp +49:37.112 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23323 network=tcp +49:37.112 [INFO] TestAgent: started state syncer +49:37.112 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23349 network=tcp +49:37.169 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:37.169 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23334 [Candidate]" term=2 +49:37.169 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:37.169 [DEBUG] TestAgent.server.raft: vote granted: from=29d9395b-ac07-e7c8-672f-9831e5e327c7 term=2 tally=1 +49:37.169 [INFO] TestAgent.server.raft: election won: tally=1 +49:37.169 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23334 [Leader]" +49:37.169 [INFO] TestAgent.server: cluster leadership acquired +49:37.169 [INFO] TestAgent.server: New leader elected: payload=Node-29d9395b-ac07-e7c8-672f-9831e5e327c7 +49:37.169 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23334 +49:37.181 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:37.183 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:37.183 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:37.183 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:37.183 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:37.183 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:37.183 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-29d9395b-ac07-e7c8-672f-9831e5e327c7 +49:37.183 [INFO] TestAgent.server: member joined, marking health alive: member=Node-29d9395b-ac07-e7c8-672f-9831e5e327c7 +49:37.184 [INFO] TestAgent.server: federation state anti-entropy synced +49:37.197 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:37.198 [INFO] TestAgent: Synced node info +49:37.198 [DEBUG] TestAgent: Node info in sync +49:37.443 [INFO] TestAgent: Requesting shutdown +49:37.444 [INFO] TestAgent.server: shutting down server +49:37.444 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:37.444 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:37.444 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:37.444 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:37.444 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:37.444 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:37.444 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:37.444 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:37.444 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:37.444 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:37.444 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:37.444Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.444Z [INFO] agent.router.manager: shutting down +49:37.444 [INFO] TestAgent: consul server down +49:37.444 [INFO] TestAgent: shutdown complete +49:37.444 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24286 network=tcp +49:37.444 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24286 network=udp +49:37.444 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23323 network=tcp +49:37.944 [INFO] TestAgent: Waiting for endpoints to shut down +49:37.944 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RegisterCheckScriptsExecDisable (0.87s) +=== CONT TestAgent_Join +2022-12-04T05:49:37.002Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.012Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.013 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5742e353-42ea-4591-9506-0726b06206e0 Address:127.0.0.1:24228}]" +49:37.013 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24228 [Follower]" leader= +49:37.014 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5742e353-42ea-4591-9506-0726b06206e0.dc1 127.0.0.1 +49:37.014 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5742e353-42ea-4591-9506-0726b06206e0 127.0.0.1 +2022-12-04T05:49:37.014Z [INFO] agent.router: Initializing LAN area manager +49:37.014 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5742e353-42ea-4591-9506-0726b06206e0.dc1 area=wan +49:37.014 [INFO] TestAgent.server: Adding LAN server: server="Node-5742e353-42ea-4591-9506-0726b06206e0 (Addr: tcp/127.0.0.1:24228) (DC: dc1)" +49:37.014 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24265 network=udp +49:37.015 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24265 network=tcp +49:37.015 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24224 network=tcp +49:37.015 [INFO] TestAgent: started state syncer +49:37.015 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23314 network=tcp +49:37.074 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:37.074 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24228 [Candidate]" term=2 +49:37.075 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:37.075 [DEBUG] TestAgent.server.raft: vote granted: from=5742e353-42ea-4591-9506-0726b06206e0 term=2 tally=1 +49:37.075 [INFO] TestAgent.server.raft: election won: tally=1 +49:37.075 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24228 [Leader]" +49:37.075 [INFO] TestAgent.server: cluster leadership acquired +49:37.076 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24228 +49:37.076 [INFO] TestAgent.server: New leader elected: payload=Node-5742e353-42ea-4591-9506-0726b06206e0 +49:37.077 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:37.079 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:37.079 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:37.079 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:37.079 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:37.079 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:37.079 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5742e353-42ea-4591-9506-0726b06206e0 +49:37.079 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5742e353-42ea-4591-9506-0726b06206e0 +49:37.081 [INFO] TestAgent.server: federation state anti-entropy synced +49:37.200 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:37.200 [INFO] TestAgent: Synced node info +49:37.200 [DEBUG] TestAgent: Node info in sync +49:37.445 [DEBUG] TestAgent: Node info in sync +49:37.446 [INFO] TestAgent: Synced check: check=test +49:37.446 [INFO] TestAgent: Requesting shutdown +49:37.446 [INFO] TestAgent.server: shutting down server +49:37.446 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:37.446 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:37.446 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:37.446 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:37.446 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:37.446 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:37.446 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:37.446 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:37.446 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:37.446 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:37.446 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:37.446Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.446Z [INFO] agent.router.manager: shutting down +49:37.446 [INFO] TestAgent: consul server down +49:37.446 [INFO] TestAgent: shutdown complete +49:37.446 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24265 network=tcp +49:37.447 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24265 network=udp +49:37.447 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24224 network=tcp +49:37.947 [INFO] TestAgent: Waiting for endpoints to shut down +49:37.947 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_RegisterCheck_Passing (0.98s) +=== CONT TestAgent_Members_ACLFilter +=== RUN TestAgent_Monitor/serf_shutdown_logging +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestAgent_Members_ACLFilter/no_token +=== RUN TestAgent_Members_ACLFilter/root_token +2022-12-04T05:49:25.057Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:25.067Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:25.068 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4a578ca4-0666-2e8a-4428-f905711392ae Address:127.0.0.1:23852}]" +49:25.068 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23852 [Follower]" leader= +49:25.069 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4a578ca4-0666-2e8a-4428-f905711392ae.dc1 127.0.0.1 +49:25.069 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4a578ca4-0666-2e8a-4428-f905711392ae 127.0.0.1 +2022-12-04T05:49:25.069Z [INFO] agent.router: Initializing LAN area manager +49:25.069 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4a578ca4-0666-2e8a-4428-f905711392ae.dc1 area=wan +49:25.069 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23798 network=udp +49:25.069 [INFO] TestAgent.server: Adding LAN server: server="Node-4a578ca4-0666-2e8a-4428-f905711392ae (Addr: tcp/127.0.0.1:23852) (DC: dc1)" +49:25.069 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23798 network=tcp +49:25.070 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23799 network=tcp +49:25.070 [INFO] TestAgent: started state syncer +49:25.070 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23853 network=tcp +49:25.105 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:25.105 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23852 [Candidate]" term=2 +49:25.105 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:25.105 [DEBUG] TestAgent.server.raft: vote granted: from=4a578ca4-0666-2e8a-4428-f905711392ae term=2 tally=1 +49:25.105 [INFO] TestAgent.server.raft: election won: tally=1 +49:25.105 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23852 [Leader]" +49:25.105 [INFO] TestAgent.server: cluster leadership acquired +49:25.105 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23852 +49:25.105 [INFO] TestAgent.server: New leader elected: payload=Node-4a578ca4-0666-2e8a-4428-f905711392ae +49:25.106 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:25.107 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:25.107 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:25.107 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:25.107 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:25.107 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:25.107 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4a578ca4-0666-2e8a-4428-f905711392ae +49:25.107 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4a578ca4-0666-2e8a-4428-f905711392ae +49:25.108 [INFO] TestAgent.server: federation state anti-entropy synced +49:25.352 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:25.352 [INFO] TestAgent: Synced node info +49:25.352 [DEBUG] TestAgent: Node info in sync +49:25.494 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=213.213µs client=127.0.0.1:42336 client_network=tcp +49:25.494 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=92.027µs client=127.0.0.1:42338 client_network=tcp +49:25.495 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=98.713µs client=127.0.0.1:49865 client_network=udp +49:25.495 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=67.66µs client=127.0.0.1:46651 client_network=udp +49:25.495 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=113.542µs client=127.0.0.1:42340 client_network=tcp +49:25.496 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=98.596µs client=127.0.0.1:42342 client_network=tcp +49:25.496 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=107.15µs client=127.0.0.1:34387 client_network=udp +49:25.496 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=74.5µs client=127.0.0.1:38881 client_network=udp +49:25.496 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=99.476µs client=127.0.0.1:42344 client_network=tcp +49:25.497 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=112.384µs client=127.0.0.1:42348 client_network=tcp +49:25.498 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=72.929µs client=127.0.0.1:55157 client_network=udp +49:25.498 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=72.173µs client=127.0.0.1:46481 client_network=udp +49:25.498 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=99.601µs client=127.0.0.1:42350 client_network=tcp +49:25.499 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=201.463µs client=127.0.0.1:42352 client_network=tcp +49:25.499 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=96.906µs client=127.0.0.1:41137 client_network=udp +49:25.500 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=88.609µs client=127.0.0.1:55392 client_network=udp +49:25.500 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=99.367µs client=127.0.0.1:42354 client_network=tcp +49:25.500 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=101.972µs client=127.0.0.1:42356 client_network=tcp +49:25.501 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=80.612µs client=127.0.0.1:45829 client_network=udp +49:25.501 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=99.159µs client=127.0.0.1:60110 client_network=udp +49:25.502 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=209.424µs client=127.0.0.1:42358 client_network=tcp +49:25.502 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=89.449µs client=127.0.0.1:42360 client_network=tcp +49:25.503 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=75.011µs client=127.0.0.1:55841 client_network=udp +49:25.503 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=54.743µs client=127.0.0.1:56407 client_network=udp +49:25.503 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=111.924µs client=127.0.0.1:42362 client_network=tcp +49:25.504 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=90.156µs client=127.0.0.1:42364 client_network=tcp +49:25.504 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=73.423µs client=127.0.0.1:45377 client_network=udp +49:25.504 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=66.278µs client=127.0.0.1:51103 client_network=udp +49:25.505 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=149.236µs client=127.0.0.1:42366 client_network=tcp +49:25.505 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=103.225µs client=127.0.0.1:42368 client_network=tcp +49:25.506 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=68.765µs client=127.0.0.1:32888 client_network=udp +49:25.506 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=60.021µs client=127.0.0.1:50413 client_network=udp +49:25.506 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=112.77µs client=127.0.0.1:42370 client_network=tcp +49:25.506 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=79.035µs client=127.0.0.1:42372 client_network=tcp +49:25.507 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=57.601µs client=127.0.0.1:34442 client_network=udp +49:25.507 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=77.714µs client=127.0.0.1:34855 client_network=udp +49:25.507 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=86.116µs client=127.0.0.1:42374 client_network=tcp +49:25.507 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=96.249µs client=127.0.0.1:42376 client_network=tcp +49:25.508 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=66.836µs client=127.0.0.1:35905 client_network=udp +49:25.508 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=67.998µs client=127.0.0.1:46344 client_network=udp +49:25.508 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=129.123µs client=127.0.0.1:42378 client_network=tcp +49:25.511 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=145.673µs client=127.0.0.1:42380 client_network=tcp +49:25.511 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=103.687µs client=127.0.0.1:55617 client_network=udp +49:25.512 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=57.529µs client=127.0.0.1:43787 client_network=udp +49:25.512 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=108.807µs client=127.0.0.1:42382 client_network=tcp +49:25.513 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=87.417µs client=127.0.0.1:42384 client_network=tcp +49:25.513 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=84.876µs client=127.0.0.1:48059 client_network=udp +49:25.513 [DEBUG] TestAgent.dns: request served from client: name=2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul. type=ANY class=IN latency=66.61µs client=127.0.0.1:42575 client_network=udp +49:27.384 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +49:27.385 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=53.291166ms client=127.0.0.1:42388 client_network=tcp +49:27.693 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +49:27.694 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:27.694 [DEBUG] TestAgent: Node info in sync +49:27.694 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=55.202031ms client=127.0.0.1:42390 client_network=tcp +49:27.955 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=68.652821ms client=127.0.0.1:44731 client_network=udp +49:28.020 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=57.565422ms client=127.0.0.1:38693 client_network=udp +49:28.101 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +49:28.104 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=72.965288ms client=127.0.0.1:42392 client_network=tcp +49:28.468 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +49:28.471 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=75.664698ms client=127.0.0.1:42394 client_network=tcp +49:28.852 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=77.017866ms client=127.0.0.1:42362 client_network=udp +49:28.931 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=72.348188ms client=127.0.0.1:54201 client_network=udp +49:28.999 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +49:29.000 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=61.842322ms client=127.0.0.1:42398 client_network=tcp +49:29.318 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +49:29.319 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=73.319853ms client=127.0.0.1:42400 client_network=tcp +49:29.666 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=84.650817ms client=127.0.0.1:52234 client_network=udp +49:29.743 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=69.964822ms client=127.0.0.1:55120 client_network=udp +49:29.820 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +49:29.822 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=74.636259ms client=127.0.0.1:42402 client_network=tcp +49:30.240 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +49:30.242 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=83.578313ms client=127.0.0.1:42404 client_network=tcp +49:30.692 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=56.550739ms client=127.0.0.1:57927 client_network=udp +49:30.756 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=55.952294ms client=127.0.0.1:48239 client_network=udp +49:30.828 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +49:30.829 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=62.399166ms client=127.0.0.1:42406 client_network=tcp +49:31.091 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +49:31.092 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=52.426928ms client=127.0.0.1:42408 client_network=tcp +49:31.441 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=86.794882ms client=127.0.0.1:37300 client_network=udp +49:31.512 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=64.726754ms client=127.0.0.1:44773 client_network=udp +49:31.599 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +49:31.600 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=83.53303ms client=127.0.0.1:42410 client_network=tcp +49:32.031 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +49:32.034 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=85.651297ms client=127.0.0.1:42412 client_network=tcp +49:32.518 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=79.127912ms client=127.0.0.1:54503 client_network=udp +49:32.601 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=73.429891ms client=127.0.0.1:43306 client_network=udp +49:32.693 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +49:32.694 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=83.07618ms client=127.0.0.1:42414 client_network=tcp +49:33.076 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +49:33.077 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=75.943695ms client=127.0.0.1:42416 client_network=tcp +49:33.386 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=72.794096ms client=127.0.0.1:57525 client_network=udp +49:33.491 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=93.635297ms client=127.0.0.1:46046 client_network=udp +49:33.574 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +49:33.577 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=79.255316ms client=127.0.0.1:42418 client_network=tcp +49:34.075 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +49:34.078 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=85.478383ms client=127.0.0.1:42420 client_network=tcp +49:34.505 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=63.979582ms client=127.0.0.1:52929 client_network=udp +49:34.567 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=53.37211ms client=127.0.0.1:57202 client_network=udp +49:34.633 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +49:34.634 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=60.991429ms client=127.0.0.1:42422 client_network=tcp +49:34.901 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +49:34.903 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=72.470005ms client=127.0.0.1:42424 client_network=tcp +49:35.158 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=51.194591ms client=127.0.0.1:42921 client_network=udp +49:35.231 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=58.712658ms client=127.0.0.1:37089 client_network=udp +49:35.291 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +49:35.293 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=56.751083ms client=127.0.0.1:42426 client_network=tcp +49:35.641 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +49:35.643 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=65.416039ms client=127.0.0.1:42428 client_network=tcp +49:36.009 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=61.56491ms client=127.0.0.1:41991 client_network=udp +49:36.079 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=62.138329ms client=127.0.0.1:49830 client_network=udp +49:36.145 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +49:36.146 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=58.58902ms client=127.0.0.1:42430 client_network=tcp +49:36.472 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 +49:36.473 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=92.424199ms client=127.0.0.1:42432 client_network=tcp +49:36.756 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=70.846543ms client=127.0.0.1:35475 client_network=udp +49:36.815 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=53.671784ms client=127.0.0.1:54584 client_network=udp +49:36.902 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +49:36.905 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=84.838901ms client=127.0.0.1:42434 client_network=tcp +49:37.269 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. 255 1}"] records=4091/5001 size=65523/80083 +49:37.271 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=62.960479ms client=127.0.0.1:42436 client_network=tcp +49:37.645 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=61.069784ms client=127.0.0.1:57288 client_network=udp +49:37.710 [DEBUG] TestAgent.dns: request served from client: name=19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul. type=ANY class=IN latency=56.743147ms client=127.0.0.1:36734 client_network=udp +49:37.716 [INFO] TestAgent: Requesting shutdown +49:37.716 [INFO] TestAgent.server: shutting down server +49:37.716 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:37.716 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:37.716 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:37.716 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:37.716 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:37.716 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:37.716 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:37.716 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:37.716 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:37.716 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:37.716 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:37.717Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.717Z [INFO] agent.router.manager: shutting down +49:37.717 [INFO] TestAgent: consul server down +49:37.717 [INFO] TestAgent: shutdown complete +49:37.717 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23798 network=tcp +49:37.717 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23798 network=udp +49:37.717 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23799 network=tcp +49:38.217 [INFO] TestAgent: Waiting for endpoints to shut down +49:38.217 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_TCP_and_UDP_Truncate (13.17s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=255)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=255)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=255)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=255)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=1)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=1)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=1)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=1)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=33)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=33)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=33)_compressed=true (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=33)_compressed=false (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=255)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=255)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=255)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=255)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=1)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=1)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=1)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=1)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=33)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._tcp_(qType:=33)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=33)_compressed=true#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_2bfd1eb8-0033-e3b6-9174-6b685ff2dba8.query.consul._udp_(qType:=33)_compressed=false#01 (0.00s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=true (0.31s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=false (0.25s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=true (0.08s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=false (0.07s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=255)_compressed=true (0.37s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=255)_compressed=false (0.38s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=255)_compressed=true (0.08s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=255)_compressed=false (0.08s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=true (0.30s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=false (0.34s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=true (0.09s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=false (0.08s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=1)_compressed=true (0.41s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=1)_compressed=false (0.48s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=1)_compressed=true (0.06s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=1)_compressed=false (0.07s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=true (0.27s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=false (0.31s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=true (0.09s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=false (0.07s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=33)_compressed=true (0.43s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=33)_compressed=false (0.49s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=33)_compressed=true (0.09s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=33)_compressed=false (0.08s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=true#01 (0.39s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=false#01 (0.31s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=true#01 (0.08s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=false#01 (0.10s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=255)_compressed=true#01 (0.49s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=255)_compressed=false#01 (0.45s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=255)_compressed=true#01 (0.07s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=255)_compressed=false#01 (0.06s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=true#01 (0.26s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=false#01 (0.28s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=true#01 (0.07s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=false#01 (0.06s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=1)_compressed=true#01 (0.34s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=1)_compressed=false#01 (0.37s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=1)_compressed=true#01 (0.07s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=1)_compressed=false#01 (0.07s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=true#01 (0.29s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=false#01 (0.30s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=true#01 (0.08s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=false#01 (0.06s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=33)_compressed=true#01 (0.39s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._tcp_(qType:=33)_compressed=false#01 (0.37s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=33)_compressed=true#01 (0.07s) + --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_19ce723f-d7ee-7421-481a-eb30b3e055d1.query.consul._udp_(qType:=33)_compressed=false#01 (0.06s) +=== CONT TestAgent_Members_WAN +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:37.514Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:37.514Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.539Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:37.539Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.541 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fe04e149-d40e-05a2-f1dc-825438f5b063 Address:127.0.0.1:24116}]" +49:37.541 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24116 [Follower]" leader= +49:37.542 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fe04e149-d40e-05a2-f1dc-825438f5b063.dc1 127.0.0.1 +49:37.542 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fe04e149-d40e-05a2-f1dc-825438f5b063 127.0.0.1 +2022-12-04T05:49:37.542Z [INFO] agent.router: Initializing LAN area manager +49:37.542 [INFO] TestAgent.server: Adding LAN server: server="Node-fe04e149-d40e-05a2-f1dc-825438f5b063 (Addr: tcp/127.0.0.1:24116) (DC: dc1)" +49:37.542 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fe04e149-d40e-05a2-f1dc-825438f5b063.dc1 area=wan +49:37.542 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24132 network=udp +49:37.543 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24132 network=tcp +49:37.543 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24112 network=tcp +49:37.543 [INFO] TestAgent: started state syncer +49:37.543 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24008 network=tcp +49:37.593 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:37.599 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:37.599 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24116 [Candidate]" term=2 +49:37.599 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:37.599 [DEBUG] TestAgent.server.raft: vote granted: from=fe04e149-d40e-05a2-f1dc-825438f5b063 term=2 tally=1 +49:37.599 [INFO] TestAgent.server.raft: election won: tally=1 +49:37.599 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24116 [Leader]" +49:37.599 [INFO] TestAgent.server: cluster leadership acquired +49:37.599 [INFO] TestAgent.server: initializing acls +49:37.599 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:37.599 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:37.600 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:37.600 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:37.600 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:37.600 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:37.600 [INFO] TestAgent.server: New leader elected: payload=Node-fe04e149-d40e-05a2-f1dc-825438f5b063 +49:37.600 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-fe04e149-d40e-05a2-f1dc-825438f5b063 +49:37.600 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-fe04e149-d40e-05a2-f1dc-825438f5b063.dc1 +49:37.601 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:37.601 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-fe04e149-d40e-05a2-f1dc-825438f5b063.dc1 area=wan +49:37.601 [INFO] TestAgent.server: Updating LAN server: server="Node-fe04e149-d40e-05a2-f1dc-825438f5b063 (Addr: tcp/127.0.0.1:24116) (DC: dc1)" +49:37.602 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:37.602 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:37.602 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:37.602 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:37.602 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:37.602 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe04e149-d40e-05a2-f1dc-825438f5b063 +49:37.602 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fe04e149-d40e-05a2-f1dc-825438f5b063 +49:37.603 [INFO] TestAgent.server: federation state anti-entropy synced +49:37.603 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe04e149-d40e-05a2-f1dc-825438f5b063 +49:37.622 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:37.622 [INFO] TestAgent: Synced node info +49:37.857 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fe04e149-d40e-05a2-f1dc-825438f5b063 +49:37.857 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fe04e149-d40e-05a2-f1dc-825438f5b063 +49:37.858 [INFO] TestAgent: Force leaving node: node=Node-fe04e149-d40e-05a2-f1dc-825438f5b063 +49:37.858 [DEBUG] TestAgent.server.serf.lan: serf: Refuting an older leave intent +49:37.858 [DEBUG] TestAgent.server.serf.wan: serf: Refuting an older leave intent +49:37.858 [INFO] TestAgent: Requesting shutdown +49:37.858 [INFO] TestAgent.server: shutting down server +49:37.858 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:37.858 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:37.858 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:37.858 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:37.858 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:37.858 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:37.858 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:37.858 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:37.859 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:37.859 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:37.859 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:37.859 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:37.859 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:37.859 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:37.859 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:37.859Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.859Z [INFO] agent.router.manager: shutting down +49:37.859 [INFO] TestAgent: consul server down +49:37.859 [INFO] TestAgent: shutdown complete +49:37.859 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24132 network=tcp +49:37.859 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24132 network=udp +49:37.859 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24112 network=tcp +49:38.359 [INFO] TestAgent: Waiting for endpoints to shut down +49:38.359 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ForceLeave_ACLDeny (0.87s) --- PASS: TestAgent_ForceLeave_ACLDeny/no_token (0.00s) --- PASS: TestAgent_ForceLeave_ACLDeny/agent_master_token (0.00s) --- PASS: TestAgent_ForceLeave_ACLDeny/read-only_token (0.00s) --- PASS: TestAgent_ForceLeave_ACLDeny/operator_write_token (0.00s) -=== CONT TestAgent_persistCheckState -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:24.034Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:24.065Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:24.075 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:803cc66e-29d2-3540-8716-373bc02d537c Address:127.0.0.1:16154}]" -13:24.076 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-803cc66e-29d2-3540-8716-373bc02d537c.dc1 127.0.0.1 -13:24.076 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-803cc66e-29d2-3540-8716-373bc02d537c 127.0.0.1 -2021-10-31T23:13:24.076Z [INFO] agent.router: Initializing LAN area manager -13:24.076 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16247 network=udp -13:24.076 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16154 [Follower]" leader= -13:24.077 [INFO] TestAgent.server: Adding LAN server: server="Node-803cc66e-29d2-3540-8716-373bc02d537c (Addr: tcp/127.0.0.1:16154) (DC: dc1)" -13:24.077 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-803cc66e-29d2-3540-8716-373bc02d537c.dc1 area=wan -13:24.077 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16247 network=tcp -13:24.077 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16150 network=tcp -13:24.077 [INFO] TestAgent: started state syncer -13:24.077 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16155 network=tcp -13:24.138 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:24.138 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16154 [Candidate]" term=2 -13:24.138 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:24.138 [DEBUG] TestAgent.server.raft: vote granted: from=803cc66e-29d2-3540-8716-373bc02d537c term=2 tally=1 -13:24.138 [INFO] TestAgent.server.raft: election won: tally=1 -13:24.138 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16154 [Leader]" -13:24.138 [INFO] TestAgent.server: cluster leadership acquired -13:24.138 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16154 -13:24.140 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:24.141 [INFO] TestAgent.server: New leader elected: payload=Node-803cc66e-29d2-3540-8716-373bc02d537c -13:24.154 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:24.154 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:24.154 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:24.154 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:24.154 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:24.154 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-803cc66e-29d2-3540-8716-373bc02d537c -13:24.154 [INFO] TestAgent.server: member joined, marking health alive: member=Node-803cc66e-29d2-3540-8716-373bc02d537c -13:24.155 [INFO] TestAgent.server: federation state anti-entropy synced -13:24.386 [INFO] TestAgent: Requesting shutdown -13:24.386 [INFO] TestAgent.server: shutting down server -13:24.386 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:24.386 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:24.386 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:24.386 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:24.386 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:24.386 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:24.386 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:24.386 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:24.386 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:24.386 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:24.386 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:24.386 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:24.386Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:24.386Z [INFO] agent.router.manager: shutting down -13:24.386 [INFO] TestAgent: consul server down -13:24.386 [INFO] TestAgent: shutdown complete -13:24.386 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16247 network=tcp -13:24.386 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16247 network=udp -13:24.386 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16150 network=tcp -13:24.891 [INFO] TestAgent: Waiting for endpoints to shut down -13:24.891 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_reloadWatches (0.89s) -=== CONT TestAgent_loadChecks_checkFails -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:24.579Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:24.605Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:24.607 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3c8cd22f-394e-e21e-2fe1-0b7e8f87ec4a Address:127.0.0.1:16168}]" -13:24.608 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3c8cd22f-394e-e21e-2fe1-0b7e8f87ec4a.dc1 127.0.0.1 -13:24.608 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3c8cd22f-394e-e21e-2fe1-0b7e8f87ec4a 127.0.0.1 -2021-10-31T23:13:24.608Z [INFO] agent.router: Initializing LAN area manager -13:24.609 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16168 [Follower]" leader= -13:24.610 [INFO] TestAgent.server: Adding LAN server: server="Node-3c8cd22f-394e-e21e-2fe1-0b7e8f87ec4a (Addr: tcp/127.0.0.1:16168) (DC: dc1)" -13:24.610 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3c8cd22f-394e-e21e-2fe1-0b7e8f87ec4a.dc1 area=wan -13:24.610 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16177 network=udp -13:24.610 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16177 network=tcp -13:24.611 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16164 network=tcp -13:24.611 [INFO] TestAgent: started state syncer -13:24.611 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16183 network=tcp -13:24.668 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:24.668 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16168 [Candidate]" term=2 -13:24.668 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:24.668 [DEBUG] TestAgent.server.raft: vote granted: from=3c8cd22f-394e-e21e-2fe1-0b7e8f87ec4a term=2 tally=1 -13:24.668 [INFO] TestAgent.server.raft: election won: tally=1 -13:24.668 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16168 [Leader]" -13:24.668 [INFO] TestAgent.server: cluster leadership acquired -13:24.669 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16168 -13:24.669 [INFO] TestAgent.server: New leader elected: payload=Node-3c8cd22f-394e-e21e-2fe1-0b7e8f87ec4a -13:24.692 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:24.696 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:24.696 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:24.696 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:24.696 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:24.696 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:24.696 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3c8cd22f-394e-e21e-2fe1-0b7e8f87ec4a -13:24.696 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3c8cd22f-394e-e21e-2fe1-0b7e8f87ec4a -13:24.706 [INFO] TestAgent.server: federation state anti-entropy synced -13:24.874 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:24.874 [INFO] TestAgent: Synced node info -13:24.990 [INFO] TestAgent: Requesting shutdown -13:24.990 [INFO] TestAgent.server: shutting down server -13:24.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:24.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:24.990 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:24.990 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:24.990 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:24.991 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:24.991 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:24.991 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:24.991 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:24.991 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:24.992 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:24.992Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:24.992Z [INFO] agent.router.manager: shutting down -13:24.992 [INFO] TestAgent: consul server down -13:24.992 [INFO] TestAgent: shutdown complete -13:24.992 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16177 network=tcp -13:24.992 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16177 network=udp -13:24.992 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16164 network=tcp -13:25.493 [INFO] TestAgent: Waiting for endpoints to shut down -13:25.493 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_purgeCheckState (0.99s) -=== CONT TestAgent_checkStateSnapshot +=== CONT TestAgent_Members === RUN TestAgent_Join_ACLDeny/no_token === RUN TestAgent_Join_ACLDeny/agent_master_token === RUN TestAgent_Join_ACLDeny/read-only_token [WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=false -2021-10-31T23:13:24.785Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:24.868Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:24.879 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:74df4086-310f-6a7e-8ba3-eba4b52e4ce9 Address:127.0.0.1:16280}]" -13:24.880 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-74df4086-310f-6a7e-8ba3-eba4b52e4ce9.dc1 127.0.0.1 -13:24.881 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-74df4086-310f-6a7e-8ba3-eba4b52e4ce9 127.0.0.1 -13:24.882 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16280 [Follower]" leader= -2021-10-31T23:13:24.889Z [INFO] agent.router: Initializing LAN area manager -13:24.890 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16191 network=udp -13:24.890 [INFO] TestAgent.server: Adding LAN server: server="Node-74df4086-310f-6a7e-8ba3-eba4b52e4ce9 (Addr: tcp/127.0.0.1:16280) (DC: dc1)" -13:24.890 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-74df4086-310f-6a7e-8ba3-eba4b52e4ce9.dc1 area=wan -13:24.891 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16191 network=tcp -13:24.892 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16276 network=tcp -13:24.892 [INFO] TestAgent: started state syncer -13:24.893 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16197 network=tcp -13:24.943 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:24.944 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16280 [Candidate]" term=2 -13:24.944 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:24.944 [DEBUG] TestAgent.server.raft: vote granted: from=74df4086-310f-6a7e-8ba3-eba4b52e4ce9 term=2 tally=1 -13:24.944 [INFO] TestAgent.server.raft: election won: tally=1 -13:24.944 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16280 [Leader]" -13:24.944 [INFO] TestAgent.server: cluster leadership acquired -13:24.972 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16280 -13:24.973 [INFO] TestAgent.server: New leader elected: payload=Node-74df4086-310f-6a7e-8ba3-eba4b52e4ce9 -13:24.973 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:24.975 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:24.975 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:24.975 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:24.975 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:24.975 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:24.975 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-74df4086-310f-6a7e-8ba3-eba4b52e4ce9 -13:24.975 [INFO] TestAgent.server: member joined, marking health alive: member=Node-74df4086-310f-6a7e-8ba3-eba4b52e4ce9 -13:24.976 [INFO] TestAgent.server: federation state anti-entropy synced -13:25.062 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:25.063 [INFO] TestAgent: Synced node info -13:25.151 [DEBUG] TestAgent: check state expired, not restoring: check=check1 -13:25.151 [INFO] TestAgent: Requesting shutdown -13:25.151 [INFO] TestAgent.server: shutting down server -13:25.151 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:25.151 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:25.151 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:25.151 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:25.151 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:25.152 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:25.152 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:25.152 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:25.152 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:25.152 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:25.152 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:25.158Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:25.158Z [INFO] agent.router.manager: shutting down -13:25.158 [INFO] TestAgent: consul server down -13:25.158 [INFO] TestAgent: shutdown complete -13:25.158 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16191 network=tcp -13:25.158 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16191 network=udp -13:25.158 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16276 network=tcp -13:25.661 [INFO] TestAgent: Waiting for endpoints to shut down -13:25.661 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_loadCheckState (0.91s) -=== CONT TestAgent_NodeMaintenanceMode -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -13:24.801 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-448ab83f-1696-b8d7-f75b-01ecf265f101 127.0.0.1 -2021-10-31T23:13:24.801Z [INFO] agent.router: Initializing LAN area manager -13:24.801 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16184 network=udp -13:24.801 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16184 network=tcp -13:24.806 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16192 network=tcp -13:24.806 [INFO] TestAgent: started state syncer -13:24.807 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16190 network=tcp -2021-10-31T23:13:24.807Z [WARN] agent.router.manager: No servers available -13:24.807 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -13:24.810 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Stream connection from=127.0.0.1:41936 -13:24.811 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-76f3d73d-1a4b-87f5-48f2-ae3aa66d23d6 127.0.0.1 -13:24.811 [INFO] TestAgent.client: adding server: server="Node-76f3d73d-1a4b-87f5-48f2-ae3aa66d23d6 (Addr: tcp/127.0.0.1:16231) (DC: dc1)" -13:24.811 [INFO] TestAgent.client: New leader elected: payload=Node-76f3d73d-1a4b-87f5-48f2-ae3aa66d23d6 -13:24.811 [INFO] TestAgent: Requesting shutdown -13:24.812 [INFO] TestAgent.client: shutting down client -13:24.812 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -13:24.812 [INFO] TestAgent: consul client down -13:24.812 [INFO] TestAgent: shutdown complete -13:24.812 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16184 network=tcp -13:24.812 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16184 network=udp -13:24.812 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16192 network=tcp -13:25.313 [INFO] TestAgent: Waiting for endpoints to shut down -13:25.359 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:24.537Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:24.596Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:24.610 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:76f3d73d-1a4b-87f5-48f2-ae3aa66d23d6 Address:127.0.0.1:16231}]" -13:24.611 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16231 [Follower]" leader= -13:24.614 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-76f3d73d-1a4b-87f5-48f2-ae3aa66d23d6.dc1 127.0.0.1 -13:24.614 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-76f3d73d-1a4b-87f5-48f2-ae3aa66d23d6 127.0.0.1 -2021-10-31T23:13:24.614Z [INFO] agent.router: Initializing LAN area manager -13:24.615 [INFO] TestAgent.server: Adding LAN server: server="Node-76f3d73d-1a4b-87f5-48f2-ae3aa66d23d6 (Addr: tcp/127.0.0.1:16231) (DC: dc1)" -13:24.615 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16240 network=udp -13:24.615 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-76f3d73d-1a4b-87f5-48f2-ae3aa66d23d6.dc1 area=wan -13:24.615 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16240 network=tcp -13:24.616 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16227 network=tcp -13:24.616 [INFO] TestAgent: started state syncer -13:24.616 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16148 network=tcp -13:24.661 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:24.662 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16231 [Candidate]" term=2 -13:24.662 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:24.662 [DEBUG] TestAgent.server.raft: vote granted: from=76f3d73d-1a4b-87f5-48f2-ae3aa66d23d6 term=2 tally=1 -13:24.662 [INFO] TestAgent.server.raft: election won: tally=1 -13:24.662 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16231 [Leader]" -13:24.662 [INFO] TestAgent.server: cluster leadership acquired -13:24.662 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16231 -13:24.664 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:24.667 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:24.667 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:24.667 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:24.667 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:24.667 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:24.667 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-76f3d73d-1a4b-87f5-48f2-ae3aa66d23d6 -13:24.667 [INFO] TestAgent.server: member joined, marking health alive: member=Node-76f3d73d-1a4b-87f5-48f2-ae3aa66d23d6 -13:24.668 [INFO] TestAgent.server: New leader elected: payload=Node-76f3d73d-1a4b-87f5-48f2-ae3aa66d23d6 -13:24.668 [INFO] TestAgent.server: federation state anti-entropy synced -13:24.806 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:16194] -13:24.810 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:16194 -13:24.811 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-448ab83f-1696-b8d7-f75b-01ecf265f101 127.0.0.1 -13:24.811 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -13:24.812 [INFO] TestAgent.server: member joined, marking health alive: member=Node-448ab83f-1696-b8d7-f75b-01ecf265f101 -13:25.004 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:25.004 [INFO] TestAgent: Synced node info -13:25.359 [INFO] TestAgent: Requesting shutdown -13:25.359 [INFO] TestAgent.server: shutting down server -13:25.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:25.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:25.359 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:25.359 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:25.359 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:25.359 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:25.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:25.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:25.359 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:25.359 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:25.359 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:25.359Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:25.359Z [INFO] agent.router.manager: shutting down -13:25.360 [INFO] TestAgent: consul server down -13:25.360 [INFO] TestAgent: shutdown complete -13:25.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16240 network=tcp -13:25.360 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16240 network=udp -13:25.360 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16227 network=tcp -13:25.861 [INFO] TestAgent: Waiting for endpoints to shut down -13:25.861 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_JoinLANNotify (1.41s) -=== CONT TestAgent_AddCheck_restoresSnapshot -2021-10-31T23:13:25.167Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:25.222Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:25.225 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8ece60a8-9ad0-7070-8291-32459876c624 Address:127.0.0.1:16259}]" -13:25.240 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8ece60a8-9ad0-7070-8291-32459876c624.dc1 127.0.0.1 -13:25.240 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16259 [Follower]" leader= -13:25.285 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:25.285 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16259 [Candidate]" term=2 -13:25.285 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=8ece60a8-9ad0-7070-8291-32459876c624 fallback=127.0.0.1:16259 error="Could not find address for server id 8ece60a8-9ad0-7070-8291-32459876c624" -13:25.285 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:25.285 [DEBUG] TestAgent.server.raft: vote granted: from=8ece60a8-9ad0-7070-8291-32459876c624 term=2 tally=1 -13:25.285 [INFO] TestAgent.server.raft: election won: tally=1 -13:25.285 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16259 [Leader]" -13:25.315 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8ece60a8-9ad0-7070-8291-32459876c624 127.0.0.1 -2021-10-31T23:13:25.315Z [INFO] agent.router: Initializing LAN area manager -13:25.359 [INFO] TestAgent.server: Adding LAN server: server="Node-8ece60a8-9ad0-7070-8291-32459876c624 (Addr: tcp/127.0.0.1:16259) (DC: dc1)" -13:25.359 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8ece60a8-9ad0-7070-8291-32459876c624.dc1 area=wan -13:25.359 [INFO] TestAgent.server: cluster leadership acquired -13:25.360 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16259 -13:25.361 [INFO] TestAgent.server: New leader elected: payload=Node-8ece60a8-9ad0-7070-8291-32459876c624 -13:25.361 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16275 network=tcp -13:25.361 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16275 network=udp -13:25.362 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16255 network=tcp -13:25.362 [INFO] TestAgent: started state syncer -13:25.362 [INFO] TestAgent: Requesting shutdown -13:25.362 [INFO] TestAgent.server: shutting down server -13:25.362 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16288 network=tcp -13:25.362 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -13:25.362 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:25.362 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:25.362Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:25.362Z [INFO] agent.router.manager: shutting down -13:25.362 [WARN] TestAgent: Syncing node info failed.: error="raft is already shutdown" -13:25.362 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="raft is already shutdown" -13:25.365 [ERROR] TestAgent.server: failed to establish leadership: error="error configuring provider: leadership lost while committing log" -13:25.367 [INFO] TestAgent: consul server down -13:25.367 [INFO] TestAgent: shutdown complete -13:25.367 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16275 network=tcp -13:25.367 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16275 network=udp -13:25.368 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16255 network=tcp -13:25.870 [INFO] TestAgent: Waiting for endpoints to shut down -13:25.870 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:24.369Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:24.383Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:24.384 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b953c0fa-33c7-9ebb-ddcf-968701285260 Address:127.0.0.1:16238}]" -13:24.385 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b953c0fa-33c7-9ebb-ddcf-968701285260.dc1 127.0.0.1 -13:24.385 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16238 [Follower]" leader= -13:24.386 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b953c0fa-33c7-9ebb-ddcf-968701285260 127.0.0.1 -2021-10-31T23:13:24.386Z [INFO] agent.router: Initializing LAN area manager -13:24.386 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16205 network=udp -13:24.386 [INFO] TestAgent.server: Adding LAN server: server="Node-b953c0fa-33c7-9ebb-ddcf-968701285260 (Addr: tcp/127.0.0.1:16238) (DC: dc1)" -13:24.386 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b953c0fa-33c7-9ebb-ddcf-968701285260.dc1 area=wan -13:24.386 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16205 network=tcp -13:24.387 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16234 network=tcp -13:24.387 [INFO] TestAgent: started state syncer -13:24.387 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16232 network=tcp -13:24.438 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:24.438 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16238 [Candidate]" term=2 -13:24.438 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:24.438 [DEBUG] TestAgent.server.raft: vote granted: from=b953c0fa-33c7-9ebb-ddcf-968701285260 term=2 tally=1 -13:24.438 [INFO] TestAgent.server.raft: election won: tally=1 -13:24.438 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16238 [Leader]" -13:24.438 [INFO] TestAgent.server: cluster leadership acquired -13:24.438 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16238 -13:24.439 [INFO] TestAgent.server: New leader elected: payload=Node-b953c0fa-33c7-9ebb-ddcf-968701285260 -13:24.439 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:24.440 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:24.440 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:24.440 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:24.440 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:24.440 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:24.440 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b953c0fa-33c7-9ebb-ddcf-968701285260 -13:24.440 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b953c0fa-33c7-9ebb-ddcf-968701285260 -13:24.441 [INFO] TestAgent.server: federation state anti-entropy synced -13:24.558 [INFO] TestAgent: Requesting shutdown -13:24.558 [INFO] TestAgent.server: shutting down server -13:24.558 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:24.558 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:24.558 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:24.558 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:24.558 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:24.558 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:24.558 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:24.558 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:24.558 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:24.558 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:24.558 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:24.558 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2021-10-31T23:13:24.558Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:24.558Z [INFO] agent.router.manager: shutting down -13:24.558 [INFO] TestAgent: consul server down -13:24.558 [INFO] TestAgent: shutdown complete -13:24.558 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16205 network=tcp -13:24.558 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16205 network=udp -13:24.559 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16234 network=tcp -13:25.062 [INFO] TestAgent: Waiting for endpoints to shut down -13:25.062 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_GetCoordinate (1.53s) -=== CONT TestAgent_Service_MaintenanceMode -2021-10-31T23:13:24.992Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:25.091Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:25.110 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:894eb92d-568e-e297-9692-383ca8ac9706 Address:127.0.0.1:16266}]" -13:25.111 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-894eb92d-568e-e297-9692-383ca8ac9706.dc1 127.0.0.1 -13:25.112 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-894eb92d-568e-e297-9692-383ca8ac9706 127.0.0.1 -2021-10-31T23:13:25.112Z [INFO] agent.router: Initializing LAN area manager -13:25.113 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16296 network=udp -13:25.113 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16266 [Follower]" leader= -13:25.115 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-894eb92d-568e-e297-9692-383ca8ac9706.dc1 area=wan -13:25.118 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16296 network=tcp -13:25.128 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16262 network=tcp -13:25.129 [INFO] TestAgent: started state syncer -13:25.129 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16267 network=tcp -13:25.149 [INFO] TestAgent.server: Adding LAN server: server="Node-894eb92d-568e-e297-9692-383ca8ac9706 (Addr: tcp/127.0.0.1:16266) (DC: dc1)" -13:25.176 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:25.177 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16266 [Candidate]" term=2 -13:25.177 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:25.177 [DEBUG] TestAgent.server.raft: vote granted: from=894eb92d-568e-e297-9692-383ca8ac9706 term=2 tally=1 -13:25.177 [INFO] TestAgent.server.raft: election won: tally=1 -13:25.177 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16266 [Leader]" -13:25.177 [INFO] TestAgent.server: cluster leadership acquired -13:25.177 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16266 -13:25.177 [INFO] TestAgent.server: New leader elected: payload=Node-894eb92d-568e-e297-9692-383ca8ac9706 -13:25.178 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:25.180 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:25.180 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:25.180 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:25.180 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:25.180 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:25.180 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-894eb92d-568e-e297-9692-383ca8ac9706 -13:25.180 [INFO] TestAgent.server: member joined, marking health alive: member=Node-894eb92d-568e-e297-9692-383ca8ac9706 -13:25.181 [INFO] TestAgent.server: federation state anti-entropy synced -13:25.354 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:25.355 [INFO] TestAgent: Synced node info -13:25.462 [INFO] TestAgent: Requesting shutdown -13:25.462 [INFO] TestAgent.server: shutting down server -13:25.462 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:25.462 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:25.462 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:25.462 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:25.462 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:25.462 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:25.462 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:25.462 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:25.462 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:25.462 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:25.462 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:25.462Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:25.462Z [INFO] agent.router.manager: shutting down -13:25.463 [INFO] TestAgent: consul server down -13:25.463 [INFO] TestAgent: shutdown complete -13:25.463 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16296 network=tcp -13:25.463 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16296 network=udp -13:25.463 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16262 network=tcp -13:25.982 [INFO] TestAgent: Waiting for endpoints to shut down -13:25.982 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_persistCheckState (1.12s) -=== CONT TestAgent_unloadChecks -2021-10-31T23:13:25.032Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:25.112Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:25.128 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d59f677e-ec6f-5d28-c119-40eae5a09e6a Address:127.0.0.1:16273}]" -13:25.129 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d59f677e-ec6f-5d28-c119-40eae5a09e6a.dc1 127.0.0.1 -13:25.129 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16273 [Follower]" leader= -13:25.150 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d59f677e-ec6f-5d28-c119-40eae5a09e6a 127.0.0.1 -2021-10-31T23:13:25.150Z [INFO] agent.router: Initializing LAN area manager -13:25.151 [INFO] TestAgent.server: Adding LAN server: server="Node-d59f677e-ec6f-5d28-c119-40eae5a09e6a (Addr: tcp/127.0.0.1:16273) (DC: dc1)" -13:25.151 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d59f677e-ec6f-5d28-c119-40eae5a09e6a.dc1 area=wan -13:25.151 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16282 network=tcp -13:25.151 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16282 network=udp -13:25.152 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16269 network=tcp -13:25.152 [INFO] TestAgent: started state syncer -13:25.157 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16260 network=tcp -13:25.186 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:25.186 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16273 [Candidate]" term=2 -13:25.186 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:25.186 [DEBUG] TestAgent.server.raft: vote granted: from=d59f677e-ec6f-5d28-c119-40eae5a09e6a term=2 tally=1 -13:25.186 [INFO] TestAgent.server.raft: election won: tally=1 -13:25.186 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16273 [Leader]" -13:25.186 [INFO] TestAgent.server: cluster leadership acquired -13:25.187 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16273 -13:25.187 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:25.189 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:25.189 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:25.189 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:25.189 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:25.189 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:25.189 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d59f677e-ec6f-5d28-c119-40eae5a09e6a -13:25.189 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d59f677e-ec6f-5d28-c119-40eae5a09e6a -13:25.213 [INFO] TestAgent.server: federation state anti-entropy synced -13:25.214 [INFO] TestAgent.server: New leader elected: payload=Node-d59f677e-ec6f-5d28-c119-40eae5a09e6a -13:25.373 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:25.374 [INFO] TestAgent: Synced node info -13:25.374 [DEBUG] TestAgent: Node info in sync -13:25.502 [WARN] TestAgent: Failed to restore check: check=service:redis error="ServiceID "nope" does not exist" -13:25.502 [DEBUG] TestAgent: restored health check from file: check=service:redis file=/tmp/consul-test/TestAgent_loadChecks_checkFails-TestAgent-agent059842939/checks/60a2ef12de014a05ecdc850d9aab46da -13:25.502 [INFO] TestAgent: Requesting shutdown -13:25.502 [INFO] TestAgent.server: shutting down server -13:25.502 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:25.502 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:25.502 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:25.502 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:25.502 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:25.502 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:25.502 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:25.502 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:25.503 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:25.503 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:25.503 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:25.503Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:25.503Z [INFO] agent.router.manager: shutting down -13:25.504 [INFO] TestAgent: consul server down -13:25.504 [INFO] TestAgent: shutdown complete -13:25.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16282 network=tcp -13:25.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16282 network=udp -13:25.504 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16269 network=tcp -13:26.006 [INFO] TestAgent: Waiting for endpoints to shut down -13:26.006 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_loadChecks_checkFails (1.11s) -=== CONT TestAgent_loadChecks_token -[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:18.010Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:18.079Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:18.087 [INFO] dc2.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2bd74813-49f3-8f71-59de-61ac3675dbb0 Address:127.0.0.1:16594}]" -13:18.088 [INFO] dc2.server.serf.wan: serf: EventMemberJoin: Node-2bd74813-49f3-8f71-59de-61ac3675dbb0.dc2 127.0.0.1 -13:18.089 [INFO] dc2.server.serf.lan: serf: EventMemberJoin: Node-2bd74813-49f3-8f71-59de-61ac3675dbb0 127.0.0.1 -2021-10-31T23:13:18.089Z [INFO] agent.router: Initializing LAN area manager -13:18.089 [INFO] dc2.server.raft: entering follower state: follower="Node at 127.0.0.1:16594 [Follower]" leader= -13:18.090 [INFO] dc2.server: Adding LAN server: server="Node-2bd74813-49f3-8f71-59de-61ac3675dbb0 (Addr: tcp/127.0.0.1:16594) (DC: dc2)" -13:18.090 [INFO] dc2.server: Handled event for server in area: event=member-join server=Node-2bd74813-49f3-8f71-59de-61ac3675dbb0.dc2 area=wan -13:18.090 [INFO] dc2: Started DNS server: address=127.0.0.1:16568 network=tcp -13:18.098 [INFO] dc2: Started DNS server: address=127.0.0.1:16568 network=udp -13:18.099 [INFO] dc2: Started HTTP server: address=127.0.0.1:16569 network=tcp -13:18.099 [INFO] dc2: started state syncer -13:18.099 [INFO] dc2: Started gRPC server: address=127.0.0.1:16595 network=tcp -13:18.156 [WARN] dc2.server.raft: heartbeat timeout reached, starting election: last-leader= -13:18.156 [INFO] dc2.server.raft: entering candidate state: node="Node at 127.0.0.1:16594 [Candidate]" term=2 -13:18.156 [DEBUG] dc2.server.raft: votes: needed=1 -13:18.156 [DEBUG] dc2.server.raft: vote granted: from=2bd74813-49f3-8f71-59de-61ac3675dbb0 term=2 tally=1 -13:18.156 [INFO] dc2.server.raft: election won: tally=1 -13:18.156 [INFO] dc2.server.raft: entering leader state: leader="Node at 127.0.0.1:16594 [Leader]" -13:18.156 [INFO] dc2.server: cluster leadership acquired -13:18.157 [DEBUG] dc2.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false -13:18.157 [WARN] dc2.server.connect: primary datacenter is configured but unreachable - deferring initialization of the secondary datacenter CA -13:18.157 [INFO] dc2.leader: started routine: routine="config entry replication" -13:18.157 [INFO] dc2.leader: started routine: routine="federation state replication" -13:18.157 [INFO] dc2.leader: started routine: routine="federation state anti-entropy" -13:18.157 [INFO] dc2.leader: started routine: routine="secondary CA roots watch" -13:18.157 [INFO] dc2.leader: started routine: routine="intention replication" -13:18.157 [INFO] dc2.leader: started routine: routine="intermediate cert renew watch" -13:18.157 [INFO] dc2.leader: started routine: routine="CA root pruning" -13:18.157 [INFO] dc2.server: New leader elected: payload=Node-2bd74813-49f3-8f71-59de-61ac3675dbb0 -13:18.157 [DEBUG] dc2.server: federation states are not enabled in the primary dc -13:18.157 [DEBUG] dc2.server: federation states are not enabled in the primary dc -13:18.157 [DEBUG] dc2.server: federation states are not enabled in the primary dc -13:18.157 [DEBUG] dc2.server: federation states are not enabled in the primary dc -13:18.157 [DEBUG] dc2.server: federation states are not enabled in the primary dc -13:18.157 [DEBUG] dc2.server: Skipping self join check for node since the cluster is too small: node=Node-2bd74813-49f3-8f71-59de-61ac3675dbb0 -13:18.157 [INFO] dc2.server: member joined, marking health alive: member=Node-2bd74813-49f3-8f71-59de-61ac3675dbb0 -13:18.157 [DEBUG] dc2.server: federation states are not enabled in the primary dc -13:18.157 [DEBUG] dc2.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 -13:18.161 [DEBUG] dc2.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 -13:18.161 [WARN] dc2.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=Intention.List -13:18.162 [ERROR] dc2.server.connect: error replicating intentions: routine="intention replication" error="No path to datacenter" -13:18.162 [WARN] dc2.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConfigEntry.ListAll -13:18.162 [WARN] dc2.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConnectCA.Roots -13:18.162 [ERROR] dc2.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: No path to datacenter" -13:18.442 [DEBUG] dc2: Skipping remote check since it is managed automatically: check=serfHealth -13:18.442 [INFO] dc2: Synced node info -13:18.482 [INFO] dc2: (WAN) joining: wan_addresses=[127.0.0.1:16502] -13:18.489 [DEBUG] dc2.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:16502 -13:18.525 [INFO] dc2.server.serf.wan: serf: EventMemberJoin: Node-df811244-1301-5244-5825-6da7cfa96d3c.dc1 127.0.0.1 -13:18.525 [INFO] dc2: (WAN) joined: number_of_nodes=1 -13:18.526 [INFO] dc2.server: Handled event for server in area: event=member-join server=Node-df811244-1301-5244-5825-6da7cfa96d3c.dc1 area=wan -13:18.882 [DEBUG] dc2.server.serf.wan: serf: messageJoinType: Node-2bd74813-49f3-8f71-59de-61ac3675dbb0.dc2 -13:19.252 [DEBUG] dc2.server: finished fetching remote objects: amount=1 -13:19.252 [DEBUG] dc2.server: diffing replication state: local_amount=0 remote_amount=1 -13:19.252 [DEBUG] dc2.server: diffed replication state: deletions=0 updates=1 -13:19.252 [DEBUG] dc2.server: performing updates: updates=1 -13:19.252 [DEBUG] dc2.server: finished updates -13:19.252 [DEBUG] dc2.replication.federation_state: replication completed through remote index: index=11 -13:19.252 [DEBUG] dc2.server.replication.config_entry: finished fetching config entries: amount=0 -13:19.252 [DEBUG] dc2.server.replication.config_entry: Config Entry replication: local=0 remote=0 -13:19.252 [DEBUG] dc2.server.replication.config_entry: Config Entry replication: deletions=0 updates=0 -13:19.252 [DEBUG] dc2.server.replication.config_entry: replication completed through remote index: index=1 -13:19.361 [DEBUG] dc2.server.serf.wan: serf: messageJoinType: Node-2bd74813-49f3-8f71-59de-61ac3675dbb0.dc2 -13:19.562 [DEBUG] dc2: Skipping remote check since it is managed automatically: check=serfHealth -13:19.562 [DEBUG] dc2: Node info in sync -13:19.562 [DEBUG] dc2: Node info in sync -13:19.862 [DEBUG] dc2.server.serf.wan: serf: messageJoinType: Node-2bd74813-49f3-8f71-59de-61ac3675dbb0.dc2 -13:20.189 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:4a:c6:ab:ef:63:c9:60:1a:51:7f:19:62:e3:e9:d9:0e:76:55:10:6e:74:24:69:28:a1:6c:b8:b9:8f:fd:89 is_primary=false -13:20.217 [INFO] dc2.server.connect: received new intermediate certificate from primary datacenter -13:20.222 [INFO] dc2.server.connect: updated root certificates from primary datacenter -13:20.234 [DEBUG] dc2: Node info in sync -13:20.236 [INFO] dc2: Synced service: service=foo -13:20.236 [DEBUG] dc2: Check in sync: check=service:foo -13:20.238 [DEBUG] dc2: Node info in sync -13:20.238 [DEBUG] dc2: Service in sync: service=foo -13:20.238 [DEBUG] dc2: Check in sync: check=service:foo -13:20.238 [DEBUG] dc2: Node info in sync -13:20.238 [DEBUG] dc2: Service in sync: service=foo -13:20.238 [DEBUG] dc2: Check in sync: check=service:foo -13:20.367 [DEBUG] dc2.server.serf.wan: serf: messageJoinType: Node-2bd74813-49f3-8f71-59de-61ac3675dbb0.dc2 -13:23.203 [DEBUG] dc2.server: finished fetching remote objects: amount=2 -13:23.203 [DEBUG] dc2.server: diffing replication state: local_amount=1 remote_amount=2 -13:23.203 [DEBUG] dc2.server: diffed replication state: deletions=0 updates=1 -13:23.203 [DEBUG] dc2.server: performing updates: updates=1 -13:23.203 [DEBUG] dc2.server: finished updates -13:23.203 [DEBUG] dc2.replication.federation_state: replication completed through remote index: index=19 -13:23.203 [INFO] dc2.server: federation state anti-entropy synced -13:25.260 [INFO] dc2.server.connect: received new intermediate certificate from primary datacenter -13:25.261 [INFO] dc2.server.connect: updated root certificates from primary datacenter -13:25.383 [INFO] dc2: Requesting shutdown -13:25.383 [INFO] dc2.server: shutting down server -13:25.383 [DEBUG] dc2.leader: stopping routine: routine="intermediate cert renew watch" -13:25.383 [DEBUG] dc2.leader: stopping routine: routine="CA root pruning" -13:25.383 [DEBUG] dc2.leader: stopping routine: routine="config entry replication" -13:25.383 [DEBUG] dc2.leader: stopping routine: routine="federation state replication" -13:25.383 [DEBUG] dc2.leader: stopping routine: routine="federation state anti-entropy" -13:25.383 [DEBUG] dc2.leader: stopping routine: routine="secondary CA roots watch" -13:25.383 [DEBUG] dc2.leader: stopping routine: routine="intention replication" -13:25.383 [WARN] dc2.server.serf.lan: serf: Shutdown without a Leave -13:25.383 [DEBUG] dc2.leader: stopped routine: routine="intermediate cert renew watch" -13:25.383 [DEBUG] dc2.leader: stopped routine: routine="CA root pruning" -13:25.383 [DEBUG] dc2.leader: stopped routine: routine="federation state anti-entropy" -13:25.383 [WARN] dc2.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:25.384Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:25.384Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:25.384Z [INFO] agent.router.manager: shutting down -13:25.384 [INFO] dc2: consul server down -13:25.384 [INFO] dc2: shutdown complete -13:25.384 [INFO] dc2: Stopping server: protocol=DNS address=127.0.0.1:16568 network=tcp -13:25.384 [INFO] dc2: Stopping server: protocol=DNS address=127.0.0.1:16568 network=udp -13:25.384 [INFO] dc2: Stopping server: protocol=HTTP address=127.0.0.1:16569 network=tcp -13:25.384 [ERROR] dc2.server.rpc: RPC failed to server in DC: server=127.0.0.1:16503 datacenter=dc1 method=ConfigEntry.ListAll error="rpc error making call: EOF" -13:25.384 [INFO] dc2.server.replication.config_entry: stopped replication -13:25.384 [DEBUG] dc2.leader: stopped routine: routine="config entry replication" -13:25.384 [ERROR] dc2.server.rpc: RPC failed to server in DC: server=127.0.0.1:16503 datacenter=dc1 method=Intention.List error="rpc error making call: EOF" -13:25.384 [ERROR] dc2.server.rpc: RPC failed to server in DC: server=127.0.0.1:16503 datacenter=dc1 method=FederationState.List error="rpc error making call: EOF" -13:25.384 [WARN] dc2.replication.federation_state: replication error (will retry if still leader): error="failed to retrieve federation states: rpc error making call: EOF" -13:25.384 [INFO] dc2.replication.federation_state: stopped replication -13:25.384 [DEBUG] dc2.leader: stopped routine: routine="federation state replication" -13:25.384 [ERROR] dc2.server.connect: error replicating intentions: routine="intention replication" error="rpc error making call: EOF" -13:25.384 [DEBUG] dc2.leader: stopped routine: routine="intention replication" -13:25.384 [ERROR] dc2.server.rpc: RPC failed to server in DC: server=127.0.0.1:16503 datacenter=dc1 method=ConnectCA.Roots error="rpc error making call: EOF" -13:25.384 [ERROR] dc2.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: rpc error making call: EOF" -13:25.384 [DEBUG] dc2.leader: stopped routine: routine="secondary CA roots watch" -13:25.885 [INFO] dc2: Waiting for endpoints to shut down -13:25.897 [INFO] dc2: Endpoints down -2021-10-31T23:13:17.815Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:17.856Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:17.859 [INFO] dc1.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:df811244-1301-5244-5825-6da7cfa96d3c Address:127.0.0.1:16503}]" -13:17.860 [INFO] dc1.server.serf.wan: serf: EventMemberJoin: Node-df811244-1301-5244-5825-6da7cfa96d3c.dc1 127.0.0.1 -13:17.860 [INFO] dc1.server.raft: entering follower state: follower="Node at 127.0.0.1:16503 [Follower]" leader= -13:17.861 [INFO] dc1.server.serf.lan: serf: EventMemberJoin: Node-df811244-1301-5244-5825-6da7cfa96d3c 127.0.0.1 -2021-10-31T23:13:17.861Z [INFO] agent.router: Initializing LAN area manager -13:17.861 [INFO] dc1: Started DNS server: address=127.0.0.1:16561 network=udp -13:17.861 [INFO] dc1.server: Handled event for server in area: event=member-join server=Node-df811244-1301-5244-5825-6da7cfa96d3c.dc1 area=wan -13:17.862 [INFO] dc1: Started DNS server: address=127.0.0.1:16561 network=tcp -13:17.862 [INFO] dc1: Started HTTP server: address=127.0.0.1:16562 network=tcp -13:17.862 [INFO] dc1: started state syncer -13:17.862 [INFO] dc1: Started gRPC server: address=127.0.0.1:16504 network=tcp -13:17.870 [INFO] dc1.server: Adding LAN server: server="Node-df811244-1301-5244-5825-6da7cfa96d3c (Addr: tcp/127.0.0.1:16503) (DC: dc1)" -13:17.911 [WARN] dc1.server.raft: heartbeat timeout reached, starting election: last-leader= -13:17.911 [INFO] dc1.server.raft: entering candidate state: node="Node at 127.0.0.1:16503 [Candidate]" term=2 -13:17.911 [DEBUG] dc1.server.raft: votes: needed=1 -13:17.911 [DEBUG] dc1.server.raft: vote granted: from=df811244-1301-5244-5825-6da7cfa96d3c term=2 tally=1 -13:17.911 [INFO] dc1.server.raft: election won: tally=1 -13:17.911 [INFO] dc1.server.raft: entering leader state: leader="Node at 127.0.0.1:16503 [Leader]" -13:17.911 [INFO] dc1.server: cluster leadership acquired -13:17.911 [DEBUG] dc1.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16503 -13:17.912 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:17.912 [INFO] dc1.server: New leader elected: payload=Node-df811244-1301-5244-5825-6da7cfa96d3c -13:17.914 [INFO] dc1.server.connect: initialized primary datacenter CA with provider: provider=consul -13:17.914 [INFO] dc1.leader: started routine: routine="federation state anti-entropy" -13:17.914 [INFO] dc1.leader: started routine: routine="federation state pruning" -13:17.914 [INFO] dc1.leader: started routine: routine="intermediate cert renew watch" -13:17.914 [INFO] dc1.leader: started routine: routine="CA root pruning" -13:17.914 [DEBUG] dc1.server: Skipping self join check for node since the cluster is too small: node=Node-df811244-1301-5244-5825-6da7cfa96d3c -13:17.914 [INFO] dc1.server: member joined, marking health alive: member=Node-df811244-1301-5244-5825-6da7cfa96d3c -13:17.915 [INFO] dc1.server: federation state anti-entropy synced -13:17.925 [DEBUG] dc1: Skipping remote check since it is managed automatically: check=serfHealth -13:17.926 [INFO] dc1: Synced node info -13:17.926 [DEBUG] dc1: Node info in sync -13:18.338 [DEBUG] dc1: Skipping remote check since it is managed automatically: check=serfHealth -13:18.338 [DEBUG] dc1: Node info in sync -13:18.516 [DEBUG] dc1.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:38854 -13:18.525 [INFO] dc1.server.serf.wan: serf: EventMemberJoin: Node-2bd74813-49f3-8f71-59de-61ac3675dbb0.dc2 127.0.0.1 -13:18.525 [INFO] dc1.server: Handled event for server in area: event=member-join server=Node-2bd74813-49f3-8f71-59de-61ac3675dbb0.dc2 area=wan -13:18.535 [DEBUG] connect.ca.consul: consul CA provider configured: id=fd:1b:ec:d9:da:64:84:9c:f5:b6:2a:12:af:49:91:02:b6:f4:48:8b:b6:40:02:6f:70:3f:84:b6:ce:06:43:dc is_primary=true -13:18.537 [INFO] dc1.server.connect: CA rotated to new root under provider: provider=consul -13:18.598 [DEBUG] dc1.server.serf.wan: serf: messageJoinType: Node-2bd74813-49f3-8f71-59de-61ac3675dbb0.dc2 -13:19.104 [DEBUG] dc1.server.serf.wan: serf: messageJoinType: Node-2bd74813-49f3-8f71-59de-61ac3675dbb0.dc2 -13:19.592 [DEBUG] dc1.server.serf.wan: serf: messageJoinType: Node-2bd74813-49f3-8f71-59de-61ac3675dbb0.dc2 -13:20.094 [DEBUG] dc1.server.serf.wan: serf: messageJoinType: Node-2bd74813-49f3-8f71-59de-61ac3675dbb0.dc2 -13:25.244 [DEBUG] connect.ca.consul: consul CA provider configured: id=48:b1:9e:0d:7c:5f:b1:8d:a7:4d:a9:c2:f7:83:5a:aa:ed:70:05:48:3e:ff:c0:59:10:ed:52:da:e8:9a:14:c0 is_primary=true -13:25.252 [INFO] dc1.server.connect: CA rotated to new root under provider: provider=consul -13:25.897 [INFO] dc1: Requesting shutdown -13:25.898 [INFO] dc1.server: shutting down server -13:25.898 [DEBUG] dc1.leader: stopping routine: routine="federation state anti-entropy" -13:25.898 [DEBUG] dc1.leader: stopping routine: routine="federation state pruning" -13:25.898 [DEBUG] dc1.leader: stopping routine: routine="intermediate cert renew watch" -13:25.898 [DEBUG] dc1.leader: stopping routine: routine="CA root pruning" -13:25.898 [WARN] dc1.server.serf.lan: serf: Shutdown without a Leave -13:25.898 [ERROR] dc1.server: error performing anti-entropy sync of federation state: error="context canceled" -13:25.898 [DEBUG] dc1.leader: stopped routine: routine="federation state anti-entropy" -13:25.898 [DEBUG] dc1.leader: stopped routine: routine="federation state pruning" -13:25.898 [DEBUG] dc1.leader: stopped routine: routine="intermediate cert renew watch" -13:25.898 [DEBUG] dc1.leader: stopped routine: routine="CA root pruning" -13:25.898 [WARN] dc1.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:25.899Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:25.899Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:25.899Z [INFO] agent.router.manager: shutting down -13:25.899 [INFO] dc1: consul server down -13:25.899 [INFO] dc1: shutdown complete -13:25.899 [INFO] dc1: Stopping server: protocol=DNS address=127.0.0.1:16561 network=tcp -13:25.899 [INFO] dc1: Stopping server: protocol=DNS address=127.0.0.1:16561 network=udp -13:25.899 [INFO] dc1: Stopping server: protocol=HTTP address=127.0.0.1:16562 network=tcp -13:26.402 [INFO] dc1: Waiting for endpoints to shut down -13:26.403 [INFO] dc1: Endpoints down ---- PASS: TestAgentConnectCALeafCert_secondaryDC_good (8.62s) -=== CONT TestAgent_DeregisterPersistedSidecarAfterRestart -2021-10-31T23:13:25.617Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:25.662Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:25.664 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ba62a393-f3a4-75df-ea88-ecd54ae65404 Address:127.0.0.1:16322}]" -13:25.664 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ba62a393-f3a4-75df-ea88-ecd54ae65404.dc1 127.0.0.1 -13:25.665 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ba62a393-f3a4-75df-ea88-ecd54ae65404 127.0.0.1 -2021-10-31T23:13:25.665Z [INFO] agent.router: Initializing LAN area manager -13:25.665 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16170 network=udp -13:25.665 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16322 [Follower]" leader= -13:25.665 [INFO] TestAgent.server: Adding LAN server: server="Node-ba62a393-f3a4-75df-ea88-ecd54ae65404 (Addr: tcp/127.0.0.1:16322) (DC: dc1)" -13:25.666 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ba62a393-f3a4-75df-ea88-ecd54ae65404.dc1 area=wan -13:25.666 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16170 network=tcp -13:25.666 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16318 network=tcp -13:25.667 [INFO] TestAgent: started state syncer -13:25.667 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16330 network=tcp -13:25.725 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:25.725 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16322 [Candidate]" term=2 -13:25.725 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:25.725 [DEBUG] TestAgent.server.raft: vote granted: from=ba62a393-f3a4-75df-ea88-ecd54ae65404 term=2 tally=1 -13:25.725 [INFO] TestAgent.server.raft: election won: tally=1 -13:25.725 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16322 [Leader]" -13:25.725 [INFO] TestAgent.server: cluster leadership acquired -13:25.725 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16322 -13:25.725 [INFO] TestAgent.server: New leader elected: payload=Node-ba62a393-f3a4-75df-ea88-ecd54ae65404 -13:25.726 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:25.727 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:25.727 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:25.727 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:25.727 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:25.727 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:25.727 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba62a393-f3a4-75df-ea88-ecd54ae65404 -13:25.727 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ba62a393-f3a4-75df-ea88-ecd54ae65404 -13:25.728 [INFO] TestAgent.server: federation state anti-entropy synced -13:25.937 [DEBUG] TestAgent: removed check: check=service:redis -13:25.937 [DEBUG] TestAgent: restored health check from file: check=service:redis file=/tmp/consul-test/TestAgent_checkStateSnapshot-TestAgent-agent270898866/checks/60a2ef12de014a05ecdc850d9aab46da -13:25.937 [INFO] TestAgent: Requesting shutdown -13:25.937 [INFO] TestAgent.server: shutting down server -13:25.937 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:25.937 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:25.937 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:25.937 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:25.937 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:25.937 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:25.937 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:25.937 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:25.937 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:25.937 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:25.938 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:25.938 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:25.938Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:25.938Z [INFO] agent.router.manager: shutting down -13:25.938 [INFO] TestAgent: consul server down -13:25.938 [INFO] TestAgent: shutdown complete -13:25.938 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16170 network=tcp -13:25.938 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16170 network=udp -13:25.938 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16318 network=tcp -13:26.438 [INFO] TestAgent: Waiting for endpoints to shut down -13:26.438 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_checkStateSnapshot (0.92s) -=== CONT TestAgent_PurgeCheckOnDuplicate -2021-10-31T23:13:25.911Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:25.943Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:25.945 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:966724cf-614a-d7b1-4d50-a788a560ad66 Address:127.0.0.1:16308}]" -13:25.945 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16308 [Follower]" leader= -13:25.946 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-966724cf-614a-d7b1-4d50-a788a560ad66.dc1 127.0.0.1 -13:25.946 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-966724cf-614a-d7b1-4d50-a788a560ad66 127.0.0.1 -2021-10-31T23:13:25.946Z [INFO] agent.router: Initializing LAN area manager -13:25.946 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16345 network=udp -13:25.946 [INFO] TestAgent.server: Adding LAN server: server="Node-966724cf-614a-d7b1-4d50-a788a560ad66 (Addr: tcp/127.0.0.1:16308) (DC: dc1)" -13:25.946 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-966724cf-614a-d7b1-4d50-a788a560ad66.dc1 area=wan -13:25.946 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16345 network=tcp -13:25.947 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16304 network=tcp -13:25.947 [INFO] TestAgent: started state syncer -13:25.947 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16337 network=tcp -13:25.989 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:25.990 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16308 [Candidate]" term=2 -13:25.990 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:25.990 [DEBUG] TestAgent.server.raft: vote granted: from=966724cf-614a-d7b1-4d50-a788a560ad66 term=2 tally=1 -13:25.990 [INFO] TestAgent.server.raft: election won: tally=1 -13:25.990 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16308 [Leader]" -13:25.990 [INFO] TestAgent.server: cluster leadership acquired -13:25.990 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16308 -13:25.991 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:25.994 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:25.994 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:25.994 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:25.994 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:25.994 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:25.994 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-966724cf-614a-d7b1-4d50-a788a560ad66 -13:25.994 [INFO] TestAgent.server: member joined, marking health alive: member=Node-966724cf-614a-d7b1-4d50-a788a560ad66 -13:25.995 [INFO] TestAgent.server: New leader elected: payload=Node-966724cf-614a-d7b1-4d50-a788a560ad66 -13:25.995 [INFO] TestAgent.server: federation state anti-entropy synced -13:26.009 [INFO] TestAgent: Requesting shutdown -13:26.009 [INFO] TestAgent.server: shutting down server -13:26.009 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:26.009 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:26.009 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:26.009 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:26.009 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:26.010 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:26.010 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:26.010 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:26.010 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:26.010 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:26.010 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:26.010 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:26.010Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:26.010Z [INFO] agent.router.manager: shutting down -13:26.010 [INFO] TestAgent: consul server down -13:26.010 [INFO] TestAgent: shutdown complete -13:26.010 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16345 network=tcp -13:26.010 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16345 network=udp -13:26.010 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16304 network=tcp -13:26.513 [INFO] TestAgent: Waiting for endpoints to shut down -13:26.514 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_restoresSnapshot (0.65s) -=== CONT TestAgent_PurgeCheck -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=true -2021-10-31T23:13:25.225Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:25.374Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:25.376 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fd3d06d0-6dbe-6462-f097-bf73cb27bef4 Address:127.0.0.1:16329}]" -13:25.377 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fd3d06d0-6dbe-6462-f097-bf73cb27bef4.dc1 127.0.0.1 -13:25.377 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16329 [Follower]" leader= -13:25.377 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fd3d06d0-6dbe-6462-f097-bf73cb27bef4 127.0.0.1 -2021-10-31T23:13:25.378Z [INFO] agent.router: Initializing LAN area manager -13:25.378 [INFO] TestAgent.server: Adding LAN server: server="Node-fd3d06d0-6dbe-6462-f097-bf73cb27bef4 (Addr: tcp/127.0.0.1:16329) (DC: dc1)" -13:25.378 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16289 network=udp -13:25.378 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16289 network=tcp -13:25.379 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16325 network=tcp -13:25.379 [INFO] TestAgent: started state syncer -13:25.379 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16316 network=tcp -13:25.381 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fd3d06d0-6dbe-6462-f097-bf73cb27bef4.dc1 area=wan -13:25.449 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:25.449 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16329 [Candidate]" term=2 -13:25.449 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:25.449 [DEBUG] TestAgent.server.raft: vote granted: from=fd3d06d0-6dbe-6462-f097-bf73cb27bef4 term=2 tally=1 -13:25.449 [INFO] TestAgent.server.raft: election won: tally=1 -13:25.449 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16329 [Leader]" -13:25.450 [INFO] TestAgent.server: cluster leadership acquired -13:25.450 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16329 -13:25.450 [INFO] TestAgent.server: New leader elected: payload=Node-fd3d06d0-6dbe-6462-f097-bf73cb27bef4 -13:25.452 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:25.475 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:25.475 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:25.475 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:25.475 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:25.475 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:25.475 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fd3d06d0-6dbe-6462-f097-bf73cb27bef4 -13:25.475 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fd3d06d0-6dbe-6462-f097-bf73cb27bef4 -13:25.477 [INFO] TestAgent.server: federation state anti-entropy synced -13:25.537 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:47318 -13:25.538 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac 127.0.0.1 -13:25.541 [INFO] TestAgent.server: Adding LAN server: server="Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac (Addr: tcp/127.0.0.1:16182) (DC: dc1)" -13:25.542 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac other=Node-fd3d06d0-6dbe-6462-f097-bf73cb27bef4 -13:25.542 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac -13:25.544 [INFO] TestAgent.server: New leader elected: payload=Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac -13:25.546 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac.dc1 127.0.0.1:16181 -13:25.547 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac.dc1 127.0.0.1 -13:25.547 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac.dc1 address=127.0.0.1:16181 -13:25.547 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac.dc1 area=wan -13:25.552 [INFO] TestAgent: Requesting shutdown -13:25.552 [INFO] TestAgent.server: shutting down server -13:25.552 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:25.552 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:25.552 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:25.552 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:25.552 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:25.553 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:25.553 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:25.553 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:25.553 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:25.553 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:25.553 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:25.553 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:25.553Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:25.553Z [INFO] agent.router.manager: shutting down -13:25.553 [INFO] TestAgent: consul server down -13:25.553 [INFO] TestAgent: shutdown complete -13:25.553 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16289 network=tcp -13:25.553 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16289 network=udp -13:25.553 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16325 network=tcp -13:26.054 [INFO] TestAgent: Waiting for endpoints to shut down -13:26.054 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:24.856Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:24.856Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:24.990Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:24.990Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:24.992 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac Address:127.0.0.1:16182}]" -13:24.992 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16182 [Follower]" leader= -13:24.993 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac.dc1 127.0.0.1 -13:24.993 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac 127.0.0.1 -2021-10-31T23:13:24.993Z [INFO] agent.router: Initializing LAN area manager -13:24.993 [INFO] TestAgent.server: Adding LAN server: server="Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac (Addr: tcp/127.0.0.1:16182) (DC: dc1)" -13:24.993 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac.dc1 area=wan -13:24.994 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16198 network=tcp -13:24.995 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16198 network=udp -13:24.995 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16178 network=tcp -13:24.995 [INFO] TestAgent: started state syncer -13:24.995 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16993 network=tcp -13:25.065 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:25.065 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16182 [Candidate]" term=2 -13:25.066 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:25.066 [DEBUG] TestAgent.server.raft: vote granted: from=b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac term=2 tally=1 -13:25.066 [INFO] TestAgent.server.raft: election won: tally=1 -13:25.066 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16182 [Leader]" -13:25.066 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=2 mode=2 found=true leader=127.0.0.1:16182 -13:25.066 [INFO] TestAgent.server: cluster leadership acquired -13:25.066 [INFO] TestAgent.server: New leader elected: payload=Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac -13:25.066 [INFO] TestAgent.server: initializing acls -13:25.067 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:25.067 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:25.067 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:25.068 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:25.068 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:25.068 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:25.068 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac -13:25.068 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac.dc1 -13:25.070 [INFO] TestAgent.server: Updating LAN server: server="Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac (Addr: tcp/127.0.0.1:16182) (DC: dc1)" -13:25.071 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac.dc1 area=wan -13:25.078 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:25.080 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:25.080 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:25.080 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:25.080 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:25.080 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:25.081 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac -13:25.081 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac -13:25.082 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac -13:25.082 [INFO] TestAgent.server: federation state anti-entropy synced -13:25.149 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac -13:25.357 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:25.358 [INFO] TestAgent: Synced node info -13:25.358 [DEBUG] TestAgent: Node info in sync -13:25.526 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac -13:25.526 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:16327] -13:25.536 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:16327 -13:25.546 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:56484 -13:25.547 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fd3d06d0-6dbe-6462-f097-bf73cb27bef4.dc1 127.0.0.1 -13:25.547 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fd3d06d0-6dbe-6462-f097-bf73cb27bef4.dc1 area=wan -13:25.550 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fd3d06d0-6dbe-6462-f097-bf73cb27bef4 127.0.0.1 -13:25.550 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -13:25.550 [DEBUG] TestAgent: systemd notify failed: error="No socket" -13:25.552 [INFO] TestAgent.server: Adding LAN server: server="Node-fd3d06d0-6dbe-6462-f097-bf73cb27bef4 (Addr: tcp/127.0.0.1:16329) (DC: dc1)" -13:25.552 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-fd3d06d0-6dbe-6462-f097-bf73cb27bef4 other=Node-b3e4cf92-d522-cb72-3f5e-0ef7f4bc59ac -13:25.552 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fd3d06d0-6dbe-6462-f097-bf73cb27bef4 -13:26.054 [INFO] TestAgent: Requesting shutdown -13:26.054 [INFO] TestAgent.server: shutting down server -13:26.054 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:26.054 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:26.054 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:26.054 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:26.054 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:26.054 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:26.054 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:26.054 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:26.054 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:26.054 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:26.054 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:26.054 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:26.054 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:26.054 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:26.054 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2021-10-31T23:13:26.054Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:26.054Z [INFO] agent.router.manager: shutting down -13:26.055 [INFO] TestAgent: consul server down -13:26.055 [INFO] TestAgent: shutdown complete -13:26.055 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16198 network=tcp -13:26.055 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16198 network=udp -13:26.055 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16178 network=tcp -13:26.507 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-fd3d06d0-6dbe-6462-f097-bf73cb27bef4 (timeout reached) -13:26.557 [INFO] TestAgent: Waiting for endpoints to shut down -13:26.557 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Join_ACLDeny (1.77s) - --- PASS: TestAgent_Join_ACLDeny/no_token (0.00s) - --- PASS: TestAgent_Join_ACLDeny/agent_master_token (0.02s) - --- PASS: TestAgent_Join_ACLDeny/read-only_token (0.00s) -=== CONT TestAgent_PersistCheck -2021-10-31T23:13:25.689Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:25.709Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:25.719 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bf9dd478-1b1c-aad2-0df6-ba4e27f83648 Address:127.0.0.1:16315}]" -13:25.720 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bf9dd478-1b1c-aad2-0df6-ba4e27f83648.dc1 127.0.0.1 -13:25.720 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bf9dd478-1b1c-aad2-0df6-ba4e27f83648 127.0.0.1 -2021-10-31T23:13:25.720Z [INFO] agent.router: Initializing LAN area manager -13:25.720 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16331 network=udp -13:25.721 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16331 network=tcp -13:25.721 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16315 [Follower]" leader= -13:25.721 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16311 network=tcp -13:25.721 [INFO] TestAgent: started state syncer -13:25.721 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bf9dd478-1b1c-aad2-0df6-ba4e27f83648.dc1 area=wan -13:25.722 [INFO] TestAgent.server: Adding LAN server: server="Node-bf9dd478-1b1c-aad2-0df6-ba4e27f83648 (Addr: tcp/127.0.0.1:16315) (DC: dc1)" -13:25.722 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16344 network=tcp -13:25.789 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:25.789 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16315 [Candidate]" term=2 -13:25.790 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:25.790 [DEBUG] TestAgent.server.raft: vote granted: from=bf9dd478-1b1c-aad2-0df6-ba4e27f83648 term=2 tally=1 -13:25.790 [INFO] TestAgent.server.raft: election won: tally=1 -13:25.790 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16315 [Leader]" -13:25.790 [INFO] TestAgent.server: cluster leadership acquired -13:25.791 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16315 -13:25.792 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:25.794 [INFO] TestAgent.server: New leader elected: payload=Node-bf9dd478-1b1c-aad2-0df6-ba4e27f83648 -13:25.802 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:25.802 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:25.802 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:25.802 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:25.802 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:25.802 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bf9dd478-1b1c-aad2-0df6-ba4e27f83648 -13:25.802 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bf9dd478-1b1c-aad2-0df6-ba4e27f83648 -13:25.803 [INFO] TestAgent.server: federation state anti-entropy synced -13:26.038 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:26.038 [INFO] TestAgent: Synced node info -13:26.074 [INFO] TestAgent: Node entered maintenance mode -13:26.074 [DEBUG] TestAgent: removed check: check=_node_maintenance -13:26.074 [INFO] TestAgent: Node left maintenance mode -13:26.074 [INFO] TestAgent: Node entered maintenance mode -13:26.074 [INFO] TestAgent: Requesting shutdown -13:26.074 [INFO] TestAgent.server: shutting down server -13:26.074 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:26.074 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:26.074 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:26.074 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:26.074 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:26.074 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:26.074 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:26.075 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:26.075 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:26.075 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:26.075 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:26.075Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:26.075Z [INFO] agent.router.manager: shutting down -13:26.075 [INFO] TestAgent: consul server down -13:26.075 [INFO] TestAgent: shutdown complete -13:26.075 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16331 network=tcp -13:26.075 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16331 network=udp -13:26.075 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16311 network=tcp -13:26.578 [INFO] TestAgent: Waiting for endpoints to shut down -13:26.578 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_NodeMaintenanceMode (0.92s) -=== CONT TestAgent_updateTTLCheck -[WARN] freeport: 10 out of 59 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:25.396Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:25.458Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:25.489 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dde5ec9f-9b1a-d1d7-2ffc-5bd7edfbfced Address:127.0.0.1:16343}]" -13:25.490 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16343 [Follower]" leader= -13:25.490 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dde5ec9f-9b1a-d1d7-2ffc-5bd7edfbfced.dc1 127.0.0.1 -13:25.491 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dde5ec9f-9b1a-d1d7-2ffc-5bd7edfbfced 127.0.0.1 -2021-10-31T23:13:25.491Z [INFO] agent.router: Initializing LAN area manager -13:25.492 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16317 network=udp -13:25.492 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dde5ec9f-9b1a-d1d7-2ffc-5bd7edfbfced.dc1 area=wan -13:25.492 [INFO] TestAgent.server: Adding LAN server: server="Node-dde5ec9f-9b1a-d1d7-2ffc-5bd7edfbfced (Addr: tcp/127.0.0.1:16343) (DC: dc1)" -13:25.492 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16317 network=tcp -13:25.493 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16339 network=tcp -13:25.493 [INFO] TestAgent: started state syncer -13:25.493 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16169 network=tcp -13:25.553 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:25.553 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16343 [Candidate]" term=2 -13:25.562 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:25.562 [DEBUG] TestAgent.server.raft: vote granted: from=dde5ec9f-9b1a-d1d7-2ffc-5bd7edfbfced term=2 tally=1 -13:25.562 [INFO] TestAgent.server.raft: election won: tally=1 -13:25.562 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16343 [Leader]" -13:25.562 [INFO] TestAgent.server: cluster leadership acquired -13:25.590 [INFO] TestAgent.server: New leader elected: payload=Node-dde5ec9f-9b1a-d1d7-2ffc-5bd7edfbfced -13:25.590 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16343 -13:25.591 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:25.592 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:25.592 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:25.592 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:25.592 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:25.592 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:25.592 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dde5ec9f-9b1a-d1d7-2ffc-5bd7edfbfced -13:25.592 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dde5ec9f-9b1a-d1d7-2ffc-5bd7edfbfced -13:25.603 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:59690 -13:25.604 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8220a2ed-7acd-6886-8286-b06726b186f2.dc1 127.0.0.1 -13:25.604 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8220a2ed-7acd-6886-8286-b06726b186f2.dc1 area=wan -13:25.611 [INFO] TestAgent: Requesting shutdown -13:25.611 [INFO] TestAgent.server: shutting down server -13:25.611 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:25.611 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:25.611 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:25.611 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:25.611 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:25.611 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:25.611 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:25.611 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:25.611 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:25.611 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:25.611Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:25.611Z [INFO] agent.router.manager: shutting down -13:25.614 [INFO] TestAgent: consul server down -13:25.614 [INFO] TestAgent: shutdown complete -13:25.614 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16317 network=tcp -13:25.614 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16317 network=udp -13:25.614 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16339 network=tcp -13:25.614 [INFO] TestAgent.server: federation state anti-entropy synced -13:25.614 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:26.118 [INFO] TestAgent: Waiting for endpoints to shut down -13:26.118 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:24.867Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:24.989Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:25.003 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8220a2ed-7acd-6886-8286-b06726b186f2 Address:127.0.0.1:16999}]" -13:25.003 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16999 [Follower]" leader= -13:25.004 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8220a2ed-7acd-6886-8286-b06726b186f2.dc1 127.0.0.1 -13:25.004 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8220a2ed-7acd-6886-8286-b06726b186f2 127.0.0.1 -2021-10-31T23:13:25.004Z [INFO] agent.router: Initializing LAN area manager -13:25.005 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16994 network=udp -13:25.005 [INFO] TestAgent.server: Adding LAN server: server="Node-8220a2ed-7acd-6886-8286-b06726b186f2 (Addr: tcp/127.0.0.1:16999) (DC: dc1)" -13:25.005 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8220a2ed-7acd-6886-8286-b06726b186f2.dc1 area=wan -13:25.005 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16994 network=tcp -13:25.005 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16995 network=tcp -13:25.006 [INFO] TestAgent: started state syncer -13:25.006 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16295 network=tcp -13:25.071 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:25.071 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16999 [Candidate]" term=2 -13:25.071 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:25.071 [DEBUG] TestAgent.server.raft: vote granted: from=8220a2ed-7acd-6886-8286-b06726b186f2 term=2 tally=1 -13:25.071 [INFO] TestAgent.server.raft: election won: tally=1 -13:25.072 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16999 [Leader]" -13:25.072 [INFO] TestAgent.server: cluster leadership acquired -13:25.072 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16999 -13:25.073 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:25.073 [INFO] TestAgent.server: New leader elected: payload=Node-8220a2ed-7acd-6886-8286-b06726b186f2 -13:25.075 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:25.075 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:25.075 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:25.075 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:25.075 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:25.075 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8220a2ed-7acd-6886-8286-b06726b186f2 -13:25.075 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8220a2ed-7acd-6886-8286-b06726b186f2 -13:25.077 [INFO] TestAgent.server: federation state anti-entropy synced -13:25.232 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:25.233 [INFO] TestAgent: Synced node info -13:25.602 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:16342] -13:25.602 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:16342 -13:25.610 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dde5ec9f-9b1a-d1d7-2ffc-5bd7edfbfced.dc1 127.0.0.1 -13:25.611 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -13:25.611 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dde5ec9f-9b1a-d1d7-2ffc-5bd7edfbfced.dc1 area=wan -13:26.118 [INFO] TestAgent: Requesting shutdown -13:26.118 [INFO] TestAgent.server: shutting down server -13:26.118 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:26.118 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:26.118 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:26.118 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:26.118 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:26.118 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:26.118 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:26.118 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:26.118 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:26.118 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:26.118 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:26.119Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:26.119Z [INFO] agent.router.manager: shutting down -13:26.119 [INFO] TestAgent: consul server down -13:26.119 [INFO] TestAgent: shutdown complete -13:26.119 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16994 network=tcp -13:26.119 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16994 network=udp -13:26.119 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16995 network=tcp -13:26.622 [INFO] TestAgent: Waiting for endpoints to shut down -13:26.622 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Join_WAN (1.83s) -=== CONT TestAgent_HTTPCheck_TLSSkipVerify -2021-10-31T23:13:26.071Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:26.132Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:26.134 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:34820a39-52f1-8ad7-eb72-883944a0e1ff Address:127.0.0.1:16350}]" -13:26.135 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-34820a39-52f1-8ad7-eb72-883944a0e1ff.dc1 127.0.0.1 -13:26.135 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-34820a39-52f1-8ad7-eb72-883944a0e1ff 127.0.0.1 -2021-10-31T23:13:26.135Z [INFO] agent.router: Initializing LAN area manager -13:26.136 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16303 network=udp -13:26.136 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16350 [Follower]" leader= -13:26.136 [INFO] TestAgent.server: Adding LAN server: server="Node-34820a39-52f1-8ad7-eb72-883944a0e1ff (Addr: tcp/127.0.0.1:16350) (DC: dc1)" -13:26.136 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-34820a39-52f1-8ad7-eb72-883944a0e1ff.dc1 area=wan -13:26.136 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16303 network=tcp -13:26.137 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16346 network=tcp -13:26.137 [INFO] TestAgent: started state syncer -13:26.137 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16309 network=tcp -13:26.200 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:26.200 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16350 [Candidate]" term=2 -13:26.200 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:26.200 [DEBUG] TestAgent.server.raft: vote granted: from=34820a39-52f1-8ad7-eb72-883944a0e1ff term=2 tally=1 -13:26.200 [INFO] TestAgent.server.raft: election won: tally=1 -13:26.200 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16350 [Leader]" -13:26.201 [INFO] TestAgent.server: cluster leadership acquired -13:26.201 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16350 -13:26.201 [INFO] TestAgent.server: New leader elected: payload=Node-34820a39-52f1-8ad7-eb72-883944a0e1ff -13:26.210 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:26.213 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:26.213 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:26.213 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:26.213 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:26.213 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:26.213 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-34820a39-52f1-8ad7-eb72-883944a0e1ff -13:26.213 [INFO] TestAgent.server: member joined, marking health alive: member=Node-34820a39-52f1-8ad7-eb72-883944a0e1ff -13:26.226 [INFO] TestAgent: Synced node info -13:26.227 [INFO] TestAgent.server: federation state anti-entropy synced -13:26.227 [INFO] TestAgent: Synced check: check=rabbitmq -13:26.238 [INFO] TestAgent: Requesting shutdown -13:26.238 [INFO] TestAgent.server: shutting down server -13:26.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:26.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:26.238 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:26.238 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:26.238 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:26.238 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:26.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:26.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:26.238 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:26.238 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:26.238 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:26.238Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:26.239Z [INFO] agent.router.manager: shutting down -13:26.239 [INFO] TestAgent: consul server down -13:26.239 [INFO] TestAgent: shutdown complete -13:26.239 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16303 network=tcp -13:26.239 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16303 network=udp -13:26.239 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16346 network=tcp -13:26.746 [INFO] TestAgent: Waiting for endpoints to shut down -13:26.746 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_loadChecks_token (0.74s) -=== CONT TestAgent_RemoveCheck -2021-10-31T23:13:25.925Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:26.020Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:26.049 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8566d4b3-0bbd-af02-d4dc-84c8a2ce5e39 Address:127.0.0.1:16301}]" -13:26.049 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16301 [Follower]" leader= -13:26.050 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8566d4b3-0bbd-af02-d4dc-84c8a2ce5e39.dc1 127.0.0.1 -13:26.051 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8566d4b3-0bbd-af02-d4dc-84c8a2ce5e39 127.0.0.1 -2021-10-31T23:13:26.051Z [INFO] agent.router: Initializing LAN area manager -13:26.051 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16338 network=udp -13:26.052 [INFO] TestAgent.server: Adding LAN server: server="Node-8566d4b3-0bbd-af02-d4dc-84c8a2ce5e39 (Addr: tcp/127.0.0.1:16301) (DC: dc1)" -13:26.052 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8566d4b3-0bbd-af02-d4dc-84c8a2ce5e39.dc1 area=wan -13:26.052 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16338 network=tcp -13:26.053 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16297 network=tcp -13:26.053 [INFO] TestAgent: started state syncer -13:26.053 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16358 network=tcp -13:26.113 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:26.113 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16301 [Candidate]" term=2 -13:26.114 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:26.114 [DEBUG] TestAgent.server.raft: vote granted: from=8566d4b3-0bbd-af02-d4dc-84c8a2ce5e39 term=2 tally=1 -13:26.114 [INFO] TestAgent.server.raft: election won: tally=1 -13:26.114 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16301 [Leader]" -13:26.114 [INFO] TestAgent.server: cluster leadership acquired -13:26.114 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16301 -13:26.115 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:26.117 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:26.117 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:26.117 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:26.117 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:26.117 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:26.117 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8566d4b3-0bbd-af02-d4dc-84c8a2ce5e39 -13:26.117 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8566d4b3-0bbd-af02-d4dc-84c8a2ce5e39 -13:26.118 [INFO] TestAgent.server: New leader elected: payload=Node-8566d4b3-0bbd-af02-d4dc-84c8a2ce5e39 -13:26.118 [INFO] TestAgent.server: federation state anti-entropy synced -13:26.201 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:26.202 [INFO] TestAgent: Synced node info -13:26.202 [DEBUG] TestAgent: Node info in sync -13:26.322 [INFO] TestAgent: Service entered maintenance mode: service=redis -13:26.322 [DEBUG] TestAgent: removed check: check=_service_maintenance:redis -13:26.322 [INFO] TestAgent: Service left maintenance mode: service=redis -13:26.322 [INFO] TestAgent: Service entered maintenance mode: service=redis -13:26.322 [INFO] TestAgent: Requesting shutdown -13:26.322 [INFO] TestAgent.server: shutting down server -13:26.322 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:26.322 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:26.322 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:26.322 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:26.322 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:26.322 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:26.322 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:26.322 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:26.322 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:26.322 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:26.323 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:26.323Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:26.323Z [INFO] agent.router.manager: shutting down -13:26.323 [INFO] TestAgent: consul server down -13:26.323 [INFO] TestAgent: shutdown complete -13:26.323 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16338 network=tcp -13:26.323 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16338 network=udp -13:26.323 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16297 network=tcp -13:26.825 [INFO] TestAgent: Waiting for endpoints to shut down -13:26.825 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Service_MaintenanceMode (0.95s) -=== CONT TestAgent_AddCheck_Alias_userAndSetToken -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=false -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:26.058Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:26.133Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:26.135 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a6bc61e5-6266-e3d9-4901-c60c544a2a95 Address:127.0.0.1:16399}]" -13:26.136 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a6bc61e5-6266-e3d9-4901-c60c544a2a95.dc1 127.0.0.1 -13:26.137 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a6bc61e5-6266-e3d9-4901-c60c544a2a95 127.0.0.1 -2021-10-31T23:13:26.137Z [INFO] agent.router: Initializing LAN area manager -13:26.137 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16359 network=udp -13:26.137 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16399 [Follower]" leader= -13:26.137 [INFO] TestAgent.server: Adding LAN server: server="Node-a6bc61e5-6266-e3d9-4901-c60c544a2a95 (Addr: tcp/127.0.0.1:16399) (DC: dc1)" -13:26.137 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16359 network=tcp -13:26.138 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16395 network=tcp -13:26.138 [INFO] TestAgent: started state syncer -13:26.138 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16302 network=tcp -13:26.145 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a6bc61e5-6266-e3d9-4901-c60c544a2a95.dc1 area=wan -13:26.183 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:26.183 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16399 [Candidate]" term=2 -13:26.183 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:26.183 [DEBUG] TestAgent.server.raft: vote granted: from=a6bc61e5-6266-e3d9-4901-c60c544a2a95 term=2 tally=1 -13:26.183 [INFO] TestAgent.server.raft: election won: tally=1 -13:26.183 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16399 [Leader]" -13:26.183 [INFO] TestAgent.server: cluster leadership acquired -13:26.183 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16399 -13:26.184 [INFO] TestAgent.server: New leader elected: payload=Node-a6bc61e5-6266-e3d9-4901-c60c544a2a95 -13:26.185 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:26.200 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:26.200 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:26.200 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:26.200 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:26.200 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:26.200 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a6bc61e5-6266-e3d9-4901-c60c544a2a95 -13:26.200 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a6bc61e5-6266-e3d9-4901-c60c544a2a95 -13:26.201 [INFO] TestAgent.server: federation state anti-entropy synced -13:26.304 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:26.304 [INFO] TestAgent: Synced node info -13:26.447 [DEBUG] TestAgent: removed check: check=service:redis -13:26.447 [INFO] TestAgent: Requesting shutdown -13:26.448 [INFO] TestAgent.server: shutting down server -13:26.448 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:26.448 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:26.448 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:26.448 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:26.448 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:26.448 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:26.448 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:26.448 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:26.448 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:26.448 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:26.448 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:26.448Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:26.448Z [INFO] agent.router.manager: shutting down -13:26.449 [INFO] TestAgent: consul server down -13:26.449 [INFO] TestAgent: shutdown complete -13:26.449 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16359 network=tcp -13:26.449 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16359 network=udp -13:26.449 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16395 network=tcp -13:26.949 [INFO] TestAgent: Waiting for endpoints to shut down -13:26.949 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_unloadChecks (0.98s) -=== CONT TestAgent_AddCheck_Alias_userToken -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=1)_compressed=true -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:26.548Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:26.603Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:26.605 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b7a97830-23f6-4201-7d91-bf1de8d09f68 Address:127.0.0.1:16420}]" -13:26.605 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b7a97830-23f6-4201-7d91-bf1de8d09f68.dc1 127.0.0.1 -13:26.605 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16420 [Follower]" leader= -13:26.614 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b7a97830-23f6-4201-7d91-bf1de8d09f68 127.0.0.1 -2021-10-31T23:13:26.614Z [INFO] agent.router: Initializing LAN area manager -13:26.614 [INFO] TestAgent.server: Adding LAN server: server="Node-b7a97830-23f6-4201-7d91-bf1de8d09f68 (Addr: tcp/127.0.0.1:16420) (DC: dc1)" -13:26.614 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b7a97830-23f6-4201-7d91-bf1de8d09f68.dc1 area=wan -13:26.615 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16394 network=udp -13:26.615 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16394 network=tcp -13:26.615 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16416 network=tcp -13:26.615 [INFO] TestAgent: started state syncer -13:26.615 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16428 network=tcp -13:26.646 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:26.646 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16420 [Candidate]" term=2 -13:26.646 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:26.646 [DEBUG] TestAgent.server.raft: vote granted: from=b7a97830-23f6-4201-7d91-bf1de8d09f68 term=2 tally=1 -13:26.646 [INFO] TestAgent.server.raft: election won: tally=1 -13:26.646 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16420 [Leader]" -13:26.646 [INFO] TestAgent.server: cluster leadership acquired -13:26.646 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16420 -13:26.647 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:26.649 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:26.649 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:26.649 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:26.649 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:26.649 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:26.649 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b7a97830-23f6-4201-7d91-bf1de8d09f68 -13:26.649 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b7a97830-23f6-4201-7d91-bf1de8d09f68 -13:26.649 [INFO] TestAgent.server: New leader elected: payload=Node-b7a97830-23f6-4201-7d91-bf1de8d09f68 -13:26.654 [INFO] TestAgent.server: federation state anti-entropy synced -13:26.674 [DEBUG] TestAgent: removed check: check=mem -13:26.674 [DEBUG] TestAgent: removed check: check=mem -13:26.674 [INFO] TestAgent: Requesting shutdown -13:26.674 [INFO] TestAgent.server: shutting down server -13:26.674 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:26.674 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:26.674 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:26.674 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:26.674 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:26.674 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:26.674 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:26.674 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:26.674 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:26.674 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:26.674 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:26.674 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:26.674Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:26.674Z [INFO] agent.router.manager: shutting down -13:26.675 [INFO] TestAgent: consul server down -13:26.675 [INFO] TestAgent: shutdown complete -13:26.675 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16394 network=tcp -13:26.675 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16394 network=udp -13:26.675 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16416 network=tcp -13:27.177 [INFO] TestAgent: Waiting for endpoints to shut down -13:27.177 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_PurgeCheck (0.66s) -=== CONT TestAgent_AddCheck_Alias_setToken -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:26.590Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:26.609Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:26.611 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7c187dc2-0441-602f-ce08-3b00aff4d879 Address:127.0.0.1:16406}]" -13:26.612 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7c187dc2-0441-602f-ce08-3b00aff4d879.dc1 127.0.0.1 -13:26.612 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7c187dc2-0441-602f-ce08-3b00aff4d879 127.0.0.1 -2021-10-31T23:13:26.612Z [INFO] agent.router: Initializing LAN area manager -13:26.613 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16401 network=udp -13:26.613 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16406 [Follower]" leader= -13:26.613 [INFO] TestAgent.server: Adding LAN server: server="Node-7c187dc2-0441-602f-ce08-3b00aff4d879 (Addr: tcp/127.0.0.1:16406) (DC: dc1)" -13:26.613 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7c187dc2-0441-602f-ce08-3b00aff4d879.dc1 area=wan -13:26.613 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16401 network=tcp -13:26.614 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16402 network=tcp -13:26.614 [INFO] TestAgent: started state syncer -13:26.614 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16421 network=tcp -13:26.685 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:26.685 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16406 [Candidate]" term=2 -13:26.685 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:26.685 [DEBUG] TestAgent.server.raft: vote granted: from=7c187dc2-0441-602f-ce08-3b00aff4d879 term=2 tally=1 -13:26.685 [INFO] TestAgent.server.raft: election won: tally=1 -13:26.685 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16406 [Leader]" -13:26.685 [INFO] TestAgent.server: cluster leadership acquired -13:26.685 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16406 -13:26.685 [INFO] TestAgent.server: New leader elected: payload=Node-7c187dc2-0441-602f-ce08-3b00aff4d879 -13:26.693 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:26.695 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:26.695 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:26.695 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:26.695 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:26.695 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:26.695 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7c187dc2-0441-602f-ce08-3b00aff4d879 -13:26.695 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7c187dc2-0441-602f-ce08-3b00aff4d879 -13:26.696 [INFO] TestAgent.server: federation state anti-entropy synced -13:26.882 [DEBUG] TestAgent: Check status updated: check=mem status=passing -13:26.882 [DEBUG] TestAgent: Check status updated: check=mem status=critical -13:26.882 [INFO] TestAgent: Requesting shutdown -13:26.882 [INFO] TestAgent.server: shutting down server -13:26.882 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:26.882 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:26.882 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:26.882 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:26.882 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:26.882 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:26.882 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:26.882 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:26.882 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:26.882 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:26.882 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:26.882 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:26.882Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:26.882Z [INFO] agent.router.manager: shutting down -13:26.883 [INFO] TestAgent: consul server down -13:26.883 [INFO] TestAgent: shutdown complete -13:26.883 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16401 network=tcp -13:26.883 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16401 network=udp -13:26.883 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16402 network=tcp -13:27.386 [INFO] TestAgent: Waiting for endpoints to shut down -13:27.386 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_updateTTLCheck (0.83s) -=== CONT TestAgent_AddCheck_Alias -2021-10-31T23:13:26.856Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:26.921Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:26.944 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d4abc4d8-f060-0ec8-ec97-81a0828f9220 Address:127.0.0.1:16392}]" -13:26.946 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16392 [Follower]" leader= -13:26.973 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d4abc4d8-f060-0ec8-ec97-81a0828f9220.dc1 127.0.0.1 -13:26.986 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:26.986 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16392 [Candidate]" term=2 -13:26.986 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=d4abc4d8-f060-0ec8-ec97-81a0828f9220 fallback=127.0.0.1:16392 error="Could not find address for server id d4abc4d8-f060-0ec8-ec97-81a0828f9220" -13:26.986 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:26.986 [DEBUG] TestAgent.server.raft: vote granted: from=d4abc4d8-f060-0ec8-ec97-81a0828f9220 term=2 tally=1 -13:26.987 [INFO] TestAgent.server.raft: election won: tally=1 -13:26.987 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16392 [Leader]" -13:26.997 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d4abc4d8-f060-0ec8-ec97-81a0828f9220 127.0.0.1 -2021-10-31T23:13:26.997Z [INFO] agent.router: Initializing LAN area manager -13:26.998 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d4abc4d8-f060-0ec8-ec97-81a0828f9220.dc1 area=wan -13:26.998 [INFO] TestAgent.server: Adding LAN server: server="Node-d4abc4d8-f060-0ec8-ec97-81a0828f9220 (Addr: tcp/127.0.0.1:16392) (DC: dc1)" -13:26.998 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16380 network=udp -13:26.998 [INFO] TestAgent.server: cluster leadership acquired -13:26.998 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16380 network=tcp -13:26.998 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16392 -13:26.999 [INFO] TestAgent.server: New leader elected: payload=Node-d4abc4d8-f060-0ec8-ec97-81a0828f9220 -13:26.999 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16388 network=tcp -13:26.999 [INFO] TestAgent: started state syncer -13:26.999 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:26.999 [INFO] TestAgent: Requesting shutdown -13:26.999 [INFO] TestAgent.server: shutting down server -13:26.999 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:26.999 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16442 network=tcp -13:26.999 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -13:27.001 [INFO] TestAgent: Synced node info -13:27.001 [INFO] TestAgent: Synced check: check=aliashealth -13:27.006 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:27.006Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:27.006Z [INFO] agent.router.manager: shutting down -13:27.018 [INFO] TestAgent: consul server down -13:27.018 [INFO] TestAgent: shutdown complete -13:27.018 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16380 network=tcp -13:27.018 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16380 network=udp -13:27.018 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16388 network=tcp -13:27.018 [ERROR] TestAgent.server.connect: Raft apply failed: error="leadership lost while committing log" -13:27.018 [ERROR] TestAgent.server: failed to establish leadership: error="leadership lost while committing log" -13:27.519 [INFO] TestAgent: Waiting for endpoints to shut down -13:27.519 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_Alias_userAndSetToken (0.69s) -=== CONT TestAgent_AddCheck_GRPC -2021-10-31T23:13:26.790Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:26.790Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2021-10-31T23:13:26.816Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:26.816Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -13:26.842 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b35ddfa2-cc1e-7db8-cc04-381da40d15c6 Address:127.0.0.1:16455}]" -13:26.843 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b35ddfa2-cc1e-7db8-cc04-381da40d15c6.dc1 127.0.0.1 -13:26.843 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16455 [Follower]" leader= -13:26.843 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b35ddfa2-cc1e-7db8-cc04-381da40d15c6 127.0.0.1 -2021-10-31T23:13:26.843Z [INFO] agent.router: Initializing LAN area manager -13:26.844 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16436 network=udp -13:26.844 [INFO] TestAgent.server: Adding LAN server: server="Node-b35ddfa2-cc1e-7db8-cc04-381da40d15c6 (Addr: tcp/127.0.0.1:16455) (DC: dc1)" -13:26.844 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b35ddfa2-cc1e-7db8-cc04-381da40d15c6.dc1 area=wan -13:26.844 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16436 network=tcp -13:26.845 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16451 network=tcp -13:26.845 [INFO] TestAgent: started state syncer -13:26.845 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16379 network=tcp -13:26.881 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:26.881 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16455 [Candidate]" term=2 -13:26.882 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:26.882 [DEBUG] TestAgent.server.raft: vote granted: from=b35ddfa2-cc1e-7db8-cc04-381da40d15c6 term=2 tally=1 -13:26.882 [INFO] TestAgent.server.raft: election won: tally=1 -13:26.882 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16455 [Leader]" -13:26.882 [INFO] TestAgent.server: cluster leadership acquired -13:26.882 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16455 -13:26.883 [INFO] TestAgent.server: New leader elected: payload=Node-b35ddfa2-cc1e-7db8-cc04-381da40d15c6 -13:26.883 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:26.885 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:26.885 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:26.885 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:26.885 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:26.885 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:26.885 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b35ddfa2-cc1e-7db8-cc04-381da40d15c6 -13:26.885 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b35ddfa2-cc1e-7db8-cc04-381da40d15c6 -13:26.890 [INFO] TestAgent.server: federation state anti-entropy synced -13:27.010 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:27.010 [INFO] TestAgent: Synced node info -13:27.010 [DEBUG] TestAgent: Node info in sync -13:27.046 [DEBUG] TestAgent: removed check: check=mem -13:27.046 [DEBUG] TestAgent: removed check: check=mem -13:27.046 [INFO] TestAgent: Requesting shutdown -13:27.046 [INFO] TestAgent.server: shutting down server -13:27.046 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:27.046 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:27.046 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:27.046 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:27.046 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:27.046 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:27.046 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:27.046 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:27.046 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:27.046 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:27.046 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:27.046Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:27.046Z [INFO] agent.router.manager: shutting down -13:27.046 [INFO] TestAgent: consul server down -13:27.046 [INFO] TestAgent: shutdown complete -13:27.046 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16436 network=tcp -13:27.046 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16436 network=udp -13:27.047 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16451 network=tcp -13:27.554 [INFO] TestAgent: Waiting for endpoints to shut down -13:27.554 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RemoveCheck (0.81s) -=== CONT TestAgent_AddCheck_ExecRemoteDisable -2021-10-31T23:13:27.062Z [WARN] agent: Node name "Node e03d5866-da1c-59d9-7c42-9cca6caa4a49" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2021-10-31T23:13:27.092Z [WARN] agent.auto_config: Node name "Node e03d5866-da1c-59d9-7c42-9cca6caa4a49" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -13:27.092 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node e03d5866-da1c-59d9-7c42-9cca6caa4a49 127.0.0.1 -2021-10-31T23:13:27.092Z [INFO] agent.router: Initializing LAN area manager -13:27.093 [DEBUG] Agent2: restored service definition from file: service=svc-sidecar-proxy file=/tmp/consul-test/TestAgent_DeregisterPersistedSidecarAfterRestart-TestAgent-agent368318248/services/18ed17ea501b78b6980ecfa920fcf57a -13:27.093 [DEBUG] Agent2: restored service definition from file: service=svc file=/tmp/consul-test/TestAgent_DeregisterPersistedSidecarAfterRestart-TestAgent-agent368318248/services/961e38b5146c0b07d078f53dec9b699a -13:27.093 [INFO] Agent2: Started DNS server: address=127.0.0.1:16450 network=udp -13:27.093 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node -13:27.094 [INFO] Agent2: Started DNS server: address=127.0.0.1:16450 network=tcp -13:27.094 [INFO] Agent2: Started HTTP server: address=127.0.0.1:16430 network=tcp -13:27.095 [INFO] Agent2: started state syncer -13:27.095 [DEBUG] Agent2: removed service: service=svc -13:27.095 [DEBUG] Agent2: removed service: service=svc-sidecar-proxy -13:27.095 [INFO] Agent2: Requesting shutdown -13:27.095 [INFO] Agent2: Started gRPC server: address=127.0.0.1:16414 network=tcp -13:27.095 [ERROR] Agent2: gRPC server failed: error="grpc: the server has been stopped" -2021-10-31T23:13:27.096Z [WARN] agent.router.manager: No servers available -13:27.096 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" -13:27.098 [INFO] Agent2.client: shutting down client -13:27.098 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave -13:27.098 [INFO] Agent2: consul client down -13:27.098 [INFO] Agent2: shutdown complete -13:27.098 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:16450 network=tcp -13:27.098 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:16450 network=udp -13:27.098 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:16430 network=tcp -13:27.598 [INFO] Agent2: Waiting for endpoints to shut down -13:27.598 [INFO] Agent2: Endpoints down -2021-10-31T23:13:26.476Z [WARN] agent: Node name "Node e03d5866-da1c-59d9-7c42-9cca6caa4a49" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2021-10-31T23:13:26.531Z [WARN] agent.auto_config: Node name "Node e03d5866-da1c-59d9-7c42-9cca6caa4a49" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -13:26.532 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node e03d5866-da1c-59d9-7c42-9cca6caa4a49 127.0.0.1 -2021-10-31T23:13:26.532Z [INFO] agent.router: Initializing LAN area manager -13:26.532 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16310 network=udp -13:26.532 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16310 network=tcp -13:26.533 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16332 network=tcp -13:26.533 [INFO] TestAgent: started state syncer -13:26.533 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16351 network=tcp -13:26.538 [INFO] TestAgent: Requesting shutdown -13:26.538 [INFO] TestAgent.client: shutting down client -13:26.538 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -13:26.538 [INFO] TestAgent: consul client down -13:26.538 [INFO] TestAgent: shutdown complete -13:26.538 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16310 network=tcp -13:26.538 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16310 network=udp -13:26.538 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16332 network=tcp -13:27.041 [INFO] TestAgent: Waiting for endpoints to shut down -13:27.041 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_DeregisterPersistedSidecarAfterRestart (1.24s) -=== CONT TestAgent_AddCheck_ExecDisable -[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:27.095Z [WARN] agent: Node name "Node f12b09bc-f758-d873-075b-53ed738c5ad0" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2021-10-31T23:13:27.095Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2021-10-31T23:13:27.121Z [WARN] agent.auto_config: Node name "Node f12b09bc-f758-d873-075b-53ed738c5ad0" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2021-10-31T23:13:27.121Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -13:27.130 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node f12b09bc-f758-d873-075b-53ed738c5ad0 127.0.0.1 -2021-10-31T23:13:27.130Z [INFO] agent.router: Initializing LAN area manager -13:27.130 [DEBUG] Agent2: check exists, not restoring from file: check=mem file=/tmp/consul-test/TestAgent_PurgeCheckOnDuplicate-TestAgent-agent637458023/checks/afc4fc7e48a0710a1dc94ef3e8bc5764 -13:27.130 [INFO] Agent2: Started DNS server: address=127.0.0.1:16415 network=udp -13:27.130 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node -13:27.131 [INFO] Agent2: Started DNS server: address=127.0.0.1:16415 network=tcp -13:27.131 [INFO] Agent2: Started HTTP server: address=127.0.0.1:16360 network=tcp -13:27.131 [INFO] Agent2: started state syncer -13:27.132 [INFO] Agent2: Requesting shutdown -13:27.132 [INFO] Agent2.client: shutting down client -13:27.132 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave -13:27.132 [INFO] Agent2: consul client down -13:27.132 [INFO] Agent2: shutdown complete -13:27.132 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:16415 network=tcp -13:27.132 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:16415 network=udp -13:27.132 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:16360 network=tcp -13:27.132 [INFO] Agent2: Started gRPC server: address=127.0.0.1:16463 network=tcp -13:27.132 [ERROR] Agent2: gRPC server failed: error="grpc: the server has been stopped" -2021-10-31T23:13:27.132Z [WARN] agent.router.manager: No servers available -13:27.132 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" -13:27.633 [INFO] Agent2: Waiting for endpoints to shut down -13:27.649 [INFO] Agent2: Endpoints down -2021-10-31T23:13:26.511Z [WARN] agent: Node name "Node f12b09bc-f758-d873-075b-53ed738c5ad0" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2021-10-31T23:13:26.511Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2021-10-31T23:13:26.560Z [WARN] agent.auto_config: Node name "Node f12b09bc-f758-d873-075b-53ed738c5ad0" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2021-10-31T23:13:26.560Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -13:26.560 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node f12b09bc-f758-d873-075b-53ed738c5ad0 127.0.0.1 -2021-10-31T23:13:26.560Z [INFO] agent.router: Initializing LAN area manager -13:26.561 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16352 network=udp -13:26.561 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16352 network=tcp -13:26.570 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16353 network=tcp -13:26.570 [INFO] TestAgent: started state syncer -13:26.570 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16393 network=tcp -13:26.571 [INFO] TestAgent: Requesting shutdown -13:26.571 [INFO] TestAgent.client: shutting down client -13:26.571 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -2021-10-31T23:13:26.571Z [WARN] agent.router.manager: No servers available -13:26.571 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -13:26.571 [INFO] TestAgent: consul client down -13:26.571 [INFO] TestAgent: shutdown complete -13:26.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16352 network=tcp -13:26.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16352 network=udp -13:26.571 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16353 network=tcp -13:27.072 [INFO] TestAgent: Waiting for endpoints to shut down -13:27.072 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_PurgeCheckOnDuplicate (1.22s) -=== CONT TestAgent_AddCheck_RestoreState -2021-10-31T23:13:27.161Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2021-10-31T23:13:27.175Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -13:27.175 [INFO] Agent2.client.serf.lan: serf: EventMemberJoin: Node-70d7d6a0-1fae-d094-0b4d-1885797b3518 127.0.0.1 -2021-10-31T23:13:27.175Z [INFO] agent.router: Initializing LAN area manager -13:27.176 [DEBUG] Agent2: restored health check from file: check=mem file=/tmp/consul-test/TestAgent_PersistCheck-TestAgent-agent421574449/checks/afc4fc7e48a0710a1dc94ef3e8bc5764 -13:27.176 [INFO] Agent2: Started DNS server: address=127.0.0.1:16464 network=udp -13:27.176 [INFO] Agent2.client.serf.lan: serf: Attempting re-join to previously known node: Node-82de0706-8265-9dd8-534f-09c1658c5f6f: 127.0.0.1:16376 -13:27.176 [INFO] Agent2: Started DNS server: address=127.0.0.1:16464 network=tcp -13:27.177 [INFO] Agent2: Started HTTP server: address=127.0.0.1:16409 network=tcp -13:27.177 [INFO] Agent2: started state syncer -13:27.177 [INFO] Agent2: Requesting shutdown -13:27.177 [INFO] Agent2.client: shutting down client -13:27.177 [WARN] Agent2.client.serf.lan: serf: Shutdown without a Leave -13:27.178 [INFO] Agent2: Started gRPC server: address=127.0.0.1:16372 network=tcp -13:27.178 [ERROR] Agent2: gRPC server failed: error="grpc: the server has been stopped" -2021-10-31T23:13:27.178Z [WARN] agent.router.manager: No servers available -13:27.178 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No known Consul servers" -13:27.178 [INFO] Agent2: consul client down -13:27.178 [INFO] Agent2: shutdown complete -13:27.178 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:16464 network=tcp -13:27.178 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:16464 network=udp -13:27.178 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:16409 network=tcp -13:27.178 [DEBUG] Agent2.client.memberlist.lan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:16376: connect: connection refused -13:27.178 [WARN] Agent2.client.serf.lan: serf: Failed to re-join any previously known node -13:27.678 [INFO] Agent2: Waiting for endpoints to shut down -13:27.678 [INFO] Agent2: Endpoints down -2021-10-31T23:13:26.600Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2021-10-31T23:13:26.641Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -13:26.642 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-82de0706-8265-9dd8-534f-09c1658c5f6f 127.0.0.1 -2021-10-31T23:13:26.642Z [INFO] agent.router: Initializing LAN area manager -13:26.642 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16429 network=udp -13:26.642 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16429 network=tcp -13:26.643 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16374 network=tcp -13:26.643 [INFO] TestAgent: started state syncer -13:26.644 [INFO] TestAgent: Requesting shutdown -13:26.644 [INFO] TestAgent.client: shutting down client -13:26.644 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave -13:26.644 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16400 network=tcp -13:26.644 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -2021-10-31T23:13:26.644Z [WARN] agent.router.manager: No servers available -13:26.644 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -13:26.644 [INFO] TestAgent: consul client down -13:26.644 [INFO] TestAgent: shutdown complete -13:26.644 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16429 network=tcp -13:26.644 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16429 network=udp -13:26.644 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16374 network=tcp -13:27.146 [INFO] TestAgent: Waiting for endpoints to shut down -13:27.146 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_PersistCheck (1.12s) -=== CONT TestAgent_AddCheck_MissingService -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:27.015Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:27.068Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:27.070 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a1b3236b-ef9b-1407-37e6-fd7840423110 Address:127.0.0.1:16448}]" -13:27.071 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a1b3236b-ef9b-1407-37e6-fd7840423110.dc1 127.0.0.1 -13:27.071 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a1b3236b-ef9b-1407-37e6-fd7840423110 127.0.0.1 -2021-10-31T23:13:27.071Z [INFO] agent.router: Initializing LAN area manager -13:27.071 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16443 network=udp -13:27.071 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16448 [Follower]" leader= -13:27.072 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16443 network=tcp -13:27.072 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a1b3236b-ef9b-1407-37e6-fd7840423110.dc1 area=wan -13:27.072 [INFO] TestAgent.server: Adding LAN server: server="Node-a1b3236b-ef9b-1407-37e6-fd7840423110 (Addr: tcp/127.0.0.1:16448) (DC: dc1)" -13:27.072 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16444 network=tcp -13:27.072 [INFO] TestAgent: started state syncer -13:27.072 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16449 network=tcp -13:27.109 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:27.109 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16448 [Candidate]" term=2 -13:27.110 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:27.110 [DEBUG] TestAgent.server.raft: vote granted: from=a1b3236b-ef9b-1407-37e6-fd7840423110 term=2 tally=1 -13:27.110 [INFO] TestAgent.server.raft: election won: tally=1 -13:27.110 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16448 [Leader]" -13:27.110 [INFO] TestAgent.server: cluster leadership acquired -13:27.110 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16448 -13:27.111 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:27.112 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:27.112 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:27.112 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:27.112 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:27.112 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:27.113 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1b3236b-ef9b-1407-37e6-fd7840423110 -13:27.113 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a1b3236b-ef9b-1407-37e6-fd7840423110 -13:27.113 [INFO] TestAgent.server: New leader elected: payload=Node-a1b3236b-ef9b-1407-37e6-fd7840423110 -13:27.113 [INFO] TestAgent.server: federation state anti-entropy synced -13:27.413 [INFO] TestAgent: Requesting shutdown -13:27.413 [INFO] TestAgent.server: shutting down server -13:27.413 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:27.413 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:27.413 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:27.413 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:27.413 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:27.413 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:27.413 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:27.413 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:27.413 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:27.413 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:27.413 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:27.413 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2021-10-31T23:13:27.413Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:27.413Z [INFO] agent.router.manager: shutting down -13:27.414 [INFO] TestAgent: consul server down -13:27.414 [INFO] TestAgent: shutdown complete -13:27.414 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16443 network=tcp -13:27.414 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16443 network=udp -13:27.414 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16444 network=tcp -13:27.914 [INFO] TestAgent: Waiting for endpoints to shut down -13:27.914 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_Alias_userToken (0.95s) -=== CONT TestAgent_AddCheck_MinInterval -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:27.213Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:27.270Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:27.273 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a325bd30-69cd-c7ee-7003-481ce7cf37f2 Address:127.0.0.1:16427}]" -13:27.273 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16427 [Follower]" leader= -13:27.274 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a325bd30-69cd-c7ee-7003-481ce7cf37f2.dc1 127.0.0.1 -13:27.275 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a325bd30-69cd-c7ee-7003-481ce7cf37f2 127.0.0.1 -2021-10-31T23:13:27.275Z [INFO] agent.router: Initializing LAN area manager -13:27.275 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16373 network=udp -13:27.275 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a325bd30-69cd-c7ee-7003-481ce7cf37f2.dc1 area=wan -13:27.275 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16373 network=tcp -13:27.277 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16423 network=tcp -13:27.277 [INFO] TestAgent: started state syncer -13:27.277 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16323 network=tcp -13:27.286 [INFO] TestAgent.server: Adding LAN server: server="Node-a325bd30-69cd-c7ee-7003-481ce7cf37f2 (Addr: tcp/127.0.0.1:16427) (DC: dc1)" -13:27.325 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:27.325 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16427 [Candidate]" term=2 -13:27.330 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:27.330 [DEBUG] TestAgent.server.raft: vote granted: from=a325bd30-69cd-c7ee-7003-481ce7cf37f2 term=2 tally=1 -13:27.330 [INFO] TestAgent.server.raft: election won: tally=1 -13:27.330 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16427 [Leader]" -13:27.330 [INFO] TestAgent.server: cluster leadership acquired -13:27.330 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16427 -13:27.330 [INFO] TestAgent.server: New leader elected: payload=Node-a325bd30-69cd-c7ee-7003-481ce7cf37f2 -13:27.331 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:27.332 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:27.332 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:27.332 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:27.332 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:27.332 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:27.332 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a325bd30-69cd-c7ee-7003-481ce7cf37f2 -13:27.332 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a325bd30-69cd-c7ee-7003-481ce7cf37f2 -13:27.333 [INFO] TestAgent.server: federation state anti-entropy synced -13:27.593 [INFO] TestAgent: Requesting shutdown -13:27.593 [INFO] TestAgent.server: shutting down server -13:27.596 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:27.596 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:27.596 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:27.596 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:27.596 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:27.597 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:27.597 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:27.598 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:27.598 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:27.598 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:27.598 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:27.599 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:27.655Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:27.655Z [INFO] agent.router.manager: shutting down -13:27.655 [INFO] TestAgent: consul server down -13:27.655 [INFO] TestAgent: shutdown complete -13:27.655 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16373 network=tcp -13:27.655 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16373 network=udp -13:27.655 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16423 network=tcp -13:28.165 [INFO] TestAgent: Waiting for endpoints to shut down -13:28.165 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_Alias_setToken (0.99s) -2021-10-31T23:13:27.494Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:27.516Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:27.531 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:32329b10-ad07-ea43-06ba-3bee1c1556fa Address:127.0.0.1:16287}]" -13:27.532 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-32329b10-ad07-ea43-06ba-3bee1c1556fa.dc1 127.0.0.1 -13:27.532 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-32329b10-ad07-ea43-06ba-3bee1c1556fa 127.0.0.1 -2021-10-31T23:13:27.532Z [INFO] agent.router: Initializing LAN area manager -13:27.532 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16324 network=udp -13:27.532 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16287 [Follower]" leader= -13:27.533 [INFO] TestAgent.server: Adding LAN server: server="Node-32329b10-ad07-ea43-06ba-3bee1c1556fa (Addr: tcp/127.0.0.1:16287) (DC: dc1)" -13:27.533 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-32329b10-ad07-ea43-06ba-3bee1c1556fa.dc1 area=wan -13:27.533 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16324 network=tcp -13:27.538 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16283 network=tcp -13:27.538 [INFO] TestAgent: started state syncer -13:27.538 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16211 network=tcp -13:27.586 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:27.586 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16287 [Candidate]" term=2 -13:27.586 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:27.586 [DEBUG] TestAgent.server.raft: vote granted: from=32329b10-ad07-ea43-06ba-3bee1c1556fa term=2 tally=1 -13:27.586 [INFO] TestAgent.server.raft: election won: tally=1 -13:27.586 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16287 [Leader]" -13:27.586 [INFO] TestAgent.server: cluster leadership acquired -13:27.586 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16287 -13:27.588 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:27.589 [INFO] TestAgent.server: New leader elected: payload=Node-32329b10-ad07-ea43-06ba-3bee1c1556fa -13:27.654 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:27.654 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:27.654 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:27.654 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:27.654 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:27.654 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-32329b10-ad07-ea43-06ba-3bee1c1556fa -13:27.654 [INFO] TestAgent.server: member joined, marking health alive: member=Node-32329b10-ad07-ea43-06ba-3bee1c1556fa -13:27.655 [INFO] TestAgent.server: federation state anti-entropy synced -13:27.658 [INFO] TestAgent: Requesting shutdown -13:27.658 [INFO] TestAgent.server: shutting down server -13:27.658 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:27.658 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:27.658 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:27.658 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:27.658 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:27.658 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:27.658 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:27.658 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:27.658 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:27.658 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:27.658 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:27.658 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:27.658Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:27.658Z [INFO] agent.router.manager: shutting down -13:27.658 [INFO] TestAgent: consul server down -13:27.658 [INFO] TestAgent: shutdown complete -13:27.658 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16324 network=tcp -13:27.659 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16324 network=udp -13:27.659 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16283 network=tcp -13:28.170 [INFO] TestAgent: Waiting for endpoints to shut down -13:28.170 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_Alias (0.76s) -=== CONT TestAgent_AddCheck -=== CONT TestAgent_AddCheck_StartPassing -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=1)_compressed=false -2021-10-31T23:13:27.684Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:27.735Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:27.738 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9383750b-5431-cce7-a3bc-7970b17feb6f Address:127.0.0.1:16469}]" -13:27.739 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16469 [Follower]" leader= -13:27.739 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9383750b-5431-cce7-a3bc-7970b17feb6f.dc1 127.0.0.1 -13:27.740 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9383750b-5431-cce7-a3bc-7970b17feb6f 127.0.0.1 -2021-10-31T23:13:27.740Z [INFO] agent.router: Initializing LAN area manager -13:27.741 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16485 network=udp -13:27.741 [INFO] TestAgent.server: Adding LAN server: server="Node-9383750b-5431-cce7-a3bc-7970b17feb6f (Addr: tcp/127.0.0.1:16469) (DC: dc1)" -13:27.742 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16485 network=tcp -13:27.743 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16465 network=tcp -13:27.743 [INFO] TestAgent: started state syncer -13:27.744 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16477 network=tcp -13:27.748 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9383750b-5431-cce7-a3bc-7970b17feb6f.dc1 area=wan -13:27.782 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:27.782 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16469 [Candidate]" term=2 -13:27.782 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:27.782 [DEBUG] TestAgent.server.raft: vote granted: from=9383750b-5431-cce7-a3bc-7970b17feb6f term=2 tally=1 -13:27.782 [INFO] TestAgent.server.raft: election won: tally=1 -13:27.782 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16469 [Leader]" -13:27.782 [INFO] TestAgent.server: cluster leadership acquired -13:27.782 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16469 -13:27.783 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:27.784 [INFO] TestAgent.server: New leader elected: payload=Node-9383750b-5431-cce7-a3bc-7970b17feb6f -13:27.788 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:27.788 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:27.788 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:27.788 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:27.788 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:27.788 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9383750b-5431-cce7-a3bc-7970b17feb6f -13:27.788 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9383750b-5431-cce7-a3bc-7970b17feb6f -13:27.811 [INFO] TestAgent.server: federation state anti-entropy synced -13:27.842 [INFO] TestAgent: Requesting shutdown -13:27.842 [INFO] TestAgent.server: shutting down server -13:27.842 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:27.842 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:27.842 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:27.842 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:27.842 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:27.842 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:27.842 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:27.842 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:27.842 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:27.842 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:27.842 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:27.842 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:27.842Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:27.842Z [INFO] agent.router.manager: shutting down -13:27.842 [INFO] TestAgent: consul server down -13:27.842 [INFO] TestAgent: shutdown complete -13:27.842 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16485 network=tcp -13:27.842 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16485 network=udp -13:27.843 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16465 network=tcp -13:28.344 [INFO] TestAgent: Waiting for endpoints to shut down -13:28.344 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_RestoreState (0.68s) -=== CONT TestAgent_IndexChurn -=== RUN TestAgent_IndexChurn/no_tags -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:27.539Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:27.602Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:27.679 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a7545b26-3c86-b58d-6839-8dff9f08811e Address:127.0.0.1:16189}]" -13:27.680 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a7545b26-3c86-b58d-6839-8dff9f08811e.dc1 127.0.0.1 -13:27.680 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a7545b26-3c86-b58d-6839-8dff9f08811e 127.0.0.1 -2021-10-31T23:13:27.680Z [INFO] agent.router: Initializing LAN area manager -13:27.681 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16212 network=udp -13:27.681 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16189 [Follower]" leader= -13:27.681 [INFO] TestAgent.server: Adding LAN server: server="Node-a7545b26-3c86-b58d-6839-8dff9f08811e (Addr: tcp/127.0.0.1:16189) (DC: dc1)" -13:27.681 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a7545b26-3c86-b58d-6839-8dff9f08811e.dc1 area=wan -13:27.681 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16212 network=tcp -13:27.690 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16185 network=tcp -13:27.690 [INFO] TestAgent: started state syncer -13:27.690 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16365 network=tcp -13:27.738 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:27.738 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16189 [Candidate]" term=2 -13:27.738 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:27.738 [DEBUG] TestAgent.server.raft: vote granted: from=a7545b26-3c86-b58d-6839-8dff9f08811e term=2 tally=1 -13:27.738 [INFO] TestAgent.server.raft: election won: tally=1 -13:27.738 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16189 [Leader]" -13:27.738 [INFO] TestAgent.server: cluster leadership acquired -13:27.738 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16189 -13:27.739 [INFO] TestAgent.server: New leader elected: payload=Node-a7545b26-3c86-b58d-6839-8dff9f08811e -13:27.739 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:27.750 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:27.750 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:27.750 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:27.750 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:27.750 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:27.751 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a7545b26-3c86-b58d-6839-8dff9f08811e -13:27.751 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a7545b26-3c86-b58d-6839-8dff9f08811e -13:27.760 [INFO] TestAgent.server: federation state anti-entropy synced -13:27.970 [INFO] TestAgent: Requesting shutdown -13:27.970 [INFO] TestAgent.server: shutting down server -13:27.970 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:27.970 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:27.970 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:27.970 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:27.970 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:27.970 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:27.970 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:27.970 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:27.970 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:27.970 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:27.970 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:27.970 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:27.971Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:27.971Z [INFO] agent.router.manager: shutting down -13:27.971 [INFO] TestAgent: consul server down -13:27.971 [INFO] TestAgent: shutdown complete -13:27.971 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16212 network=tcp -13:27.971 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16212 network=udp -13:27.971 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16185 network=tcp -13:28.473 [INFO] TestAgent: Waiting for endpoints to shut down -13:28.473 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_GRPC (0.95s) -=== CONT TestAgent_ReconnectConfigWanDisabled -2021-10-31T23:13:27.732Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:27.732Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2021-10-31T23:13:27.799Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:27.799Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -13:27.813 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8e27982d-3d51-b0d4-0eee-e492929a858d Address:127.0.0.1:16385}]" -13:27.813 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16385 [Follower]" leader= -13:27.814 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8e27982d-3d51-b0d4-0eee-e492929a858d.dc1 127.0.0.1 -13:27.815 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8e27982d-3d51-b0d4-0eee-e492929a858d 127.0.0.1 -2021-10-31T23:13:27.815Z [INFO] agent.router: Initializing LAN area manager -13:27.815 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16478 network=udp -13:27.816 [INFO] TestAgent.server: Adding LAN server: server="Node-8e27982d-3d51-b0d4-0eee-e492929a858d (Addr: tcp/127.0.0.1:16385) (DC: dc1)" -13:27.816 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8e27982d-3d51-b0d4-0eee-e492929a858d.dc1 area=wan -13:27.818 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16478 network=tcp -13:27.818 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16381 network=tcp -13:27.818 [INFO] TestAgent: started state syncer -13:27.821 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16491 network=tcp -13:27.857 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:27.857 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16385 [Candidate]" term=2 -13:27.858 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:27.858 [DEBUG] TestAgent.server.raft: vote granted: from=8e27982d-3d51-b0d4-0eee-e492929a858d term=2 tally=1 -13:27.858 [INFO] TestAgent.server.raft: election won: tally=1 -13:27.858 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16385 [Leader]" -13:27.858 [INFO] TestAgent.server: cluster leadership acquired -13:27.858 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16385 -13:27.858 [INFO] TestAgent.server: New leader elected: payload=Node-8e27982d-3d51-b0d4-0eee-e492929a858d -13:27.859 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:27.860 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:27.860 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:27.860 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:27.860 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:27.860 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:27.860 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8e27982d-3d51-b0d4-0eee-e492929a858d -13:27.860 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8e27982d-3d51-b0d4-0eee-e492929a858d -13:27.861 [INFO] TestAgent.server: federation state anti-entropy synced -13:28.026 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:28.026 [INFO] TestAgent: Synced node info -13:28.026 [DEBUG] TestAgent: Node info in sync -13:28.030 [INFO] TestAgent: Requesting shutdown -13:28.030 [INFO] TestAgent.server: shutting down server -13:28.030 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:28.030 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:28.030 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:28.030 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:28.030 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:28.030 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:28.030 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:28.030 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:28.030 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:28.030 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:28.030 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:28.046Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:28.046Z [INFO] agent.router.manager: shutting down -13:28.046 [INFO] TestAgent: consul server down -13:28.046 [INFO] TestAgent: shutdown complete -13:28.046 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16478 network=tcp -13:28.046 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16478 network=udp -13:28.046 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16381 network=tcp -13:28.547 [INFO] TestAgent: Waiting for endpoints to shut down -13:28.547 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_MissingService (0.87s) -=== CONT TestAgent_ReconnectConfigSettings -2021-10-31T23:13:26.664Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:26.695Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:26.697 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:194bdff8-d349-2311-3797-afff4d7bbf9d Address:127.0.0.1:16441}]" -13:26.697 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-194bdff8-d349-2311-3797-afff4d7bbf9d.dc1 127.0.0.1 -13:26.697 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16441 [Follower]" leader= -13:26.706 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-194bdff8-d349-2311-3797-afff4d7bbf9d 127.0.0.1 -2021-10-31T23:13:26.706Z [INFO] agent.router: Initializing LAN area manager -13:26.710 [INFO] TestAgent.server: Adding LAN server: server="Node-194bdff8-d349-2311-3797-afff4d7bbf9d (Addr: tcp/127.0.0.1:16441) (DC: dc1)" -13:26.710 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-194bdff8-d349-2311-3797-afff4d7bbf9d.dc1 area=wan -13:26.714 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16422 network=udp -13:26.714 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16422 network=tcp -13:26.714 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16437 network=tcp -13:26.714 [INFO] TestAgent: started state syncer -13:26.715 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16435 network=tcp -13:26.741 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:26.741 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16441 [Candidate]" term=2 -13:26.742 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:26.742 [DEBUG] TestAgent.server.raft: vote granted: from=194bdff8-d349-2311-3797-afff4d7bbf9d term=2 tally=1 -13:26.742 [INFO] TestAgent.server.raft: election won: tally=1 -13:26.742 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16441 [Leader]" -13:26.742 [INFO] TestAgent.server: cluster leadership acquired -13:26.742 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16441 -13:26.743 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:26.744 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:26.744 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:26.744 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:26.744 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:26.744 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:26.744 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-194bdff8-d349-2311-3797-afff4d7bbf9d -13:26.744 [INFO] TestAgent.server: member joined, marking health alive: member=Node-194bdff8-d349-2311-3797-afff4d7bbf9d -13:26.745 [INFO] TestAgent.server: New leader elected: payload=Node-194bdff8-d349-2311-3797-afff4d7bbf9d -13:26.745 [INFO] TestAgent.server: federation state anti-entropy synced -13:26.922 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:26.922 [INFO] TestAgent: Synced node info -13:27.153 [WARN] TestAgent: check has interval below minimum: check=tls minimum_interval=1s -13:28.058 [DEBUG] TestAgent: Check status updated: check=tls status=passing -13:28.077 [INFO] TestAgent: Requesting shutdown -13:28.078 [INFO] TestAgent.server: shutting down server -13:28.078 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:28.078 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:28.078 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:28.078 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:28.078 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:28.078 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:28.078 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:28.078 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:28.078 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:28.078 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:28.078 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:28.078Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:28.078Z [INFO] agent.router.manager: shutting down -13:28.078 [INFO] TestAgent: consul server down -13:28.078 [INFO] TestAgent: shutdown complete -13:28.078 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16422 network=tcp -13:28.078 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16422 network=udp -13:28.078 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16437 network=tcp -13:28.591 [INFO] TestAgent: Waiting for endpoints to shut down -13:28.591 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_HTTPCheck_TLSSkipVerify (1.97s) -=== CONT TestAgent_TokenStore -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:27.592Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:27.682Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:27.684 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6f277c5a-0fa0-f133-0716-62260f5d87b1 Address:127.0.0.1:16483}]" -13:27.684 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6f277c5a-0fa0-f133-0716-62260f5d87b1.dc1 127.0.0.1 -13:27.685 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6f277c5a-0fa0-f133-0716-62260f5d87b1 127.0.0.1 -2021-10-31T23:13:27.685Z [INFO] agent.router: Initializing LAN area manager -13:27.685 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16483 [Follower]" leader= -13:27.686 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6f277c5a-0fa0-f133-0716-62260f5d87b1.dc1 area=wan -13:27.686 [INFO] TestAgent.server: Adding LAN server: server="Node-6f277c5a-0fa0-f133-0716-62260f5d87b1 (Addr: tcp/127.0.0.1:16483) (DC: dc1)" -13:27.686 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16366 network=udp -13:27.694 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16366 network=tcp -13:27.694 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16479 network=tcp -13:27.695 [INFO] TestAgent: started state syncer -13:27.695 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16456 network=tcp -13:27.754 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:27.754 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16483 [Candidate]" term=2 -13:27.770 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:27.770 [DEBUG] TestAgent.server.raft: vote granted: from=6f277c5a-0fa0-f133-0716-62260f5d87b1 term=2 tally=1 -13:27.770 [INFO] TestAgent.server.raft: election won: tally=1 -13:27.770 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16483 [Leader]" -13:27.770 [INFO] TestAgent.server: cluster leadership acquired -13:27.770 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16483 -13:27.771 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:27.773 [INFO] TestAgent.server: New leader elected: payload=Node-6f277c5a-0fa0-f133-0716-62260f5d87b1 -13:27.789 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:27.789 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:27.789 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:27.789 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:27.789 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:27.789 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6f277c5a-0fa0-f133-0716-62260f5d87b1 -13:27.789 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6f277c5a-0fa0-f133-0716-62260f5d87b1 -13:27.812 [INFO] TestAgent.server: federation state anti-entropy synced -13:28.117 [INFO] TestAgent: Requesting shutdown -13:28.118 [INFO] TestAgent.server: shutting down server -13:28.118 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:28.118 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:28.118 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:28.118 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:28.118 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:28.118 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:28.118 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:28.118 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:28.118 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:28.118 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:28.118 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:28.118 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:28.118Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:28.118Z [INFO] agent.router.manager: shutting down -13:28.118 [INFO] TestAgent: consul server down -13:28.118 [INFO] TestAgent: shutdown complete -13:28.118 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16366 network=tcp -13:28.118 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16366 network=udp -13:28.118 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16479 network=tcp -13:28.621 [INFO] TestAgent: Waiting for endpoints to shut down -13:28.621 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_ExecRemoteDisable (1.07s) -=== CONT TestAgent_RPCPing -2021-10-31T23:13:27.771Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:27.825Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:27.827 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c39df437-a003-fcc4-6b4e-abf8499cc720 Address:127.0.0.1:16462}]" -13:27.828 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c39df437-a003-fcc4-6b4e-abf8499cc720.dc1 127.0.0.1 -13:27.829 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c39df437-a003-fcc4-6b4e-abf8499cc720 127.0.0.1 -2021-10-31T23:13:27.829Z [INFO] agent.router: Initializing LAN area manager -13:27.829 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16457 network=udp -13:27.829 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16462 [Follower]" leader= -13:27.829 [INFO] TestAgent.server: Adding LAN server: server="Node-c39df437-a003-fcc4-6b4e-abf8499cc720 (Addr: tcp/127.0.0.1:16462) (DC: dc1)" -13:27.829 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c39df437-a003-fcc4-6b4e-abf8499cc720.dc1 area=wan -13:27.830 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16457 network=tcp -13:27.830 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16458 network=tcp -13:27.830 [INFO] TestAgent: started state syncer -13:27.830 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16484 network=tcp -13:27.889 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:27.889 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16462 [Candidate]" term=2 -13:27.890 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:27.890 [DEBUG] TestAgent.server.raft: vote granted: from=c39df437-a003-fcc4-6b4e-abf8499cc720 term=2 tally=1 -13:27.890 [INFO] TestAgent.server.raft: election won: tally=1 -13:27.890 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16462 [Leader]" -13:27.890 [INFO] TestAgent.server: cluster leadership acquired -13:27.890 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16462 -13:27.891 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:27.892 [INFO] TestAgent.server: New leader elected: payload=Node-c39df437-a003-fcc4-6b4e-abf8499cc720 -13:27.898 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:27.898 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:27.898 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:27.898 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:27.898 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:27.898 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c39df437-a003-fcc4-6b4e-abf8499cc720 -13:27.898 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c39df437-a003-fcc4-6b4e-abf8499cc720 -13:27.899 [INFO] TestAgent.server: federation state anti-entropy synced -13:28.179 [INFO] TestAgent: Requesting shutdown -13:28.179 [INFO] TestAgent.server: shutting down server -13:28.179 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:28.179 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:28.179 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:28.179 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:28.179 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:28.179 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:28.179 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:28.179 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:28.180 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:28.180 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:28.180 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:28.180 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:28.180Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:28.180Z [INFO] agent.router.manager: shutting down -13:28.180 [INFO] TestAgent: consul server down -13:28.180 [INFO] TestAgent: shutdown complete -13:28.180 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16457 network=tcp -13:28.180 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16457 network=udp -13:28.180 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16458 network=tcp -13:28.682 [INFO] TestAgent: Waiting for endpoints to shut down -13:28.682 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_ExecDisable (1.04s) -=== CONT TestAgent_StartStop -2021-10-31T23:13:27.979Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:27.979Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2021-10-31T23:13:28.058Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:28.058Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -13:28.073 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:45dfe295-5996-654e-c3f2-31fdd8fc92db Address:127.0.0.1:16476}]" -13:28.074 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16476 [Follower]" leader= -13:28.075 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-45dfe295-5996-654e-c3f2-31fdd8fc92db.dc1 127.0.0.1 -13:28.075 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-45dfe295-5996-654e-c3f2-31fdd8fc92db 127.0.0.1 -2021-10-31T23:13:28.075Z [INFO] agent.router: Initializing LAN area manager -13:28.076 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16492 network=udp -13:28.076 [INFO] TestAgent.server: Adding LAN server: server="Node-45dfe295-5996-654e-c3f2-31fdd8fc92db (Addr: tcp/127.0.0.1:16476) (DC: dc1)" -13:28.076 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-45dfe295-5996-654e-c3f2-31fdd8fc92db.dc1 area=wan -13:28.076 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16492 network=tcp -13:28.086 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16472 network=tcp -13:28.086 [INFO] TestAgent: started state syncer -13:28.087 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16407 network=tcp -13:28.129 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:28.129 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16476 [Candidate]" term=2 -13:28.130 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:28.130 [DEBUG] TestAgent.server.raft: vote granted: from=45dfe295-5996-654e-c3f2-31fdd8fc92db term=2 tally=1 -13:28.130 [INFO] TestAgent.server.raft: election won: tally=1 -13:28.130 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16476 [Leader]" -13:28.130 [INFO] TestAgent.server: cluster leadership acquired -13:28.130 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16476 -13:28.131 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:28.132 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:28.132 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:28.132 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:28.132 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:28.132 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:28.132 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-45dfe295-5996-654e-c3f2-31fdd8fc92db -13:28.133 [INFO] TestAgent.server: member joined, marking health alive: member=Node-45dfe295-5996-654e-c3f2-31fdd8fc92db -13:28.133 [INFO] TestAgent.server: New leader elected: payload=Node-45dfe295-5996-654e-c3f2-31fdd8fc92db -13:28.133 [INFO] TestAgent.server: federation state anti-entropy synced -13:28.193 [WARN] TestAgent: check has interval below minimum: check=mem minimum_interval=1s -13:28.194 [INFO] TestAgent: Requesting shutdown -13:28.194 [INFO] TestAgent.server: shutting down server -13:28.194 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:28.194 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:28.194 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:28.194 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:28.194 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:28.194 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:28.194 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:28.194 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:28.194 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:28.194 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:28.194 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:28.194 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:28.195Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:28.195Z [INFO] agent.router.manager: shutting down -13:28.195 [INFO] TestAgent: consul server down -13:28.195 [INFO] TestAgent: shutdown complete -13:28.195 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16492 network=tcp -13:28.195 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16492 network=udp -13:28.195 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16472 network=tcp -13:28.698 [INFO] TestAgent: Waiting for endpoints to shut down -13:28.698 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_MinInterval (0.79s) -=== CONT TestAgent_Services_ExposeConfig -2021-10-31T23:13:28.246Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:28.246Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2021-10-31T23:13:28.289Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:28.289Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -13:28.292 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9f612285-dc57-a56e-640f-7a9f99c318f8 Address:127.0.0.1:16371}]" -13:28.293 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9f612285-dc57-a56e-640f-7a9f99c318f8.dc1 127.0.0.1 -13:28.293 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16371 [Follower]" leader= -13:28.294 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9f612285-dc57-a56e-640f-7a9f99c318f8 127.0.0.1 -2021-10-31T23:13:28.294Z [INFO] agent.router: Initializing LAN area manager -13:28.294 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16408 network=udp -13:28.294 [INFO] TestAgent.server: Adding LAN server: server="Node-9f612285-dc57-a56e-640f-7a9f99c318f8 (Addr: tcp/127.0.0.1:16371) (DC: dc1)" -13:28.294 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9f612285-dc57-a56e-640f-7a9f99c318f8.dc1 area=wan -13:28.295 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16408 network=tcp -13:28.295 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16367 network=tcp -13:28.295 [INFO] TestAgent: started state syncer -13:28.295 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16386 network=tcp -13:28.347 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:28.347 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16371 [Candidate]" term=2 -13:28.347 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:28.347 [DEBUG] TestAgent.server.raft: vote granted: from=9f612285-dc57-a56e-640f-7a9f99c318f8 term=2 tally=1 -13:28.347 [INFO] TestAgent.server.raft: election won: tally=1 -13:28.347 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16371 [Leader]" -13:28.347 [INFO] TestAgent.server: cluster leadership acquired -13:28.347 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16371 -13:28.348 [INFO] TestAgent.server: New leader elected: payload=Node-9f612285-dc57-a56e-640f-7a9f99c318f8 -13:28.348 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:28.350 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:28.350 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:28.350 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:28.350 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:28.350 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:28.350 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9f612285-dc57-a56e-640f-7a9f99c318f8 -13:28.350 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9f612285-dc57-a56e-640f-7a9f99c318f8 -13:28.351 [INFO] TestAgent: Requesting shutdown -13:28.351 [INFO] TestAgent.server: shutting down server -13:28.351 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:28.351 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:28.351 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:28.351 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:28.351 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:28.351 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:28.351 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:28.351 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:28.351 [INFO] TestAgent.server: federation state anti-entropy synced -13:28.351 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:28.352 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:28.352 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:28.352Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:28.352Z [INFO] agent.router.manager: shutting down -13:28.352 [INFO] TestAgent: consul server down -13:28.352 [INFO] TestAgent: shutdown complete -13:28.352 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16408 network=tcp -13:28.352 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16408 network=udp -13:28.352 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16367 network=tcp -13:28.853 [INFO] TestAgent: Waiting for endpoints to shut down -13:28.853 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck (0.68s) -=== CONT TestAgent_HostBadACL -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:28.239Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:28.239Z [WARN] agent: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -2021-10-31T23:13:28.279Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:28.279Z [WARN] agent.auto_config: using enable-script-checks without ACLs and without allow_write_http_from is DANGEROUS, use enable-local-script-checks instead, see https://www.hashicorp.com/blog/protecting-consul-from-rce-risk-in-specific-configurations/ -13:28.286 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ed496a92-97b2-be5c-7849-cecef3ad6e86 Address:127.0.0.1:16497}]" -13:28.287 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ed496a92-97b2-be5c-7849-cecef3ad6e86.dc1 127.0.0.1 -13:28.287 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ed496a92-97b2-be5c-7849-cecef3ad6e86 127.0.0.1 -2021-10-31T23:13:28.287Z [INFO] agent.router: Initializing LAN area manager -13:28.288 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16387 network=udp -13:28.288 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16497 [Follower]" leader= -13:28.288 [INFO] TestAgent.server: Adding LAN server: server="Node-ed496a92-97b2-be5c-7849-cecef3ad6e86 (Addr: tcp/127.0.0.1:16497) (DC: dc1)" -13:28.288 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ed496a92-97b2-be5c-7849-cecef3ad6e86.dc1 area=wan -13:28.288 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16387 network=tcp -13:28.289 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16493 network=tcp -13:28.289 [INFO] TestAgent: started state syncer -13:28.289 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16470 network=tcp -13:28.354 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:28.354 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16497 [Candidate]" term=2 -13:28.355 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:28.355 [DEBUG] TestAgent.server.raft: vote granted: from=ed496a92-97b2-be5c-7849-cecef3ad6e86 term=2 tally=1 -13:28.355 [INFO] TestAgent.server.raft: election won: tally=1 -13:28.355 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16497 [Leader]" -13:28.355 [INFO] TestAgent.server: cluster leadership acquired -13:28.355 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16497 -13:28.356 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:28.357 [INFO] TestAgent.server: New leader elected: payload=Node-ed496a92-97b2-be5c-7849-cecef3ad6e86 -13:28.358 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:28.358 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:28.358 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:28.358 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:28.358 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:28.358 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ed496a92-97b2-be5c-7849-cecef3ad6e86 -13:28.358 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ed496a92-97b2-be5c-7849-cecef3ad6e86 -13:28.359 [INFO] TestAgent.server: federation state anti-entropy synced -13:28.535 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:28.536 [INFO] TestAgent: Synced node info -13:28.546 [INFO] TestAgent: Requesting shutdown -13:28.546 [INFO] TestAgent.server: shutting down server -13:28.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:28.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:28.546 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:28.546 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:28.546 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:28.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:28.546 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:28.546 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:28.546 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:28.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:28.546 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:28.546Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:28.546Z [INFO] agent.router.manager: shutting down -13:28.546 [INFO] TestAgent: consul server down -13:28.546 [INFO] TestAgent: shutdown complete -13:28.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16387 network=tcp -13:28.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16387 network=udp -13:28.546 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16493 network=tcp -13:29.055 [INFO] TestAgent: Waiting for endpoints to shut down -13:29.055 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_AddCheck_StartPassing (0.88s) -=== CONT TestAgent_Host -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:28.613Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:28.672Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:28.683 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2ca774aa-ed2d-5ba2-5800-7e3b52e66eeb Address:127.0.0.1:16490}]" -13:28.684 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2ca774aa-ed2d-5ba2-5800-7e3b52e66eeb.dc1 127.0.0.1 -13:28.685 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2ca774aa-ed2d-5ba2-5800-7e3b52e66eeb 127.0.0.1 -2021-10-31T23:13:28.685Z [INFO] agent.router: Initializing LAN area manager -13:28.685 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16555 network=udp -13:28.687 [INFO] TestAgent.server: Adding LAN server: server="Node-2ca774aa-ed2d-5ba2-5800-7e3b52e66eeb (Addr: tcp/127.0.0.1:16490) (DC: dc1)" -13:28.687 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2ca774aa-ed2d-5ba2-5800-7e3b52e66eeb.dc1 area=wan -13:28.687 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16555 network=tcp -13:28.688 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16486 network=tcp -13:28.688 [INFO] TestAgent: started state syncer -13:28.688 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16512 network=tcp -13:28.689 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16490 [Follower]" leader= -13:28.732 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:28.732 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16490 [Candidate]" term=2 -13:28.733 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:28.733 [DEBUG] TestAgent.server.raft: vote granted: from=2ca774aa-ed2d-5ba2-5800-7e3b52e66eeb term=2 tally=1 -13:28.733 [INFO] TestAgent.server.raft: election won: tally=1 -13:28.733 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16490 [Leader]" -13:28.733 [INFO] TestAgent.server: cluster leadership acquired -13:28.733 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16490 -13:28.733 [INFO] TestAgent.server: New leader elected: payload=Node-2ca774aa-ed2d-5ba2-5800-7e3b52e66eeb -13:28.747 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:28.749 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:28.749 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:28.749 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:28.749 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:28.749 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:28.749 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2ca774aa-ed2d-5ba2-5800-7e3b52e66eeb -13:28.749 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2ca774aa-ed2d-5ba2-5800-7e3b52e66eeb -13:28.753 [INFO] TestAgent.server: federation state anti-entropy synced -13:28.785 [INFO] TestAgent: Requesting shutdown -13:28.786 [INFO] TestAgent.server: shutting down server -13:28.786 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:28.786 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:28.786 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:28.786 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:28.786 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:28.786 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:28.786 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:28.786 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:28.786 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:28.786 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:28.786 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:28.786 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:28.786Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:28.786Z [INFO] agent.router.manager: shutting down -13:28.786 [INFO] TestAgent: consul server down -13:28.786 [INFO] TestAgent: shutdown complete -13:28.786 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16555 network=tcp -13:28.786 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16555 network=udp -13:28.786 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16486 network=tcp -13:29.289 [INFO] TestAgent: Waiting for endpoints to shut down -13:29.289 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_TokenStore (0.70s) -=== CONT TestAgentConnectAuthorize_defaultAllow -2021-10-31T23:13:28.519Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:28.604Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:28.607 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8d85792c-cbf6-4db9-4466-64dcbad7b8fd Address:127.0.0.1:16518}]" -13:28.608 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8d85792c-cbf6-4db9-4466-64dcbad7b8fd 127.0.0.1 -2021-10-31T23:13:28.608Z [INFO] agent.router: Initializing LAN area manager -13:28.608 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16518 [Follower]" leader= -13:28.608 [INFO] TestAgent.server: Adding LAN server: server="Node-8d85792c-cbf6-4db9-4466-64dcbad7b8fd (Addr: tcp/127.0.0.1:16518) (DC: dc1)" -13:28.608 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16548 network=tcp -13:28.608 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16548 network=udp -13:28.609 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16514 network=tcp -13:28.609 [INFO] TestAgent: started state syncer -13:28.609 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16498 network=tcp -13:28.678 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:28.678 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16518 [Candidate]" term=2 -13:28.678 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:28.678 [DEBUG] TestAgent.server.raft: vote granted: from=8d85792c-cbf6-4db9-4466-64dcbad7b8fd term=2 tally=1 -13:28.678 [INFO] TestAgent.server.raft: election won: tally=1 -13:28.678 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16518 [Leader]" -13:28.678 [INFO] TestAgent.server: cluster leadership acquired -13:28.678 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16518 -13:28.679 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:28.680 [INFO] TestAgent.server: New leader elected: payload=Node-8d85792c-cbf6-4db9-4466-64dcbad7b8fd -13:28.681 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:28.681 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:28.681 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:28.681 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:28.681 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:28.681 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8d85792c-cbf6-4db9-4466-64dcbad7b8fd -13:28.681 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8d85792c-cbf6-4db9-4466-64dcbad7b8fd -13:28.682 [INFO] TestAgent.server: federation state anti-entropy synced -13:28.817 [INFO] TestAgent: Requesting shutdown -13:28.817 [INFO] TestAgent.server: shutting down server -13:28.817 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:28.817 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:28.817 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:28.817 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:28.817 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:28.817 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:28.817 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:28.817 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:28.817 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:28.817 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:28.817Z [INFO] agent.router.manager: shutting down -13:28.817 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:28.818 [INFO] TestAgent: consul server down -13:28.818 [INFO] TestAgent: shutdown complete -13:28.818 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16548 network=tcp -13:28.818 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16548 network=udp -13:28.818 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16514 network=tcp -13:29.321 [INFO] TestAgent: Waiting for endpoints to shut down -13:29.321 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ReconnectConfigWanDisabled (0.85s) -=== CONT TestAgentConnectAuthorize_defaultDeny -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=1)_compressed=true -2021-10-31T23:13:28.644Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:28.702Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:28.705 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c569f1f0-2cca-87d4-1e34-6369940aff5b Address:127.0.0.1:16546}]" -13:28.705 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16546 [Follower]" leader= -13:28.710 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c569f1f0-2cca-87d4-1e34-6369940aff5b.dc1 127.0.0.1 -13:28.711 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c569f1f0-2cca-87d4-1e34-6369940aff5b 127.0.0.1 -2021-10-31T23:13:28.711Z [INFO] agent.router: Initializing LAN area manager -13:28.711 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16513 network=udp -13:28.711 [INFO] TestAgent.server: Adding LAN server: server="Node-c569f1f0-2cca-87d4-1e34-6369940aff5b (Addr: tcp/127.0.0.1:16546) (DC: dc1)" -13:28.711 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c569f1f0-2cca-87d4-1e34-6369940aff5b.dc1 area=wan -13:28.711 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16513 network=tcp -13:28.712 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16542 network=tcp -13:28.712 [INFO] TestAgent: started state syncer -13:28.713 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16533 network=tcp -13:28.767 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:28.767 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16546 [Candidate]" term=2 -13:28.767 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:28.767 [DEBUG] TestAgent.server.raft: vote granted: from=c569f1f0-2cca-87d4-1e34-6369940aff5b term=2 tally=1 -13:28.767 [INFO] TestAgent.server.raft: election won: tally=1 -13:28.767 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16546 [Leader]" -13:28.767 [INFO] TestAgent.server: cluster leadership acquired -13:28.782 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16546 -13:28.783 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:28.783 [INFO] TestAgent.server: New leader elected: payload=Node-c569f1f0-2cca-87d4-1e34-6369940aff5b -13:28.785 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:28.785 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:28.785 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:28.785 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:28.785 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:28.785 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c569f1f0-2cca-87d4-1e34-6369940aff5b -13:28.785 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c569f1f0-2cca-87d4-1e34-6369940aff5b -13:28.794 [INFO] TestAgent.server: federation state anti-entropy synced -13:28.974 [INFO] TestAgent: Requesting shutdown -13:28.974 [INFO] TestAgent.server: shutting down server -13:28.974 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:28.974 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:28.974 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:28.974 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:28.974 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:28.974 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:28.974 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:28.975 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:28.975 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:28.975 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:28.975 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:28.975 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:28.975Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:28.975Z [INFO] agent.router.manager: shutting down -13:28.975 [INFO] TestAgent: consul server down -13:28.975 [INFO] TestAgent: shutdown complete -13:28.975 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16513 network=tcp -13:28.975 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16513 network=udp -13:28.975 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16542 network=tcp -13:29.490 [INFO] TestAgent: Waiting for endpoints to shut down -13:29.490 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_RPCPing (0.87s) -=== CONT TestAgentConnectAuthorize_serviceWrite -2021-10-31T23:13:28.782Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:28.813Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:28.824 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:41849c35-5686-855f-bc58-cebb5712aa76 Address:127.0.0.1:16588}]" -13:28.824 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-41849c35-5686-855f-bc58-cebb5712aa76.dc1 127.0.0.1 -13:28.825 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-41849c35-5686-855f-bc58-cebb5712aa76 127.0.0.1 -2021-10-31T23:13:28.825Z [INFO] agent.router: Initializing LAN area manager -13:28.825 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16604 network=udp -13:28.825 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16588 [Follower]" leader= -13:28.825 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-41849c35-5686-855f-bc58-cebb5712aa76.dc1 area=wan -13:28.825 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16604 network=tcp -13:28.826 [INFO] TestAgent.server: Adding LAN server: server="Node-41849c35-5686-855f-bc58-cebb5712aa76 (Addr: tcp/127.0.0.1:16588) (DC: dc1)" -13:28.826 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16584 network=tcp -13:28.826 [INFO] TestAgent: started state syncer -13:28.826 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16589 network=tcp -13:28.881 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:28.881 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16588 [Candidate]" term=2 -13:28.882 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:28.882 [DEBUG] TestAgent.server.raft: vote granted: from=41849c35-5686-855f-bc58-cebb5712aa76 term=2 tally=1 -13:28.882 [INFO] TestAgent.server.raft: election won: tally=1 -13:28.882 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16588 [Leader]" -13:28.882 [INFO] TestAgent.server: cluster leadership acquired -13:28.882 [INFO] TestAgent.server: New leader elected: payload=Node-41849c35-5686-855f-bc58-cebb5712aa76 -13:28.882 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16588 -13:28.883 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:28.886 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:28.886 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:28.886 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:28.886 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:28.886 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:28.886 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-41849c35-5686-855f-bc58-cebb5712aa76 -13:28.886 [INFO] TestAgent.server: member joined, marking health alive: member=Node-41849c35-5686-855f-bc58-cebb5712aa76 -13:28.887 [INFO] TestAgent.server: federation state anti-entropy synced -13:28.990 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:28.990 [INFO] TestAgent: Synced node info -13:29.086 [INFO] TestAgent: Requesting shutdown -13:29.086 [INFO] TestAgent.server: shutting down server -13:29.086 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:29.086 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:29.086 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:29.086 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:29.086 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:29.087 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:29.087 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:29.087 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:29.087 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:29.087 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:29.087 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:29.087Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:29.087Z [INFO] agent.router.manager: shutting down -13:29.087 [INFO] TestAgent: consul server down -13:29.087 [INFO] TestAgent: shutdown complete -13:29.087 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16604 network=tcp -13:29.087 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16604 network=udp -13:29.087 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16584 network=tcp -13:29.588 [INFO] TestAgent: Waiting for endpoints to shut down -13:29.588 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Services_ExposeConfig (0.88s) -=== CONT TestAgentConnectAuthorize_denyWildcard -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=1)_compressed=false -2021-10-31T23:13:28.904Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:28.904Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:28.969Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:28.969Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:28.977 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9b35981e-183c-678b-c5a1-5b7cf83ff411 Address:127.0.0.1:16539}]" -13:28.977 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16539 [Follower]" leader= -13:28.978 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9b35981e-183c-678b-c5a1-5b7cf83ff411.dc1 127.0.0.1 -13:28.978 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9b35981e-183c-678b-c5a1-5b7cf83ff411 127.0.0.1 -2021-10-31T23:13:28.978Z [INFO] agent.router: Initializing LAN area manager -13:28.978 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16590 network=udp -13:28.979 [INFO] TestAgent.server: Adding LAN server: server="Node-9b35981e-183c-678b-c5a1-5b7cf83ff411 (Addr: tcp/127.0.0.1:16539) (DC: dc1)" -13:28.979 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9b35981e-183c-678b-c5a1-5b7cf83ff411.dc1 area=wan -13:28.979 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16590 network=tcp -13:28.979 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16535 network=tcp -13:28.979 [INFO] TestAgent: started state syncer -13:28.980 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16596 network=tcp -13:29.030 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:29.030 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16539 [Candidate]" term=2 -13:29.030 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:29.030 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:29.030 [DEBUG] TestAgent.server.raft: vote granted: from=9b35981e-183c-678b-c5a1-5b7cf83ff411 term=2 tally=1 -13:29.030 [INFO] TestAgent.server.raft: election won: tally=1 -13:29.030 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16539 [Leader]" -13:29.030 [INFO] TestAgent.server: cluster leadership acquired -13:29.031 [INFO] TestAgent.server: initializing acls -13:29.031 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:29.031 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:29.032 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:29.032 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:29.032 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:29.032 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:29.032 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9b35981e-183c-678b-c5a1-5b7cf83ff411 -13:29.033 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9b35981e-183c-678b-c5a1-5b7cf83ff411.dc1 -13:29.033 [INFO] TestAgent.server: New leader elected: payload=Node-9b35981e-183c-678b-c5a1-5b7cf83ff411 -13:29.034 [INFO] TestAgent.server: Updating LAN server: server="Node-9b35981e-183c-678b-c5a1-5b7cf83ff411 (Addr: tcp/127.0.0.1:16539) (DC: dc1)" -13:29.034 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9b35981e-183c-678b-c5a1-5b7cf83ff411.dc1 area=wan -13:29.052 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:29.060 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:29.060 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:29.060 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:29.060 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:29.060 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:29.060 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9b35981e-183c-678b-c5a1-5b7cf83ff411 -13:29.060 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9b35981e-183c-678b-c5a1-5b7cf83ff411 -13:29.061 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9b35981e-183c-678b-c5a1-5b7cf83ff411 -13:29.070 [INFO] TestAgent.server: federation state anti-entropy synced -13:29.113 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" -13:29.130 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9b35981e-183c-678b-c5a1-5b7cf83ff411 -13:29.130 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9b35981e-183c-678b-c5a1-5b7cf83ff411 -13:29.130 [INFO] TestAgent: Requesting shutdown -13:29.130 [INFO] TestAgent.server: shutting down server -13:29.130 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:29.130 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:29.130 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:29.130 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:29.130 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:29.130 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:29.130 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:29.130 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:29.130 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:29.131 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:29.131 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:29.131 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:29.131 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:29.131 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:29.131 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:29.131Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:29.131Z [INFO] agent.router.manager: shutting down -13:29.131 [INFO] TestAgent: consul server down -13:29.131 [INFO] TestAgent: shutdown complete -13:29.131 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16590 network=tcp -13:29.131 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16590 network=udp -13:29.131 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16535 network=tcp -13:29.632 [INFO] TestAgent: Waiting for endpoints to shut down -13:29.632 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_HostBadACL (0.79s) -=== CONT TestAgentConnectAuthorize_allowTrustDomain -2021-10-31T23:13:28.379Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:28.398Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:28.400 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e9ff3595-1fc5-4117-069c-ac43ce1799b4 Address:127.0.0.1:16525}]" -13:28.401 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e9ff3595-1fc5-4117-069c-ac43ce1799b4.dc1 127.0.0.1 -13:28.402 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16525 [Follower]" leader= -13:28.410 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e9ff3595-1fc5-4117-069c-ac43ce1799b4 127.0.0.1 -2021-10-31T23:13:28.410Z [INFO] agent.router: Initializing LAN area manager -13:28.411 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16471 network=udp -13:28.411 [INFO] TestAgent.server: Adding LAN server: server="Node-e9ff3595-1fc5-4117-069c-ac43ce1799b4 (Addr: tcp/127.0.0.1:16525) (DC: dc1)" -13:28.411 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16471 network=tcp -13:28.411 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e9ff3595-1fc5-4117-069c-ac43ce1799b4.dc1 area=wan -13:28.412 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16521 network=tcp -13:28.412 [INFO] TestAgent: started state syncer -13:28.412 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16547 network=tcp -13:28.450 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:28.450 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16525 [Candidate]" term=2 -13:28.450 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:28.450 [DEBUG] TestAgent.server.raft: vote granted: from=e9ff3595-1fc5-4117-069c-ac43ce1799b4 term=2 tally=1 -13:28.450 [INFO] TestAgent.server.raft: election won: tally=1 -13:28.450 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16525 [Leader]" -13:28.450 [INFO] TestAgent.server: cluster leadership acquired -13:28.450 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16525 -13:28.451 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:28.452 [INFO] TestAgent.server: New leader elected: payload=Node-e9ff3595-1fc5-4117-069c-ac43ce1799b4 -13:28.453 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:28.453 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:28.453 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:28.453 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:28.453 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:28.453 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e9ff3595-1fc5-4117-069c-ac43ce1799b4 -13:28.453 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e9ff3595-1fc5-4117-069c-ac43ce1799b4 -13:28.458 [INFO] TestAgent.server: federation state anti-entropy synced -13:28.722 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:28.723 [INFO] TestAgent: Synced node info -13:28.724 [INFO] TestAgent: Synced service: service=redis -13:28.724 [INFO] TestAgent: Synced check: check=node-check -13:28.724 [DEBUG] TestAgent: Check in sync: check=redis-check -13:28.753 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:28.753 [DEBUG] TestAgent: Node info in sync -13:28.753 [DEBUG] TestAgent: Service in sync: service=redis -13:28.753 [DEBUG] TestAgent: Check in sync: check=redis-check -13:28.753 [DEBUG] TestAgent: Check in sync: check=node-check -13:28.753 [DEBUG] TestAgent: Node info in sync -13:28.753 [DEBUG] TestAgent: Service in sync: service=redis -13:28.753 [DEBUG] TestAgent: Check in sync: check=redis-check -13:28.753 [DEBUG] TestAgent: Check in sync: check=node-check -13:29.228 [DEBUG] TestAgent: Registered node: node="Node-level check" -13:29.228 [DEBUG] TestAgent: Registered node: node="Serf Health Status" -13:29.228 [DEBUG] TestAgent: Registered node: node="Service-level check" -13:29.228 [INFO] TestAgent: Sync in progress: iteration=1 -13:29.228 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:29.228 [DEBUG] TestAgent: Node info in sync -13:29.228 [DEBUG] TestAgent: Service in sync: service=redis -13:29.228 [DEBUG] TestAgent: Check in sync: check=redis-check -13:29.228 [DEBUG] TestAgent: Check in sync: check=node-check -13:29.228 [INFO] TestAgent: Sync in progress: iteration=2 -13:29.228 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:29.228 [DEBUG] TestAgent: Node info in sync -13:29.228 [DEBUG] TestAgent: Service in sync: service=redis -13:29.228 [DEBUG] TestAgent: Check in sync: check=redis-check -13:29.228 [DEBUG] TestAgent: Check in sync: check=node-check -13:29.228 [INFO] TestAgent: Sync in progress: iteration=3 -13:29.228 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:29.228 [DEBUG] TestAgent: Node info in sync -13:29.228 [DEBUG] TestAgent: Service in sync: service=redis -13:29.228 [DEBUG] TestAgent: Check in sync: check=redis-check -13:29.228 [DEBUG] TestAgent: Check in sync: check=node-check -13:29.228 [INFO] TestAgent: Sync in progress: iteration=4 -13:29.228 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:29.228 [DEBUG] TestAgent: Node info in sync -13:29.228 [DEBUG] TestAgent: Service in sync: service=redis -13:29.228 [DEBUG] TestAgent: Check in sync: check=redis-check -13:29.228 [DEBUG] TestAgent: Check in sync: check=node-check -13:29.228 [INFO] TestAgent: Sync in progress: iteration=5 -13:29.228 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:29.228 [DEBUG] TestAgent: Node info in sync -13:29.228 [DEBUG] TestAgent: Service in sync: service=redis -13:29.228 [DEBUG] TestAgent: Check in sync: check=redis-check -13:29.228 [DEBUG] TestAgent: Check in sync: check=node-check -13:29.228 [INFO] TestAgent: Sync in progress: iteration=6 -13:29.229 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:29.229 [DEBUG] TestAgent: Node info in sync -13:29.229 [DEBUG] TestAgent: Service in sync: service=redis -13:29.229 [DEBUG] TestAgent: Check in sync: check=redis-check -13:29.229 [DEBUG] TestAgent: Check in sync: check=node-check -13:29.229 [INFO] TestAgent: Sync in progress: iteration=7 -13:29.229 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:29.229 [DEBUG] TestAgent: Node info in sync -13:29.229 [DEBUG] TestAgent: Service in sync: service=redis -13:29.229 [DEBUG] TestAgent: Check in sync: check=node-check -13:29.229 [DEBUG] TestAgent: Check in sync: check=redis-check -13:29.229 [INFO] TestAgent: Sync in progress: iteration=8 -13:29.229 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:29.229 [DEBUG] TestAgent: Node info in sync -13:29.229 [DEBUG] TestAgent: Service in sync: service=redis -13:29.229 [DEBUG] TestAgent: Check in sync: check=redis-check -13:29.229 [DEBUG] TestAgent: Check in sync: check=node-check -13:29.229 [INFO] TestAgent: Sync in progress: iteration=9 -13:29.229 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:29.238 [DEBUG] TestAgent: Node info in sync -13:29.238 [DEBUG] TestAgent: Service in sync: service=redis -13:29.238 [DEBUG] TestAgent: Check in sync: check=redis-check -13:29.238 [DEBUG] TestAgent: Check in sync: check=node-check -13:29.238 [INFO] TestAgent: Sync in progress: iteration=10 -13:29.238 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:29.238 [DEBUG] TestAgent: Node info in sync -13:29.238 [DEBUG] TestAgent: Service in sync: service=redis -13:29.238 [DEBUG] TestAgent: Check in sync: check=redis-check -13:29.238 [DEBUG] TestAgent: Check in sync: check=node-check -13:29.238 [INFO] TestAgent: Requesting shutdown -13:29.238 [INFO] TestAgent.server: shutting down server -13:29.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:29.238 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:29.238 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:29.238 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:29.238 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:29.238 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:29.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:29.238 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:29.238 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:29.238 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:29.239 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:29.249Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:29.249Z [INFO] agent.router.manager: shutting down -13:29.250 [INFO] TestAgent: consul server down -13:29.250 [INFO] TestAgent: shutdown complete -13:29.250 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16471 network=tcp -13:29.250 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16471 network=udp -13:29.250 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16521 network=tcp -13:29.753 [INFO] TestAgent: Waiting for endpoints to shut down -13:29.753 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_IndexChurn/with_tags -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=true -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:29.115Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:29.115Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:29.162Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:29.162Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:29.175 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:63ab5e1f-c472-ca99-3bf7-e2764a65716b Address:127.0.0.1:16602}]" -13:29.176 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-63ab5e1f-c472-ca99-3bf7-e2764a65716b.dc1 127.0.0.1 -13:29.177 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-63ab5e1f-c472-ca99-3bf7-e2764a65716b 127.0.0.1 -2021-10-31T23:13:29.177Z [INFO] agent.router: Initializing LAN area manager -13:29.177 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16597 network=udp -13:29.177 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16602 [Follower]" leader= -13:29.178 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-63ab5e1f-c472-ca99-3bf7-e2764a65716b.dc1 area=wan -13:29.178 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16597 network=tcp -13:29.178 [INFO] TestAgent.server: Adding LAN server: server="Node-63ab5e1f-c472-ca99-3bf7-e2764a65716b (Addr: tcp/127.0.0.1:16602) (DC: dc1)" -13:29.178 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16598 network=tcp -13:29.179 [INFO] TestAgent: started state syncer -13:29.179 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16519 network=tcp -13:29.221 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:29.221 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16602 [Candidate]" term=2 -13:29.221 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:29.221 [DEBUG] TestAgent.server.raft: vote granted: from=63ab5e1f-c472-ca99-3bf7-e2764a65716b term=2 tally=1 -13:29.221 [INFO] TestAgent.server.raft: election won: tally=1 -13:29.221 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16602 [Leader]" -13:29.221 [INFO] TestAgent.server: cluster leadership acquired -13:29.222 [INFO] TestAgent.server: New leader elected: payload=Node-63ab5e1f-c472-ca99-3bf7-e2764a65716b -13:29.222 [INFO] TestAgent.server: initializing acls -13:29.222 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:29.222 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:29.223 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:29.223 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:29.223 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:29.223 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:29.224 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-63ab5e1f-c472-ca99-3bf7-e2764a65716b -13:29.224 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-63ab5e1f-c472-ca99-3bf7-e2764a65716b.dc1 -13:29.225 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:29.228 [INFO] TestAgent.server: Updating LAN server: server="Node-63ab5e1f-c472-ca99-3bf7-e2764a65716b (Addr: tcp/127.0.0.1:16602) (DC: dc1)" -13:29.228 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-63ab5e1f-c472-ca99-3bf7-e2764a65716b.dc1 area=wan -13:29.235 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:29.235 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:29.235 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:29.235 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:29.235 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:29.235 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-63ab5e1f-c472-ca99-3bf7-e2764a65716b -13:29.235 [INFO] TestAgent.server: member joined, marking health alive: member=Node-63ab5e1f-c472-ca99-3bf7-e2764a65716b -13:29.236 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-63ab5e1f-c472-ca99-3bf7-e2764a65716b -13:29.237 [INFO] TestAgent.server: federation state anti-entropy synced -13:29.395 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" -13:29.427 [INFO] TestAgent: Requesting shutdown -13:29.427 [INFO] TestAgent.server: shutting down server -13:29.427 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:29.427 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:29.427 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:29.427 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:29.427 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:29.427 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:29.427 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:29.427 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:29.427 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:29.427 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:29.427 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:29.427 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:29.427 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:29.427 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:29.427 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:29.428Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:29.428Z [INFO] agent.router.manager: shutting down -13:29.428 [INFO] TestAgent: consul server down -13:29.428 [INFO] TestAgent: shutdown complete -13:29.428 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16597 network=tcp -13:29.428 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16597 network=udp -13:29.428 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16598 network=tcp -13:29.929 [INFO] TestAgent: Waiting for endpoints to shut down -13:29.929 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Host (0.87s) -=== CONT TestAgentConnectAuthorize_deny -2021-10-31T23:13:29.309Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:29.309Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:29.324Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:29.324Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:29.326 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e22d978b-8481-ab0a-64c4-353245832fe2 Address:127.0.0.1:16567}]" -13:29.327 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16567 [Follower]" leader= -13:29.327 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e22d978b-8481-ab0a-64c4-353245832fe2.dc1 127.0.0.1 -13:29.328 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e22d978b-8481-ab0a-64c4-353245832fe2 127.0.0.1 -2021-10-31T23:13:29.328Z [INFO] agent.router: Initializing LAN area manager -13:29.328 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16541 network=udp -13:29.328 [INFO] TestAgent.server: Adding LAN server: server="Node-e22d978b-8481-ab0a-64c4-353245832fe2 (Addr: tcp/127.0.0.1:16567) (DC: dc1)" -13:29.328 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e22d978b-8481-ab0a-64c4-353245832fe2.dc1 area=wan -13:29.328 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16541 network=tcp -13:29.329 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16563 network=tcp -13:29.329 [INFO] TestAgent: started state syncer -13:29.329 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16526 network=tcp -13:29.380 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:29.396 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:29.396 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16567 [Candidate]" term=2 -13:29.396 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:29.396 [DEBUG] TestAgent.server.raft: vote granted: from=e22d978b-8481-ab0a-64c4-353245832fe2 term=2 tally=1 -13:29.396 [INFO] TestAgent.server.raft: election won: tally=1 -13:29.396 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16567 [Leader]" -13:29.396 [INFO] TestAgent.server: cluster leadership acquired -13:29.396 [INFO] TestAgent.server: New leader elected: payload=Node-e22d978b-8481-ab0a-64c4-353245832fe2 -13:29.402 [INFO] TestAgent.server: initializing acls -13:29.402 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:29.402 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:29.403 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:29.403 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:29.403 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:29.403 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:29.404 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e22d978b-8481-ab0a-64c4-353245832fe2 -13:29.404 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e22d978b-8481-ab0a-64c4-353245832fe2.dc1 -13:29.404 [INFO] TestAgent.server: Updating LAN server: server="Node-e22d978b-8481-ab0a-64c4-353245832fe2 (Addr: tcp/127.0.0.1:16567) (DC: dc1)" -13:29.404 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e22d978b-8481-ab0a-64c4-353245832fe2.dc1 area=wan -13:29.405 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:29.415 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:29.415 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:29.415 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:29.415 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:29.415 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:29.415 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e22d978b-8481-ab0a-64c4-353245832fe2 -13:29.415 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e22d978b-8481-ab0a-64c4-353245832fe2 -13:29.417 [INFO] TestAgent.server: federation state anti-entropy synced -13:29.417 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e22d978b-8481-ab0a-64c4-353245832fe2 -13:29.439 [INFO] TestAgent: Requesting shutdown -13:29.439 [INFO] TestAgent.server: shutting down server -13:29.439 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:29.439 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:29.439 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:29.439 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:29.439 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:29.439 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:29.439 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:29.439 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:29.439 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:29.439 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:29.439 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:29.439 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:29.439 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:29.439 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:29.439 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:29.439 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:29.440Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:29.440Z [INFO] agent.router.manager: shutting down -13:29.440 [INFO] TestAgent: consul server down -13:29.440 [INFO] TestAgent: shutdown complete -13:29.440 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16541 network=tcp -13:29.440 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16541 network=udp -13:29.440 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16563 network=tcp -13:29.940 [INFO] TestAgent: Waiting for endpoints to shut down -13:29.940 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_defaultAllow (0.65s) -=== CONT TestAgentConnectAuthorize_allow -2021-10-31T23:13:29.376Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:29.376Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:29.415Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:29.415Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:29.427 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d7de7030-b4b6-652a-af59-443957cf161d Address:127.0.0.1:16560}]" -13:29.428 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d7de7030-b4b6-652a-af59-443957cf161d.dc1 127.0.0.1 -13:29.428 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16560 [Follower]" leader= -13:29.429 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d7de7030-b4b6-652a-af59-443957cf161d 127.0.0.1 -2021-10-31T23:13:29.429Z [INFO] agent.router: Initializing LAN area manager -13:29.429 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d7de7030-b4b6-652a-af59-443957cf161d.dc1 area=wan -13:29.429 [INFO] TestAgent.server: Adding LAN server: server="Node-d7de7030-b4b6-652a-af59-443957cf161d (Addr: tcp/127.0.0.1:16560) (DC: dc1)" -13:29.429 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16527 network=udp -13:29.429 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16527 network=tcp -13:29.439 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16556 network=tcp -13:29.439 [INFO] TestAgent: started state syncer -13:29.439 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16610 network=tcp -13:29.486 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:29.498 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:29.498 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16560 [Candidate]" term=2 -13:29.499 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:29.499 [DEBUG] TestAgent.server.raft: vote granted: from=d7de7030-b4b6-652a-af59-443957cf161d term=2 tally=1 -13:29.499 [INFO] TestAgent.server.raft: election won: tally=1 -13:29.499 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16560 [Leader]" -13:29.499 [INFO] TestAgent.server: cluster leadership acquired -13:29.499 [INFO] TestAgent.server: initializing acls -13:29.499 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:29.499 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:29.500 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:29.500 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:29.500 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:29.500 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:29.500 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d7de7030-b4b6-652a-af59-443957cf161d -13:29.500 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d7de7030-b4b6-652a-af59-443957cf161d.dc1 -13:29.501 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:29.501 [INFO] TestAgent.server: New leader elected: payload=Node-d7de7030-b4b6-652a-af59-443957cf161d -13:29.501 [INFO] TestAgent.server: Updating LAN server: server="Node-d7de7030-b4b6-652a-af59-443957cf161d (Addr: tcp/127.0.0.1:16560) (DC: dc1)" -13:29.501 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d7de7030-b4b6-652a-af59-443957cf161d.dc1 area=wan -13:29.510 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:29.510 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:29.510 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:29.510 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:29.510 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:29.510 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d7de7030-b4b6-652a-af59-443957cf161d -13:29.510 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d7de7030-b4b6-652a-af59-443957cf161d -13:29.511 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d7de7030-b4b6-652a-af59-443957cf161d -13:29.511 [INFO] TestAgent.server: federation state anti-entropy synced -13:29.538 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d7de7030-b4b6-652a-af59-443957cf161d -13:29.538 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d7de7030-b4b6-652a-af59-443957cf161d -13:29.539 [INFO] TestAgent: Requesting shutdown -13:29.539 [INFO] TestAgent.server: shutting down server -13:29.539 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:29.539 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:29.539 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:29.539 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:29.539 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:29.539 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:29.539 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:29.539 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:29.539 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:29.539 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:29.539 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:29.539 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:29.539 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:29.539 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:29.539 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:29.539 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:29.539Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:29.539Z [INFO] agent.router.manager: shutting down -13:29.539 [INFO] TestAgent: consul server down -13:29.539 [INFO] TestAgent: shutdown complete -13:29.539 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16527 network=tcp -13:29.539 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16527 network=udp -13:29.539 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16556 network=tcp -13:30.042 [INFO] TestAgent: Waiting for endpoints to shut down -13:30.042 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_defaultDeny (0.72s) -=== CONT TestAgentConnectAuthorize_idNotService -2021-10-31T23:13:29.251Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:29.321Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:29.331 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bd583525-973d-b7b6-b11a-56d108f751aa Address:127.0.0.1:16511}]" -13:29.331 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bd583525-973d-b7b6-b11a-56d108f751aa.dc1 127.0.0.1 -13:29.331 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16511 [Follower]" leader= -13:29.332 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bd583525-973d-b7b6-b11a-56d108f751aa 127.0.0.1 -2021-10-31T23:13:29.332Z [INFO] agent.router: Initializing LAN area manager -13:29.332 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16520 network=udp -13:29.332 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bd583525-973d-b7b6-b11a-56d108f751aa.dc1 area=wan -13:29.332 [INFO] TestAgent.server: Adding LAN server: server="Node-bd583525-973d-b7b6-b11a-56d108f751aa (Addr: tcp/127.0.0.1:16511) (DC: dc1)" -13:29.332 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16520 network=tcp -13:29.333 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16507 network=tcp -13:29.333 [INFO] TestAgent: started state syncer -13:29.333 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16540 network=tcp -13:29.393 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:29.393 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16511 [Candidate]" term=2 -13:29.394 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:29.394 [DEBUG] TestAgent.server.raft: vote granted: from=bd583525-973d-b7b6-b11a-56d108f751aa term=2 tally=1 -13:29.394 [INFO] TestAgent.server.raft: election won: tally=1 -13:29.394 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16511 [Leader]" -13:29.394 [INFO] TestAgent.server: cluster leadership acquired -13:29.394 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16511 -13:29.395 [INFO] TestAgent.server: New leader elected: payload=Node-bd583525-973d-b7b6-b11a-56d108f751aa -13:29.395 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:29.398 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:29.398 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:29.398 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:29.398 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:29.398 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:29.398 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bd583525-973d-b7b6-b11a-56d108f751aa -13:29.398 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bd583525-973d-b7b6-b11a-56d108f751aa -13:29.399 [INFO] TestAgent.server: federation state anti-entropy synced -13:29.596 [INFO] TestAgent: Requesting shutdown -13:29.597 [INFO] TestAgent.server: shutting down server -13:29.597 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:29.597 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:29.597 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:29.597 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:29.597 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:29.597 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:29.597 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:29.597 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:29.597 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:29.597 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:29.597 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:29.597 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:29.597Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:29.597Z [INFO] agent.router.manager: shutting down -13:29.597 [INFO] TestAgent: consul server down -13:29.597 [INFO] TestAgent: shutdown complete -13:29.597 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16520 network=tcp -13:29.597 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16520 network=udp -13:29.597 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16507 network=tcp -13:30.099 [INFO] TestAgent: Waiting for endpoints to shut down -13:30.099 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:28.604Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:28.627Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:28.629 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7cf23306-1add-a398-9394-b60058c629d1 Address:127.0.0.1:16553}]" -13:28.629 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16553 [Follower]" leader= -13:28.631 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7cf23306-1add-a398-9394-b60058c629d1.dc1 127.0.0.1 -13:28.632 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7cf23306-1add-a398-9394-b60058c629d1 127.0.0.1 -2021-10-31T23:13:28.632Z [INFO] agent.router: Initializing LAN area manager -13:28.632 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16499 network=udp -13:28.632 [INFO] TestAgent.server: Adding LAN server: server="Node-7cf23306-1add-a398-9394-b60058c629d1 (Addr: tcp/127.0.0.1:16553) (DC: dc1)" -13:28.632 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7cf23306-1add-a398-9394-b60058c629d1.dc1 area=wan -13:28.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16499 network=tcp -13:28.633 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16549 network=tcp -13:28.633 [INFO] TestAgent: started state syncer -13:28.633 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16554 network=tcp -13:28.672 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:28.672 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16553 [Candidate]" term=2 -13:28.673 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:28.673 [DEBUG] TestAgent.server.raft: vote granted: from=7cf23306-1add-a398-9394-b60058c629d1 term=2 tally=1 -13:28.673 [INFO] TestAgent.server.raft: election won: tally=1 -13:28.673 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16553 [Leader]" -13:28.673 [INFO] TestAgent.server: cluster leadership acquired -13:28.673 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16553 -13:28.673 [INFO] TestAgent.server: New leader elected: payload=Node-7cf23306-1add-a398-9394-b60058c629d1 -13:28.675 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:28.677 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:28.677 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:28.677 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:28.677 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:28.677 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:28.677 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7cf23306-1add-a398-9394-b60058c629d1 -13:28.677 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7cf23306-1add-a398-9394-b60058c629d1 -13:28.683 [INFO] TestAgent.server: federation state anti-entropy synced -13:28.698 [INFO] TestAgent: Requesting shutdown -13:28.699 [INFO] TestAgent.server: shutting down server -13:28.699 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:28.699 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:28.699 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:28.699 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:28.699 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:28.699 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:28.699 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:28.699 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:28.699 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:28.699 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:28.699 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:28.699 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:28.700Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:28.700Z [INFO] agent.router.manager: shutting down -13:28.700 [INFO] TestAgent: consul server down -13:28.700 [INFO] TestAgent: shutdown complete -13:28.700 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16499 network=tcp -13:28.700 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16499 network=udp -13:28.700 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16549 network=tcp -13:29.206 [INFO] TestAgent: Waiting for endpoints to shut down -13:29.206 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ReconnectConfigSettings (1.55s) -=== CONT TestAgentConnectAuthorize_idInvalidFormat -[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! -13:25.025 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-92b6e0d7-095e-78b6-1caa-504cab287dfd 127.0.0.1 -2021-10-31T23:13:25.025Z [INFO] agent.router: Initializing LAN area manager -13:25.078 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16261 network=udp -13:25.080 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16261 network=tcp -13:25.091 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16290 network=tcp -13:25.091 [INFO] TestAgent: started state syncer -13:25.092 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16274 network=tcp -2021-10-31T23:13:25.092Z [WARN] agent.router.manager: No servers available -13:25.092 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" -13:25.093 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Stream connection from=127.0.0.1:44452 -13:25.093 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-f51711da-2429-f702-b863-6cf695af2b4d 127.0.0.1 -13:25.094 [INFO] TestAgent.client: adding server: server="Node-f51711da-2429-f702-b863-6cf695af2b4d (Addr: tcp/127.0.0.1:16210) (DC: dc1)" -13:25.094 [INFO] TestAgent.client: New leader elected: payload=Node-f51711da-2429-f702-b863-6cf695af2b4d -13:25.098 [INFO] TestAgent.client: client starting leave -13:25.210 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader -13:25.210 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-f51711da-2429-f702-b863-6cf695af2b4d -13:25.386 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader -13:25.386 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-f51711da-2429-f702-b863-6cf695af2b4d -13:25.386 [DEBUG] TestAgent.client.serf.lan: serf: messageLeaveType: Node-92b6e0d7-095e-78b6-1caa-504cab287dfd -13:25.587 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader -13:25.587 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-f51711da-2429-f702-b863-6cf695af2b4d -13:25.587 [DEBUG] TestAgent.client.serf.lan: serf: messageLeaveType: Node-92b6e0d7-095e-78b6-1caa-504cab287dfd -13:25.587 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader -13:25.587 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-f51711da-2429-f702-b863-6cf695af2b4d -13:25.587 [DEBUG] TestAgent.client.serf.lan: serf: messageLeaveType: Node-92b6e0d7-095e-78b6-1caa-504cab287dfd -13:25.626 [INFO] TestAgent.client.serf.lan: serf: EventMemberLeave: Node-92b6e0d7-095e-78b6-1caa-504cab287dfd 127.0.0.1 -13:25.794 [DEBUG] TestAgent.client.serf.lan: serf: messageLeaveType: Node-92b6e0d7-095e-78b6-1caa-504cab287dfd -13:26.151 [INFO] TestAgent: Synced node info -13:28.430 [DEBUG] TestAgent: Node info in sync -13:28.430 [DEBUG] TestAgent: Node info in sync -13:29.227 [INFO] TestAgent: Requesting shutdown -13:29.228 [INFO] TestAgent.client: shutting down client -13:29.232 [INFO] TestAgent: consul client down -13:29.232 [INFO] TestAgent: shutdown complete -13:29.232 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16261 network=tcp -13:29.232 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16261 network=udp -13:29.232 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16290 network=tcp -13:29.733 [INFO] TestAgent: Waiting for endpoints to shut down -13:29.733 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:24.510Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:24.580Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:24.582 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f51711da-2429-f702-b863-6cf695af2b4d Address:127.0.0.1:16210}]" -13:24.583 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f51711da-2429-f702-b863-6cf695af2b4d.dc1 127.0.0.1 -13:24.583 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f51711da-2429-f702-b863-6cf695af2b4d 127.0.0.1 -2021-10-31T23:13:24.584Z [INFO] agent.router: Initializing LAN area manager -13:24.584 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16233 network=udp -13:24.584 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16210 [Follower]" leader= -13:24.584 [INFO] TestAgent.server: Adding LAN server: server="Node-f51711da-2429-f702-b863-6cf695af2b4d (Addr: tcp/127.0.0.1:16210) (DC: dc1)" -13:24.584 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f51711da-2429-f702-b863-6cf695af2b4d.dc1 area=wan -13:24.584 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16233 network=tcp -13:24.585 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16206 network=tcp -13:24.585 [INFO] TestAgent: started state syncer -13:24.585 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16239 network=tcp -13:24.649 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:24.649 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16210 [Candidate]" term=2 -13:24.649 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:24.649 [DEBUG] TestAgent.server.raft: vote granted: from=f51711da-2429-f702-b863-6cf695af2b4d term=2 tally=1 -13:24.649 [INFO] TestAgent.server.raft: election won: tally=1 -13:24.649 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16210 [Leader]" -13:24.649 [INFO] TestAgent.server: cluster leadership acquired -13:24.661 [INFO] TestAgent.server: New leader elected: payload=Node-f51711da-2429-f702-b863-6cf695af2b4d -13:24.666 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16210 -13:24.667 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:24.670 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:24.670 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:24.670 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:24.670 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:24.670 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:24.670 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f51711da-2429-f702-b863-6cf695af2b4d -13:24.670 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f51711da-2429-f702-b863-6cf695af2b4d -13:24.671 [INFO] TestAgent.server: federation state anti-entropy synced -13:24.854 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:24.854 [INFO] TestAgent: Synced node info -13:24.854 [DEBUG] TestAgent: Node info in sync -13:25.092 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:16292] -13:25.092 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:16292 -13:25.098 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-92b6e0d7-095e-78b6-1caa-504cab287dfd 127.0.0.1 -13:25.098 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -13:25.098 [DEBUG] TestAgent: systemd notify failed: error="No socket" -13:25.098 [INFO] TestAgent.server: member joined, marking health alive: member=Node-92b6e0d7-095e-78b6-1caa-504cab287dfd -13:25.234 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-f51711da-2429-f702-b863-6cf695af2b4d -13:25.234 [DEBUG] TestAgent.server.serf.lan: serf: messageLeaveType: Node-92b6e0d7-095e-78b6-1caa-504cab287dfd -13:25.450 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-f51711da-2429-f702-b863-6cf695af2b4d -13:25.450 [DEBUG] TestAgent.server.serf.lan: serf: messageLeaveType: Node-92b6e0d7-095e-78b6-1caa-504cab287dfd -13:25.587 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-f51711da-2429-f702-b863-6cf695af2b4d -13:25.587 [DEBUG] TestAgent.server.serf.lan: serf: messageLeaveType: Node-92b6e0d7-095e-78b6-1caa-504cab287dfd -13:25.630 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-f51711da-2429-f702-b863-6cf695af2b4d -13:25.630 [DEBUG] TestAgent.server.serf.lan: serf: messageLeaveType: Node-92b6e0d7-095e-78b6-1caa-504cab287dfd -13:25.831 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-92b6e0d7-095e-78b6-1caa-504cab287dfd 127.0.0.1 -13:25.831 [INFO] TestAgent.server: deregistering member: member=Node-92b6e0d7-095e-78b6-1caa-504cab287dfd reason=left -13:26.442 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:26.442 [DEBUG] TestAgent: Node info in sync -13:29.733 [INFO] TestAgent: Requesting shutdown -13:29.733 [INFO] TestAgent.server: shutting down server -13:29.733 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:29.733 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:29.733 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:29.733 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:29.733 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:29.733 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:29.733 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:29.733 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:29.733 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:29.733 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:29.733 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:29.733Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:29.733Z [INFO] agent.router.manager: shutting down -13:29.733 [INFO] TestAgent: consul server down -13:29.733 [INFO] TestAgent: shutdown complete -13:29.733 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16233 network=tcp -13:29.733 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16233 network=udp -13:29.733 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16206 network=tcp -13:30.235 [INFO] TestAgent: Waiting for endpoints to shut down -13:30.235 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Leave (5.78s) -=== CONT TestAgentConnectAuthorize_noTarget -=== RUN TestAgent_JoinWAN_viaMeshGateway/dc1_to_dc2 -=== RUN TestAgent_JoinWAN_viaMeshGateway/dc1_to_dc3 -=== RUN TestAgent_JoinWAN_viaMeshGateway/dc2_to_dc1 -=== RUN TestAgent_JoinWAN_viaMeshGateway/dc2_to_dc3 -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestAgent_JoinWAN_viaMeshGateway/dc3_to_dc1 -=== RUN TestAgent_JoinWAN_viaMeshGateway/dc3_to_dc2 -2021-10-31T23:13:29.508Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:29.508Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:29.534Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:29.534Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:29.536 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cfea8638-4bcc-9806-5fef-265c11869eb0 Address:127.0.0.1:16636}]" -13:29.537 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cfea8638-4bcc-9806-5fef-265c11869eb0.dc1 127.0.0.1 -13:29.537 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cfea8638-4bcc-9806-5fef-265c11869eb0 127.0.0.1 -2021-10-31T23:13:29.538Z [INFO] agent.router: Initializing LAN area manager -13:29.538 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16611 network=udp -13:29.537 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16636 [Follower]" leader= -13:29.538 [INFO] TestAgent.server: Adding LAN server: server="Node-cfea8638-4bcc-9806-5fef-265c11869eb0 (Addr: tcp/127.0.0.1:16636) (DC: dc1)" -13:29.538 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cfea8638-4bcc-9806-5fef-265c11869eb0.dc1 area=wan -13:29.538 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16611 network=tcp -13:29.539 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16632 network=tcp -13:29.539 [INFO] TestAgent: started state syncer -13:29.539 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16575 network=tcp -13:29.589 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:29.589 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16636 [Candidate]" term=2 -13:29.590 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:29.590 [DEBUG] TestAgent.server.raft: vote granted: from=cfea8638-4bcc-9806-5fef-265c11869eb0 term=2 tally=1 -13:29.590 [INFO] TestAgent.server.raft: election won: tally=1 -13:29.590 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16636 [Leader]" -13:29.589 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:29.590 [INFO] TestAgent.server: cluster leadership acquired -13:29.590 [INFO] TestAgent.server: initializing acls -13:29.590 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:29.590 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:29.591 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:29.591 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:29.591 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:29.591 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:29.591 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-cfea8638-4bcc-9806-5fef-265c11869eb0 -13:29.591 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-cfea8638-4bcc-9806-5fef-265c11869eb0.dc1 -13:29.592 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:29.593 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:29.593 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:29.593 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:29.593 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:29.593 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:29.593 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cfea8638-4bcc-9806-5fef-265c11869eb0 -13:29.593 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cfea8638-4bcc-9806-5fef-265c11869eb0 -13:29.594 [INFO] TestAgent.server: New leader elected: payload=Node-cfea8638-4bcc-9806-5fef-265c11869eb0 -13:29.594 [INFO] TestAgent.server: Updating LAN server: server="Node-cfea8638-4bcc-9806-5fef-265c11869eb0 (Addr: tcp/127.0.0.1:16636) (DC: dc1)" -13:29.594 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-cfea8638-4bcc-9806-5fef-265c11869eb0.dc1 area=wan -13:29.594 [INFO] TestAgent.server: federation state anti-entropy synced -13:29.594 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cfea8638-4bcc-9806-5fef-265c11869eb0 -13:29.863 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:29.866 [INFO] TestAgent: Synced node info -13:29.911 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-cfea8638-4bcc-9806-5fef-265c11869eb0 -13:29.911 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-cfea8638-4bcc-9806-5fef-265c11869eb0 -13:29.912 [INFO] TestAgent: Requesting shutdown -13:29.912 [INFO] TestAgent.server: shutting down server -13:29.912 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:29.912 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:29.912 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:29.912 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:29.912 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:29.912 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:29.912 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:29.912 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:29.912 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:29.912 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:29.912 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:29.912 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:29.912 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:29.912 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:29.912 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:29.912Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:29.913Z [INFO] agent.router.manager: shutting down -13:29.913 [INFO] TestAgent: consul server down -13:29.913 [INFO] TestAgent: shutdown complete -13:29.913 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16611 network=tcp -13:29.913 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16611 network=udp -13:29.913 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16632 network=tcp -13:30.413 [INFO] TestAgent: Waiting for endpoints to shut down -13:30.413 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_serviceWrite (0.92s) -=== CONT TestACL_filterChecks -2021-10-31T23:13:30.449Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:30.449Z [WARN] agent: bootstrap = true: do not enable unless necessary -13:30.450 [DEBUG] TestACL_filterChecks: dropping check from result due to ACLs: check=my-node -13:30.450 [DEBUG] TestACL_filterChecks: dropping check from result due to ACLs: check=my-other -13:30.450 [DEBUG] TestACL_filterChecks: dropping check from result due to ACLs: check=my-service -13:30.450 [DEBUG] TestACL_filterChecks: dropping check from result due to ACLs: check=my-other ---- PASS: TestACL_filterChecks (0.04s) -=== CONT TestAgent_Members_ACLFilter -2021-10-31T23:13:29.671Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:29.726Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:29.728 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d87368de-e839-5883-5a12-30057eb09837 Address:127.0.0.1:16650}]" -13:29.728 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16650 [Follower]" leader= -13:29.729 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d87368de-e839-5883-5a12-30057eb09837.dc1 127.0.0.1 -13:29.742 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d87368de-e839-5883-5a12-30057eb09837 127.0.0.1 -2021-10-31T23:13:29.742Z [INFO] agent.router: Initializing LAN area manager -13:29.742 [INFO] TestAgent.server: Adding LAN server: server="Node-d87368de-e839-5883-5a12-30057eb09837 (Addr: tcp/127.0.0.1:16650) (DC: dc1)" -13:29.742 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d87368de-e839-5883-5a12-30057eb09837.dc1 area=wan -13:29.742 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16659 network=udp -13:29.742 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16659 network=tcp -13:29.743 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16646 network=tcp -13:29.743 [INFO] TestAgent: started state syncer -13:29.743 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16637 network=tcp -13:29.772 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:29.772 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16650 [Candidate]" term=2 -13:29.772 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:29.772 [DEBUG] TestAgent.server.raft: vote granted: from=d87368de-e839-5883-5a12-30057eb09837 term=2 tally=1 -13:29.772 [INFO] TestAgent.server.raft: election won: tally=1 -13:29.772 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16650 [Leader]" -13:29.772 [INFO] TestAgent.server: cluster leadership acquired -13:29.772 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16650 -13:29.773 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:29.773 [INFO] TestAgent.server: New leader elected: payload=Node-d87368de-e839-5883-5a12-30057eb09837 -13:29.775 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:29.775 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:29.775 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:29.775 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:29.775 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:29.775 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d87368de-e839-5883-5a12-30057eb09837 -13:29.775 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d87368de-e839-5883-5a12-30057eb09837 -13:29.776 [INFO] TestAgent.server: federation state anti-entropy synced -13:29.929 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:29.930 [INFO] TestAgent: Synced node info -13:29.930 [DEBUG] TestAgent: Node info in sync -13:29.993 [INFO] TestAgent: Requesting shutdown -13:29.993 [INFO] TestAgent.server: shutting down server -13:29.993 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:29.993 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:29.993 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:29.993 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:29.993 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:29.993 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:29.993 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:29.993 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:29.993 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:29.993 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:29.993 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:29.994Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:29.994Z [INFO] agent.router.manager: shutting down -13:29.994 [INFO] TestAgent: consul server down -13:29.994 [INFO] TestAgent: shutdown complete -13:29.994 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16659 network=tcp -13:29.994 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16659 network=udp -13:29.994 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16646 network=tcp -13:30.495 [INFO] TestAgent: Waiting for endpoints to shut down -13:30.495 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_allowTrustDomain (0.86s) -=== CONT TestAgent_Members_WAN -2021-10-31T23:13:29.608Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:29.635Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:29.637 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bf954fcf-227e-016d-4e35-d2dde8c560ec Address:127.0.0.1:16609}]" -13:29.637 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16609 [Follower]" leader= -13:29.639 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bf954fcf-227e-016d-4e35-d2dde8c560ec.dc1 127.0.0.1 -13:29.639 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bf954fcf-227e-016d-4e35-d2dde8c560ec 127.0.0.1 -2021-10-31T23:13:29.639Z [INFO] agent.router: Initializing LAN area manager -13:29.640 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16576 network=udp -13:29.640 [INFO] TestAgent.server: Adding LAN server: server="Node-bf954fcf-227e-016d-4e35-d2dde8c560ec (Addr: tcp/127.0.0.1:16609) (DC: dc1)" -13:29.640 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bf954fcf-227e-016d-4e35-d2dde8c560ec.dc1 area=wan -13:29.640 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16576 network=tcp -13:29.641 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16605 network=tcp -13:29.641 [INFO] TestAgent: started state syncer -13:29.641 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16658 network=tcp -13:29.689 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:29.689 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16609 [Candidate]" term=2 -13:29.690 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:29.690 [DEBUG] TestAgent.server.raft: vote granted: from=bf954fcf-227e-016d-4e35-d2dde8c560ec term=2 tally=1 -13:29.690 [INFO] TestAgent.server.raft: election won: tally=1 -13:29.690 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16609 [Leader]" -13:29.690 [INFO] TestAgent.server: cluster leadership acquired -13:29.690 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16609 -13:29.691 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:29.691 [INFO] TestAgent.server: New leader elected: payload=Node-bf954fcf-227e-016d-4e35-d2dde8c560ec -13:29.693 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:29.693 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:29.693 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:29.693 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:29.693 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:29.693 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bf954fcf-227e-016d-4e35-d2dde8c560ec -13:29.693 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bf954fcf-227e-016d-4e35-d2dde8c560ec -13:29.698 [INFO] TestAgent.server: federation state anti-entropy synced -13:29.970 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:29.970 [INFO] TestAgent: Synced node info -13:29.970 [DEBUG] TestAgent: Node info in sync -13:30.059 [INFO] TestAgent: Requesting shutdown -13:30.059 [INFO] TestAgent.server: shutting down server -13:30.059 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:30.059 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:30.059 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:30.059 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:30.059 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:30.059 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:30.059 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:30.059 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:30.059 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:30.059 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:30.059 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:30.059Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.059Z [INFO] agent.router.manager: shutting down -13:30.060 [INFO] TestAgent: consul server down -13:30.060 [INFO] TestAgent: shutdown complete -13:30.060 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16576 network=tcp -13:30.060 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16576 network=udp -13:30.060 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16605 network=tcp -13:30.567 [INFO] TestAgent: Waiting for endpoints to shut down -13:30.567 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_denyWildcard (0.98s) -=== CONT TestAgent_Members -2021-10-31T23:13:29.965Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:29.996Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:30.004 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fc2b2771-e40e-aaa7-8d00-fb33d3621b5b Address:127.0.0.1:16643}]" -13:30.004 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fc2b2771-e40e-aaa7-8d00-fb33d3621b5b.dc1 127.0.0.1 -13:30.005 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16643 [Follower]" leader= -13:30.005 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fc2b2771-e40e-aaa7-8d00-fb33d3621b5b 127.0.0.1 -2021-10-31T23:13:30.005Z [INFO] agent.router: Initializing LAN area manager -13:30.005 [INFO] TestAgent.server: Adding LAN server: server="Node-fc2b2771-e40e-aaa7-8d00-fb33d3621b5b (Addr: tcp/127.0.0.1:16643) (DC: dc1)" -13:30.005 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fc2b2771-e40e-aaa7-8d00-fb33d3621b5b.dc1 area=wan -13:30.005 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16645 network=udp -13:30.006 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16645 network=tcp -13:30.006 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16639 network=tcp -13:30.006 [INFO] TestAgent: started state syncer -13:30.006 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16644 network=tcp -13:30.066 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:30.066 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16643 [Candidate]" term=2 -13:30.066 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:30.066 [DEBUG] TestAgent.server.raft: vote granted: from=fc2b2771-e40e-aaa7-8d00-fb33d3621b5b term=2 tally=1 -13:30.066 [INFO] TestAgent.server.raft: election won: tally=1 -13:30.066 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16643 [Leader]" -13:30.066 [INFO] TestAgent.server: cluster leadership acquired -13:30.066 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16643 -13:30.067 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:30.068 [INFO] TestAgent.server: New leader elected: payload=Node-fc2b2771-e40e-aaa7-8d00-fb33d3621b5b -13:30.069 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:30.069 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:30.069 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:30.069 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:30.069 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:30.069 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fc2b2771-e40e-aaa7-8d00-fb33d3621b5b -13:30.069 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fc2b2771-e40e-aaa7-8d00-fb33d3621b5b -13:30.071 [INFO] TestAgent.server: federation state anti-entropy synced -13:30.072 [INFO] TestAgent: Requesting shutdown -13:30.072 [INFO] TestAgent.server: shutting down server -13:30.072 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:30.072 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:30.072 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:30.072 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:30.072 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:30.072 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:30.072 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:30.072 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:30.072 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:30.072 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:30.072 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:30.072 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:30.072Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.072Z [INFO] agent.router.manager: shutting down -13:30.072 [INFO] TestAgent: consul server down -13:30.072 [INFO] TestAgent: shutdown complete -13:30.072 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16645 network=tcp -13:30.072 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16645 network=udp -13:30.072 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16639 network=tcp -13:30.573 [INFO] TestAgent: Waiting for endpoints to shut down -13:30.573 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_deny (0.64s) +2022-12-04T05:49:35.091Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:35.103Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:35.105 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7359191d-0df3-8215-a93f-f034321cbf95 Address:127.0.0.1:24081}]" +49:35.105 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24081 [Follower]" leader= +49:35.106 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7359191d-0df3-8215-a93f-f034321cbf95.dc1 127.0.0.1 +49:35.106 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7359191d-0df3-8215-a93f-f034321cbf95 127.0.0.1 +2022-12-04T05:49:35.106Z [INFO] agent.router: Initializing LAN area manager +49:35.107 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7359191d-0df3-8215-a93f-f034321cbf95.dc1 area=wan +49:35.107 [INFO] TestAgent.server: Adding LAN server: server="Node-7359191d-0df3-8215-a93f-f034321cbf95 (Addr: tcp/127.0.0.1:24081) (DC: dc1)" +49:35.107 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24111 network=udp +49:35.107 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24111 network=tcp +49:35.107 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24090 network=tcp +49:35.107 [INFO] TestAgent: started state syncer +49:35.107 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24071 network=tcp +49:35.165 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.165 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24081 [Candidate]" term=2 +49:35.165 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:35.165 [DEBUG] TestAgent.server.raft: vote granted: from=7359191d-0df3-8215-a93f-f034321cbf95 term=2 tally=1 +49:35.165 [INFO] TestAgent.server.raft: election won: tally=1 +49:35.165 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24081 [Leader]" +49:35.165 [INFO] TestAgent.server: cluster leadership acquired +49:35.165 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24081 +49:35.167 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:35.167 [INFO] TestAgent.server: New leader elected: payload=Node-7359191d-0df3-8215-a93f-f034321cbf95 +49:35.170 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:35.170 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:35.170 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:35.170 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:35.170 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:35.170 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7359191d-0df3-8215-a93f-f034321cbf95 +49:35.170 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7359191d-0df3-8215-a93f-f034321cbf95 +49:35.172 [INFO] TestAgent.server: federation state anti-entropy synced +49:35.183 [INFO] TestAgent: Synced node info +49:35.183 [INFO] TestAgent: Synced service: service=monitor +49:35.183 [DEBUG] TestAgent: Check in sync: check=service:monitor +49:35.448 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.448 [DEBUG] TestAgent: Node info in sync +49:35.448 [DEBUG] TestAgent: Service in sync: service=monitor +49:35.448 [DEBUG] TestAgent: Check in sync: check=service:monitor +49:35.513 [DEBUG] TestAgent: Node info in sync +49:35.514 [INFO] TestAgent: Synced service: service=monitor +49:35.514 [DEBUG] TestAgent: Check in sync: check=service:monitor +49:37.560 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:37.560 [DEBUG] TestAgent: Node info in sync +49:37.560 [DEBUG] TestAgent: Service in sync: service=monitor +49:37.560 [DEBUG] TestAgent: Check in sync: check=service:monitor +49:37.560 [DEBUG] TestAgent: Node info in sync +49:37.560 [DEBUG] TestAgent: Service in sync: service=monitor +49:37.560 [DEBUG] TestAgent: Check in sync: check=service:monitor +49:37.640 [DEBUG] TestAgent: Node info in sync +49:37.641 [INFO] TestAgent: Synced service: service=monitor +49:37.641 [DEBUG] TestAgent: Check in sync: check=service:monitor +49:37.641 [DEBUG] TestAgent: Node info in sync +49:37.641 [DEBUG] TestAgent: Service in sync: service=monitor +49:37.641 [DEBUG] TestAgent: Check in sync: check=service:monitor +49:37.641 [DEBUG] TestAgent: Node info in sync +49:37.641 [DEBUG] TestAgent: Service in sync: service=monitor +49:37.641 [DEBUG] TestAgent: Check in sync: check=service:monitor +49:37.767 [DEBUG] TestAgent: Node info in sync +49:37.768 [INFO] TestAgent: Synced service: service=monitor +49:37.768 [DEBUG] TestAgent: Check in sync: check=service:monitor +49:37.768 [DEBUG] TestAgent: Node info in sync +49:37.768 [DEBUG] TestAgent: Service in sync: service=monitor +49:37.768 [DEBUG] TestAgent: Check in sync: check=service:monitor +49:37.768 [DEBUG] TestAgent: Node info in sync +49:37.768 [DEBUG] TestAgent: Service in sync: service=monitor +49:37.768 [DEBUG] TestAgent: Check in sync: check=service:monitor +49:37.869 [DEBUG] TestAgent: Node info in sync +49:37.869 [INFO] TestAgent: Synced service: service=monitor +49:37.869 [DEBUG] TestAgent: Check in sync: check=service:monitor +49:37.869 [DEBUG] TestAgent: Node info in sync +49:37.869 [DEBUG] TestAgent: Service in sync: service=monitor +49:37.869 [DEBUG] TestAgent: Check in sync: check=service:monitor +49:37.970 [INFO] TestAgent: Requesting shutdown +49:37.970 [INFO] TestAgent.server: shutting down server +49:37.970 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:37.970 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:37.970 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:37.970 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:37.970 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:37.970 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:37.970 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:37.970 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:37.970 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:37.970 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:37.970 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:37.970Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.970Z [INFO] agent.router.manager: shutting down +49:37.971 [INFO] TestAgent: consul server down +49:37.971 [INFO] TestAgent: shutdown complete +49:37.971 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24111 network=tcp +49:37.971 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24111 network=udp +49:37.971 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24090 network=tcp +49:38.471 [INFO] TestAgent: Waiting for endpoints to shut down +49:38.471 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Monitor (3.50s) + --- PASS: TestAgent_Monitor/unknown_log_level (0.00s) + --- PASS: TestAgent_Monitor/stream_unstructured_logs (2.69s) + --- PASS: TestAgent_Monitor/stream_JSON_logs (0.10s) + --- PASS: TestAgent_Monitor/serf_shutdown_logging (0.60s) === CONT TestAgent_Reload_ACLDeny -2021-10-31T23:13:24.193Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:24.230Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:24.231 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:466bb74b-3fd4-c7c5-1822-cee038ecbbb4 Address:127.0.0.1:16147}]" -13:24.232 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16147 [Follower]" leader= -13:24.232 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4.dc1 127.0.0.1 -13:24.232 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 127.0.0.1 -2021-10-31T23:13:24.232Z [INFO] agent.router: Initializing LAN area manager -13:24.233 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16219 network=udp -13:24.233 [INFO] TestAgent.server: Adding LAN server: server="Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 (Addr: tcp/127.0.0.1:16147) (DC: dc1)" -13:24.233 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4.dc1 area=wan -13:24.233 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16219 network=tcp -13:24.239 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16143 network=tcp -13:24.240 [INFO] TestAgent: started state syncer -13:24.240 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17077 network=tcp -13:24.290 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:24.290 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16147 [Candidate]" term=2 -13:24.290 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:24.290 [DEBUG] TestAgent.server.raft: vote granted: from=466bb74b-3fd4-c7c5-1822-cee038ecbbb4 term=2 tally=1 -13:24.290 [INFO] TestAgent.server.raft: election won: tally=1 -13:24.290 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16147 [Leader]" -13:24.290 [INFO] TestAgent.server: cluster leadership acquired -13:24.290 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16147 -13:24.291 [INFO] TestAgent.server: New leader elected: payload=Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 -13:24.291 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:24.293 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:24.293 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:24.293 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:24.293 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:24.293 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:24.293 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 -13:24.293 [INFO] TestAgent.server: member joined, marking health alive: member=Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 -13:24.294 [INFO] TestAgent.server: federation state anti-entropy synced -13:24.589 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:24.590 [INFO] TestAgent: Synced node info -13:24.634 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:53622 -13:24.634 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9e851342-f12a-ebe4-133a-2b491f1e50ce 127.0.0.1 -13:24.635 [INFO] TestAgent.server: Adding LAN server: server="Node-9e851342-f12a-ebe4-133a-2b491f1e50ce (Addr: tcp/127.0.0.1:16224) (DC: dc1)" -13:24.635 [INFO] TestAgent.server: New leader elected: payload=Node-9e851342-f12a-ebe4-133a-2b491f1e50ce -13:24.635 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-9e851342-f12a-ebe4-133a-2b491f1e50ce other=Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 -13:24.635 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9e851342-f12a-ebe4-133a-2b491f1e50ce -13:24.636 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-9e851342-f12a-ebe4-133a-2b491f1e50ce.dc1 127.0.0.1:16223 -13:24.638 [INFO] TestAgent: Requesting shutdown -13:24.638 [INFO] TestAgent.server: shutting down server -13:24.638 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:24.638 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:24.638 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:24.638 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:24.638 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:24.646 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:24.646 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:24.646 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:24.646 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9e851342-f12a-ebe4-133a-2b491f1e50ce.dc1 127.0.0.1 -13:24.646 [ERROR] TestAgent.server.memberlist.lan: memberlist: Failed to send gossip to 127.0.0.1:16222: write udp 127.0.0.1:16145->127.0.0.1:16222: use of closed network connection -13:24.646 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-9e851342-f12a-ebe4-133a-2b491f1e50ce.dc1 address=127.0.0.1:16223 -13:24.646 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9e851342-f12a-ebe4-133a-2b491f1e50ce.dc1 area=wan -13:24.646 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:24.647Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:24.647Z [INFO] agent.router.manager: shutting down -13:24.647 [INFO] TestAgent: consul server down -13:24.647 [INFO] TestAgent: shutdown complete -13:24.647 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16219 network=tcp -13:24.647 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16219 network=udp -13:24.647 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:24.647 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:24.648 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16143 network=tcp -13:25.150 [INFO] TestAgent: Waiting for endpoints to shut down -13:25.150 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:24.016Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:24.031Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:24.033 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9e851342-f12a-ebe4-133a-2b491f1e50ce Address:127.0.0.1:16224}]" -13:24.033 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9e851342-f12a-ebe4-133a-2b491f1e50ce.dc1 127.0.0.1 -13:24.033 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16224 [Follower]" leader= -13:24.045 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9e851342-f12a-ebe4-133a-2b491f1e50ce 127.0.0.1 -2021-10-31T23:13:24.045Z [INFO] agent.router: Initializing LAN area manager -13:24.045 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9e851342-f12a-ebe4-133a-2b491f1e50ce.dc1 area=wan -13:24.046 [INFO] TestAgent.server: Adding LAN server: server="Node-9e851342-f12a-ebe4-133a-2b491f1e50ce (Addr: tcp/127.0.0.1:16224) (DC: dc1)" -13:24.046 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16142 network=udp -13:24.046 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16142 network=tcp -13:24.047 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16220 network=tcp -13:24.047 [INFO] TestAgent: started state syncer -13:24.047 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16246 network=tcp -13:24.088 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:24.088 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16224 [Candidate]" term=2 -13:24.088 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:24.088 [DEBUG] TestAgent.server.raft: vote granted: from=9e851342-f12a-ebe4-133a-2b491f1e50ce term=2 tally=1 -13:24.088 [INFO] TestAgent.server.raft: election won: tally=1 -13:24.088 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16224 [Leader]" -13:24.088 [INFO] TestAgent.server: cluster leadership acquired -13:24.088 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16224 -13:24.089 [INFO] TestAgent.server: New leader elected: payload=Node-9e851342-f12a-ebe4-133a-2b491f1e50ce -13:24.089 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:24.091 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:24.091 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:24.091 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:24.091 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:24.091 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:24.091 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9e851342-f12a-ebe4-133a-2b491f1e50ce -13:24.091 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9e851342-f12a-ebe4-133a-2b491f1e50ce -13:24.091 [INFO] TestAgent.server: federation state anti-entropy synced -13:24.406 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:24.406 [INFO] TestAgent: Synced node info -13:24.631 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:16145] -13:24.633 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:16145 -13:24.637 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:35012 -13:24.637 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4.dc1 127.0.0.1 -13:24.637 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4.dc1 area=wan -13:24.638 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 127.0.0.1 -13:24.638 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -13:24.638 [DEBUG] TestAgent: systemd notify failed: error="No socket" -13:24.646 [INFO] TestAgent.server: Adding LAN server: server="Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 (Addr: tcp/127.0.0.1:16147) (DC: dc1)" -13:24.646 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 other=Node-9e851342-f12a-ebe4-133a-2b491f1e50ce -13:24.646 [INFO] TestAgent.server: member joined, marking health alive: member=Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 -13:25.454 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:25.454 [DEBUG] TestAgent: Node info in sync -13:25.454 [DEBUG] TestAgent: Node info in sync -13:25.590 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 (timeout reached) -13:26.068 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 has failed, no acks received -13:26.103 [WARN] TestAgent: error getting server health from server: server=Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:16147: connect: connection refused" -13:27.105 [WARN] TestAgent: error getting server health from server: server=Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 error="context deadline exceeded" -13:27.553 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 (timeout reached) -13:28.091 [WARN] TestAgent: error getting server health from server: server=Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:16147: connect: connection refused" -13:29.053 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 has failed, no acks received -13:29.094 [WARN] TestAgent: error getting server health from server: server=Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 error="context deadline exceeded" -13:30.070 [INFO] TestAgent.server.memberlist.lan: memberlist: Marking Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 as failed, suspect timeout reached (0 peer confirmations) -13:30.070 [INFO] TestAgent.server.serf.lan: serf: EventMemberFailed: Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 127.0.0.1 -13:30.070 [INFO] TestAgent.server: Removing LAN server: server="Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 (Addr: tcp/127.0.0.1:16147) (DC: dc1)" -13:30.071 [INFO] TestAgent.server: member failed, marking health critical: member=Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 -13:30.071 [INFO] TestAgent: Force leaving node: node=Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 -13:30.071 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave (forced): Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 127.0.0.1 -13:30.072 [INFO] TestAgent.server: Removing LAN server: server="Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 (Addr: tcp/127.0.0.1:16147) (DC: dc1)" -13:30.072 [INFO] TestAgent.server: deregistering member: member=Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 reason=left -13:30.098 [INFO] TestAgent: Requesting shutdown -13:30.098 [INFO] TestAgent.server: shutting down server -13:30.098 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:30.098 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:30.098 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:30.098 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:30.098 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:30.098 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:30.098 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:30.098 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:30.098 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:30.098 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:30.098 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:30.099Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.099Z [INFO] agent.router.manager: shutting down -13:30.099 [INFO] TestAgent: consul server down -13:30.099 [INFO] TestAgent: shutdown complete -13:30.099 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16142 network=tcp -13:30.099 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16142 network=udp -13:30.099 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16220 network=tcp -13:30.546 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-466bb74b-3fd4-c7c5-1822-cee038ecbbb4 (timeout reached) -13:30.600 [INFO] TestAgent: Waiting for endpoints to shut down -13:30.600 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ForceLeave (6.60s) +2022-12-04T05:49:37.963Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:37.963Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.974Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:37.974Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.976 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8e0d6942-9d60-3d2c-22b3-934c6422813a Address:127.0.0.1:23460}]" +49:37.976 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23460 [Follower]" leader= +49:37.976 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8e0d6942-9d60-3d2c-22b3-934c6422813a.dc1 127.0.0.1 +49:37.976 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8e0d6942-9d60-3d2c-22b3-934c6422813a 127.0.0.1 +2022-12-04T05:49:37.977Z [INFO] agent.router: Initializing LAN area manager +49:37.977 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23392 network=udp +49:37.977 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8e0d6942-9d60-3d2c-22b3-934c6422813a.dc1 area=wan +49:37.977 [INFO] TestAgent.server: Adding LAN server: server="Node-8e0d6942-9d60-3d2c-22b3-934c6422813a (Addr: tcp/127.0.0.1:23460) (DC: dc1)" +49:37.977 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23392 network=tcp +49:37.977 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23435 network=tcp +49:37.977 [INFO] TestAgent: started state syncer +49:37.977 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23419 network=tcp +49:38.028 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:38.037 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:38.037 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23460 [Candidate]" term=2 +49:38.037 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:38.037 [DEBUG] TestAgent.server.raft: vote granted: from=8e0d6942-9d60-3d2c-22b3-934c6422813a term=2 tally=1 +49:38.037 [INFO] TestAgent.server.raft: election won: tally=1 +49:38.037 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23460 [Leader]" +49:38.037 [INFO] TestAgent.server: cluster leadership acquired +49:38.038 [INFO] TestAgent.server: initializing acls +49:38.038 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:38.038 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:38.038 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:38.038 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:38.038 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:38.038 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:38.038 [INFO] TestAgent.server: New leader elected: payload=Node-8e0d6942-9d60-3d2c-22b3-934c6422813a +49:38.038 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-8e0d6942-9d60-3d2c-22b3-934c6422813a +49:38.038 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-8e0d6942-9d60-3d2c-22b3-934c6422813a.dc1 +49:38.039 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:38.039 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-8e0d6942-9d60-3d2c-22b3-934c6422813a.dc1 area=wan +49:38.039 [INFO] TestAgent.server: Updating LAN server: server="Node-8e0d6942-9d60-3d2c-22b3-934c6422813a (Addr: tcp/127.0.0.1:23460) (DC: dc1)" +49:38.040 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:38.040 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:38.040 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:38.040 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:38.040 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:38.040 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8e0d6942-9d60-3d2c-22b3-934c6422813a +49:38.040 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8e0d6942-9d60-3d2c-22b3-934c6422813a +49:38.040 [INFO] TestAgent.server: federation state anti-entropy synced +49:38.041 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8e0d6942-9d60-3d2c-22b3-934c6422813a +49:38.079 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-8e0d6942-9d60-3d2c-22b3-934c6422813a +49:38.080 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-8e0d6942-9d60-3d2c-22b3-934c6422813a +49:38.080 [DEBUG] TestAgent: dropping node from result due to ACLs: node=Node-8e0d6942-9d60-3d2c-22b3-934c6422813a accessorID=00000000-0000-0000-0000-000000000002 +49:38.080 [INFO] TestAgent: Requesting shutdown +49:38.080 [INFO] TestAgent.server: shutting down server +49:38.080 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:38.080 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:38.080 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:38.080 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:38.080 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:38.080 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:38.080 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:38.080 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:38.080 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:38.080 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:38.080 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:38.080 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:38.080 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:38.080 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:38.080 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:38.080 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:38.081Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.081Z [INFO] agent.router.manager: shutting down +49:38.081 [INFO] TestAgent: consul server down +49:38.081 [INFO] TestAgent: shutdown complete +49:38.081 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23392 network=tcp +49:38.081 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23392 network=udp +49:38.081 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23435 network=tcp +49:38.581 [INFO] TestAgent: Waiting for endpoints to shut down +49:38.581 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Members_ACLFilter (0.63s) + --- PASS: TestAgent_Members_ACLFilter/no_token (0.00s) + --- PASS: TestAgent_Members_ACLFilter/root_token (0.00s) === CONT TestAgent_Reload -[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=false -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestAgent_Members_ACLFilter/no_token -=== RUN TestAgent_Members_ACLFilter/root_token -2021-10-31T23:13:30.076Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:30.143Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:30.159 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2d4013e5-6dca-9658-136e-4f507b29be67 Address:127.0.0.1:16664}]" -13:30.159 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2d4013e5-6dca-9658-136e-4f507b29be67.dc1 127.0.0.1 -13:30.160 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2d4013e5-6dca-9658-136e-4f507b29be67 127.0.0.1 -2021-10-31T23:13:30.160Z [INFO] agent.router: Initializing LAN area manager -13:30.161 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16631 network=udp -13:30.161 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16664 [Follower]" leader= -13:30.162 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2d4013e5-6dca-9658-136e-4f507b29be67.dc1 area=wan -13:30.170 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16631 network=tcp -13:30.171 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16660 network=tcp -13:30.171 [INFO] TestAgent: started state syncer -13:30.179 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16665 network=tcp -13:30.179 [INFO] TestAgent.server: Adding LAN server: server="Node-2d4013e5-6dca-9658-136e-4f507b29be67 (Addr: tcp/127.0.0.1:16664) (DC: dc1)" -13:30.230 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:30.230 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16664 [Candidate]" term=2 -13:30.230 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:30.230 [DEBUG] TestAgent.server.raft: vote granted: from=2d4013e5-6dca-9658-136e-4f507b29be67 term=2 tally=1 -13:30.230 [INFO] TestAgent.server.raft: election won: tally=1 -13:30.230 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16664 [Leader]" -13:30.230 [INFO] TestAgent.server: cluster leadership acquired -13:30.231 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16664 -13:30.231 [INFO] TestAgent.server: New leader elected: payload=Node-2d4013e5-6dca-9658-136e-4f507b29be67 -13:30.233 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:30.237 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:30.237 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:30.237 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:30.237 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:30.237 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:30.237 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2d4013e5-6dca-9658-136e-4f507b29be67 -13:30.237 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2d4013e5-6dca-9658-136e-4f507b29be67 -13:30.238 [INFO] TestAgent.server: federation state anti-entropy synced -13:30.407 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:30.408 [INFO] TestAgent: Synced node info -13:30.408 [DEBUG] TestAgent: Node info in sync -13:30.408 [INFO] TestAgent: Requesting shutdown -13:30.408 [INFO] TestAgent.server: shutting down server -13:30.408 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:30.408 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:30.408 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:30.408 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:30.408 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:30.408 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:30.408 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:30.408 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:30.408 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:30.408 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:30.408 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:30.409Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.409Z [INFO] agent.router.manager: shutting down -13:30.409 [INFO] TestAgent: consul server down -13:30.409 [INFO] TestAgent: shutdown complete -13:30.409 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16631 network=tcp -13:30.409 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16631 network=udp -13:30.409 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16660 network=tcp -13:30.909 [INFO] TestAgent: Waiting for endpoints to shut down -13:30.909 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:29.974Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:30.015Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:30.017 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e1683b60-5e08-8c30-e414-0e5f01ee65da Address:127.0.0.1:16616}]" -13:30.017 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e1683b60-5e08-8c30-e414-0e5f01ee65da.dc1 127.0.0.1 -13:30.017 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16616 [Follower]" leader= -13:30.026 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e1683b60-5e08-8c30-e414-0e5f01ee65da 127.0.0.1 -2021-10-31T23:13:30.026Z [INFO] agent.router: Initializing LAN area manager -13:30.026 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16652 network=udp -13:30.026 [INFO] TestAgent.server: Adding LAN server: server="Node-e1683b60-5e08-8c30-e414-0e5f01ee65da (Addr: tcp/127.0.0.1:16616) (DC: dc1)" -13:30.026 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e1683b60-5e08-8c30-e414-0e5f01ee65da.dc1 area=wan -13:30.026 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16652 network=tcp -13:30.027 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16612 network=tcp -13:30.027 [INFO] TestAgent: started state syncer -13:30.027 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16619 network=tcp -13:30.087 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:30.087 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16616 [Candidate]" term=2 -13:30.088 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:30.088 [DEBUG] TestAgent.server.raft: vote granted: from=e1683b60-5e08-8c30-e414-0e5f01ee65da term=2 tally=1 -13:30.088 [INFO] TestAgent.server.raft: election won: tally=1 -13:30.088 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16616 [Leader]" -13:30.088 [INFO] TestAgent.server: cluster leadership acquired -13:30.088 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16616 -13:30.089 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:30.089 [INFO] TestAgent.server: New leader elected: payload=Node-e1683b60-5e08-8c30-e414-0e5f01ee65da -13:30.091 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:30.091 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:30.091 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:30.091 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:30.091 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:30.091 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e1683b60-5e08-8c30-e414-0e5f01ee65da -13:30.091 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e1683b60-5e08-8c30-e414-0e5f01ee65da -13:30.092 [INFO] TestAgent.server: federation state anti-entropy synced -13:30.185 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:30.219 [INFO] TestAgent: Synced node info -13:30.219 [DEBUG] TestAgent: Node info in sync -13:30.406 [INFO] TestAgent: Requesting shutdown -13:30.406 [INFO] TestAgent.server: shutting down server -13:30.406 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:30.406 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:30.406 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:30.406 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:30.406 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:30.406 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:30.406 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:30.406 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:30.406 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:30.406 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:30.406 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:30.407Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.407Z [INFO] agent.router.manager: shutting down -13:30.407 [INFO] TestAgent: consul server down -13:30.407 [INFO] TestAgent: shutdown complete -13:30.407 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16652 network=tcp -13:30.407 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16652 network=udp -13:30.407 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16612 network=tcp -13:30.909 [INFO] TestAgent: Waiting for endpoints to shut down -13:30.909 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_idNotService (0.87s) +=== RUN TestAgent_Reload_ACLDeny/no_token +=== RUN TestAgent_Reload_ACLDeny/read-only_token +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:32.439Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:32.456Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:32.457 [INFO] bonnie.server.gateway_locator: will dial the primary datacenter through its mesh gateways +49:32.459 [INFO] bonnie.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:28774dcb-9ad1-b226-0139-1f524ca5a7ce Address:127.0.0.1:23449}]" +49:32.459 [INFO] bonnie.server.raft: entering follower state: follower="Node at 127.0.0.1:23449 [Follower]" leader= +49:32.460 [INFO] bonnie.server.serf.wan: serf: EventMemberJoin: bonnie.dc3 127.0.0.1 +49:32.460 [INFO] bonnie.server.serf.lan: serf: EventMemberJoin: bonnie 127.0.0.1 +2022-12-04T05:49:32.460Z [INFO] agent.router: Initializing LAN area manager +49:32.461 [INFO] bonnie.server: Adding LAN server: server="bonnie (Addr: tcp/127.0.0.1:23449) (DC: dc3)" +49:32.461 [INFO] bonnie.server: Handled event for server in area: event=member-join server=bonnie.dc3 area=wan +49:32.461 [INFO] bonnie: Started DNS server: address=127.0.0.1:23744 network=udp +49:32.461 [INFO] bonnie: Started DNS server: address=127.0.0.1:23744 network=tcp +49:32.461 [INFO] bonnie: Started HTTP server: address=127.0.0.1:23745 network=tcp +49:32.461 [INFO] bonnie: started state syncer +49:32.461 [INFO] bonnie: Started gRPC server: address=127.0.0.1:23736 network=tcp +49:32.461 [INFO] bonnie: Refreshing mesh gateways is supported for the following discovery methods: discovery_methods="aliyun aws gce mdns os packet" +49:32.461 [INFO] bonnie: Refreshing mesh gateways... +49:32.461 [INFO] bonnie.server.gateway_locator: updated fallback list of primary mesh gateways: mesh_gateways=[127.0.0.1:23412] +49:32.461 [INFO] bonnie: Refreshing mesh gateways completed +49:32.462 [INFO] bonnie: Retry join is supported for the following discovery methods: cluster=WAN discovery_methods="aliyun aws gce mdns os packet" +49:32.462 [INFO] bonnie: Joining cluster...: cluster=WAN +49:32.462 [INFO] bonnie: (WAN) joining: wan_addresses=[*.dc1/192.0.2.2] +49:32.475 [DEBUG] bonnie.server.memberlist.wan: memberlist: Initiating push/pull sync with: *.dc1 127.0.0.1:23412 +49:32.476 [INFO] bonnie.server.serf.wan: serf: EventMemberJoin: betty.dc2 127.0.0.1 +49:32.477 [INFO] bonnie.server.serf.wan: serf: EventMemberJoin: bob.dc1 127.0.0.1 +49:32.477 [INFO] bonnie: (WAN) joined: number_of_nodes=1 +49:32.477 [INFO] bonnie: Join cluster completed. Synced with initial agents: cluster=WAN num_agents=1 +49:32.477 [INFO] bonnie.server: Handled event for server in area: event=member-join server=betty.dc2 area=wan +49:32.477 [INFO] bonnie.server: Handled event for server in area: event=member-join server=bob.dc1 area=wan +49:32.507 [WARN] bonnie.server.raft: heartbeat timeout reached, starting election: last-leader= +49:32.507 [INFO] bonnie.server.raft: entering candidate state: node="Node at 127.0.0.1:23449 [Candidate]" term=2 +49:32.507 [DEBUG] bonnie.server.raft: votes: needed=1 +49:32.507 [DEBUG] bonnie.server.raft: vote granted: from=28774dcb-9ad1-b226-0139-1f524ca5a7ce term=2 tally=1 +49:32.507 [INFO] bonnie.server.raft: election won: tally=1 +49:32.507 [INFO] bonnie.server.raft: entering leader state: leader="Node at 127.0.0.1:23449 [Leader]" +49:32.507 [INFO] bonnie.server: cluster leadership acquired +49:32.508 [DEBUG] bonnie.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=0 found=true +49:32.508 [INFO] bonnie.server: New leader elected: payload=bonnie +49:32.525 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:4a:c6:ab:ef:63:c9:60:1a:51:7f:19:62:e3:e9:d9:0e:76:55:10:6e:74:24:69:28:a1:6c:b8:b9:8f:fd:89 is_primary=false +49:32.530 [INFO] bonnie.server.connect: received new intermediate certificate from primary datacenter +49:32.530 [INFO] bonnie.server.connect: updated root certificates from primary datacenter +49:32.530 [INFO] bonnie.server.connect: initialized secondary datacenter CA with provider: provider=consul +49:32.530 [INFO] bonnie.leader: started routine: routine="config entry replication" +49:32.530 [INFO] bonnie.leader: started routine: routine="federation state replication" +49:32.530 [INFO] bonnie.leader: started routine: routine="federation state anti-entropy" +49:32.530 [INFO] bonnie.leader: started routine: routine="secondary CA roots watch" +49:32.530 [INFO] bonnie.leader: started routine: routine="intention replication" +49:32.530 [INFO] bonnie.leader: started routine: routine="intermediate cert renew watch" +49:32.530 [INFO] bonnie.leader: started routine: routine="CA root pruning" +49:32.530 [DEBUG] bonnie.server: Skipping self join check for node since the cluster is too small: node=bonnie +49:32.530 [INFO] bonnie.server: member joined, marking health alive: member=bonnie +49:32.531 [DEBUG] bonnie.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 +49:32.532 [DEBUG] bonnie.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 +49:32.532 [INFO] bonnie.server.gateway_locator: will dial the primary datacenter using our local mesh gateways if possible +49:32.532 [DEBUG] bonnie.server: finished fetching remote objects: amount=2 +49:32.532 [DEBUG] bonnie.server: diffing replication state: local_amount=0 remote_amount=2 +49:32.532 [DEBUG] bonnie.server: diffed replication state: deletions=0 updates=2 +49:32.532 [DEBUG] bonnie.server: performing updates: updates=2 +49:32.533 [INFO] bonnie.server: federation state anti-entropy synced +49:32.533 [DEBUG] bonnie.server.replication.config_entry: finished fetching config entries: amount=0 +49:32.533 [DEBUG] bonnie.server.replication.config_entry: Config Entry replication: local=0 remote=0 +49:32.533 [DEBUG] bonnie.server.replication.config_entry: Config Entry replication: deletions=0 updates=0 +49:32.533 [DEBUG] bonnie.server.replication.config_entry: replication completed through remote index: index=1 +49:32.534 [INFO] bonnie.server: federation state anti-entropy synced +49:32.534 [INFO] bonnie.server.connect: updated root certificates from primary datacenter +49:32.543 [DEBUG] bonnie.server: finished updates +49:32.543 [DEBUG] bonnie.replication.federation_state: replication completed through remote index: index=17 +49:32.544 [DEBUG] bonnie.server: finished fetching remote objects: amount=3 +49:32.544 [DEBUG] bonnie.server: diffing replication state: local_amount=2 remote_amount=3 +49:32.544 [DEBUG] bonnie.server: diffed replication state: deletions=0 updates=1 +49:32.544 [DEBUG] bonnie.server: performing updates: updates=1 +49:32.544 [DEBUG] bonnie.server: finished updates +49:32.544 [DEBUG] bonnie.replication.federation_state: replication completed through remote index: index=21 +49:32.544 [INFO] bonnie.server: federation state anti-entropy synced +49:32.779 [INFO] bonnie: Synced node info +49:32.779 [INFO] bonnie: Synced service: service=mesh-gateway +49:32.781 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:23413 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +49:32.781 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:23413 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +49:32.781 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:23413 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +49:32.781 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +49:32.781 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:23413 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +49:32.781 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:23413 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +49:32.781 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +49:32.781 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:23413 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +49:32.781 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +49:32.781 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +49:32.781 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:23413 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +49:32.781 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:23413 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" +49:32.782 [INFO] bonnie.server: federation state anti-entropy synced +49:32.782 [DEBUG] bonnie.server: finished fetching remote objects: amount=3 +49:32.782 [DEBUG] bonnie.server: diffing replication state: local_amount=3 remote_amount=3 +49:32.782 [DEBUG] bonnie.server: diffed replication state: deletions=0 updates=1 +49:32.782 [DEBUG] bonnie.server: performing updates: updates=1 +49:32.782 [DEBUG] bonnie.server: finished updates +49:32.782 [DEBUG] bonnie.replication.federation_state: replication completed through remote index: index=22 +49:32.814 [DEBUG] bonnie: Skipping remote check since it is managed automatically: check=serfHealth +49:32.814 [DEBUG] bonnie: Node info in sync +49:32.814 [DEBUG] bonnie: Service in sync: service=mesh-gateway +49:32.814 [DEBUG] bonnie: Node info in sync +49:32.814 [DEBUG] bonnie: Service in sync: service=mesh-gateway +49:33.238 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: betty.dc2 +49:33.238 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: bonnie.dc3 +49:33.460 [ERROR] bonnie.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:23713: Remote DC has no server currently reachable +49:33.737 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: betty.dc2 +49:33.737 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: bonnie.dc3 +49:33.961 [ERROR] bonnie.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:23713: Remote DC has no server currently reachable +49:34.459 [INFO] bonnie.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:23412] local=[127.0.0.1:23412] +49:35.360 [DEBUG] bonnie: Skipping remote check since it is managed automatically: check=serfHealth +49:35.360 [DEBUG] bonnie: Node info in sync +49:35.360 [DEBUG] bonnie: Service in sync: service=mesh-gateway +49:37.399 [DEBUG] bonnie.server: finished fetching remote objects: amount=3 +49:37.399 [DEBUG] bonnie.server: diffing replication state: local_amount=3 remote_amount=3 +49:37.399 [DEBUG] bonnie.server: diffed replication state: deletions=0 updates=1 +49:37.399 [DEBUG] bonnie.server: performing updates: updates=1 +49:37.399 [DEBUG] bonnie.server: finished updates +49:37.399 [DEBUG] bonnie.replication.federation_state: replication completed through remote index: index=23 +49:37.471 [INFO] bonnie: Requesting shutdown +49:37.471 [INFO] bonnie.server: shutting down server +49:37.471 [DEBUG] bonnie.leader: stopping routine: routine="intention replication" +49:37.471 [DEBUG] bonnie.leader: stopping routine: routine="intermediate cert renew watch" +49:37.471 [DEBUG] bonnie.leader: stopping routine: routine="CA root pruning" +49:37.471 [DEBUG] bonnie.leader: stopping routine: routine="config entry replication" +49:37.471 [DEBUG] bonnie.leader: stopping routine: routine="federation state replication" +49:37.471 [DEBUG] bonnie.leader: stopping routine: routine="federation state anti-entropy" +49:37.471 [DEBUG] bonnie.leader: stopping routine: routine="secondary CA roots watch" +49:37.471 [WARN] bonnie.server.serf.lan: serf: Shutdown without a Leave +49:37.472 [DEBUG] bonnie.leader: stopped routine: routine="intermediate cert renew watch" +49:37.472 [DEBUG] bonnie.leader: stopped routine: routine="CA root pruning" +49:37.472 [DEBUG] bonnie.leader: stopped routine: routine="federation state anti-entropy" +49:37.472 [WARN] bonnie.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:37.477Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.477Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.477Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.477Z [INFO] agent.router.manager: shutting down +49:37.477 [INFO] bonnie: consul server down +49:37.477 [INFO] bonnie: shutdown complete +49:37.477 [INFO] bonnie: Stopping server: protocol=DNS address=127.0.0.1:23744 network=tcp +49:37.477 [INFO] bonnie: Stopping server: protocol=DNS address=127.0.0.1:23744 network=udp +49:37.478 [INFO] bonnie: Stopping server: protocol=HTTP address=127.0.0.1:23745 network=tcp +49:37.478 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:23413 datacenter=dc2 method=Internal.ServiceDump error="rpc error making call: EOF" +49:37.478 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:23497 datacenter=dc1 method=Intention.List error="rpc error making call: EOF" +49:37.478 [ERROR] bonnie.server.connect: error replicating intentions: routine="intention replication" error="rpc error making call: EOF" +49:37.478 [DEBUG] bonnie.leader: stopped routine: routine="intention replication" +49:37.478 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:23497 datacenter=dc1 method=Internal.ServiceDump error="rpc error making call: EOF" +49:37.478 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:23497 datacenter=dc1 method=ConnectCA.Roots error="rpc error making call: EOF" +49:37.478 [ERROR] bonnie.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: rpc error making call: EOF" +49:37.478 [DEBUG] bonnie.leader: stopped routine: routine="secondary CA roots watch" +49:37.478 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:23497 datacenter=dc1 method=ConfigEntry.ListAll error="rpc error making call: EOF" +49:37.478 [INFO] bonnie.server.replication.config_entry: stopped replication +49:37.478 [DEBUG] bonnie.leader: stopped routine: routine="config entry replication" +49:37.478 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:23497 datacenter=dc1 method=FederationState.List error="rpc error making call: EOF" +49:37.478 [WARN] bonnie.replication.federation_state: replication error (will retry if still leader): error="failed to retrieve federation states: rpc error making call: EOF" +49:37.478 [INFO] bonnie.replication.federation_state: stopped replication +49:37.478 [DEBUG] bonnie.leader: stopped routine: routine="federation state replication" +49:37.978 [INFO] bonnie: Waiting for endpoints to shut down +49:37.978 [INFO] bonnie: Endpoints down +2022-12-04T05:49:32.283Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:32.298Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:32.299 [INFO] betty.server.gateway_locator: will dial the primary datacenter through its mesh gateways +49:32.302 [INFO] betty.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b4ab91c7-ae6c-f68d-1cac-70758ef0458e Address:127.0.0.1:23413}]" +49:32.303 [INFO] betty.server.raft: entering follower state: follower="Node at 127.0.0.1:23413 [Follower]" leader= +49:32.303 [INFO] betty.server.serf.wan: serf: EventMemberJoin: betty.dc2 127.0.0.1 +49:32.303 [INFO] betty.server.serf.lan: serf: EventMemberJoin: betty 127.0.0.1 +2022-12-04T05:49:32.303Z [INFO] agent.router: Initializing LAN area manager +49:32.304 [INFO] betty.server: Adding LAN server: server="betty (Addr: tcp/127.0.0.1:23413) (DC: dc2)" +49:32.304 [INFO] betty.server: Handled event for server in area: event=member-join server=betty.dc2 area=wan +49:32.304 [INFO] betty: Started DNS server: address=127.0.0.1:23723 network=udp +49:32.304 [INFO] betty: Started DNS server: address=127.0.0.1:23723 network=tcp +49:32.305 [INFO] betty: Started HTTP server: address=127.0.0.1:23710 network=tcp +49:32.305 [INFO] betty: started state syncer +49:32.305 [INFO] betty: Started gRPC server: address=127.0.0.1:23777 network=tcp +49:32.305 [INFO] betty: Refreshing mesh gateways is supported for the following discovery methods: discovery_methods="aliyun aws gce mdns os packet" +49:32.305 [INFO] betty: Refreshing mesh gateways... +49:32.305 [INFO] betty.server.gateway_locator: updated fallback list of primary mesh gateways: mesh_gateways=[127.0.0.1:23412] +49:32.305 [INFO] betty: Refreshing mesh gateways completed +49:32.305 [INFO] betty: Retry join is supported for the following discovery methods: cluster=WAN discovery_methods="aliyun aws gce mdns os packet" +49:32.305 [INFO] betty: Joining cluster...: cluster=WAN +49:32.305 [INFO] betty: (WAN) joining: wan_addresses=[*.dc1/192.0.2.2] +49:32.332 [DEBUG] betty.server.memberlist.wan: memberlist: Initiating push/pull sync with: *.dc1 127.0.0.1:23412 +49:32.336 [INFO] betty.server.serf.wan: serf: EventMemberJoin: bob.dc1 127.0.0.1 +49:32.337 [INFO] betty: (WAN) joined: number_of_nodes=1 +49:32.337 [INFO] betty: Join cluster completed. Synced with initial agents: cluster=WAN num_agents=1 +49:32.339 [INFO] betty.server: Handled event for server in area: event=member-join server=bob.dc1 area=wan +49:32.369 [WARN] betty.server.raft: heartbeat timeout reached, starting election: last-leader= +49:32.369 [INFO] betty.server.raft: entering candidate state: node="Node at 127.0.0.1:23413 [Candidate]" term=2 +49:32.369 [DEBUG] betty.server.raft: votes: needed=1 +49:32.369 [DEBUG] betty.server.raft: vote granted: from=b4ab91c7-ae6c-f68d-1cac-70758ef0458e term=2 tally=1 +49:32.369 [INFO] betty.server.raft: election won: tally=1 +49:32.369 [INFO] betty.server.raft: entering leader state: leader="Node at 127.0.0.1:23413 [Leader]" +49:32.370 [INFO] betty.server: cluster leadership acquired +49:32.370 [INFO] betty.server: New leader elected: payload=betty +49:32.370 [DEBUG] betty.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=0 found=true +49:32.386 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:4a:c6:ab:ef:63:c9:60:1a:51:7f:19:62:e3:e9:d9:0e:76:55:10:6e:74:24:69:28:a1:6c:b8:b9:8f:fd:89 is_primary=false +49:32.393 [INFO] betty.server.connect: received new intermediate certificate from primary datacenter +49:32.394 [INFO] betty.server.connect: updated root certificates from primary datacenter +49:32.394 [INFO] betty.server.connect: initialized secondary datacenter CA with provider: provider=consul +49:32.394 [INFO] betty.leader: started routine: routine="config entry replication" +49:32.394 [INFO] betty.leader: started routine: routine="federation state replication" +49:32.394 [INFO] betty.leader: started routine: routine="federation state anti-entropy" +49:32.394 [INFO] betty.leader: started routine: routine="secondary CA roots watch" +49:32.394 [INFO] betty.leader: started routine: routine="intention replication" +49:32.394 [INFO] betty.leader: started routine: routine="intermediate cert renew watch" +49:32.394 [INFO] betty.leader: started routine: routine="CA root pruning" +49:32.394 [DEBUG] betty.server: Skipping self join check for node since the cluster is too small: node=betty +49:32.394 [INFO] betty.server: member joined, marking health alive: member=betty +49:32.396 [DEBUG] betty.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 +49:32.396 [DEBUG] betty.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 +49:32.399 [DEBUG] betty.server.replication.config_entry: finished fetching config entries: amount=0 +49:32.399 [DEBUG] betty.server.replication.config_entry: Config Entry replication: local=0 remote=0 +49:32.399 [DEBUG] betty.server.replication.config_entry: Config Entry replication: deletions=0 updates=0 +49:32.399 [DEBUG] betty.server.replication.config_entry: replication completed through remote index: index=1 +49:32.399 [INFO] betty.server.gateway_locator: will dial the primary datacenter using our local mesh gateways if possible +49:32.399 [DEBUG] betty.server: finished fetching remote objects: amount=1 +49:32.399 [DEBUG] betty.server: diffing replication state: local_amount=0 remote_amount=1 +49:32.399 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 +49:32.399 [DEBUG] betty.server: performing updates: updates=1 +49:32.400 [DEBUG] betty.server: finished updates +49:32.400 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=14 +49:32.401 [INFO] betty.server: federation state anti-entropy synced +49:32.402 [DEBUG] betty.server: finished fetching remote objects: amount=2 +49:32.402 [DEBUG] betty.server: diffing replication state: local_amount=1 remote_amount=2 +49:32.402 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 +49:32.402 [DEBUG] betty.server: performing updates: updates=1 +49:32.402 [DEBUG] betty.server: finished updates +49:32.402 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=16 +49:32.402 [INFO] betty.server.connect: updated root certificates from primary datacenter +49:32.402 [INFO] betty.server: federation state anti-entropy synced +49:32.403 [DEBUG] betty.server: finished fetching remote objects: amount=2 +49:32.403 [DEBUG] betty.server: diffing replication state: local_amount=2 remote_amount=2 +49:32.403 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 +49:32.403 [DEBUG] betty.server: performing updates: updates=1 +49:32.403 [DEBUG] betty.server: finished updates +49:32.403 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=17 +49:32.533 [DEBUG] betty.server: finished fetching remote objects: amount=3 +49:32.533 [DEBUG] betty.server: diffing replication state: local_amount=2 remote_amount=3 +49:32.533 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 +49:32.533 [DEBUG] betty.server: performing updates: updates=1 +49:32.534 [DEBUG] betty.server: finished updates +49:32.534 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=19 +49:32.534 [DEBUG] betty.server: finished fetching remote objects: amount=3 +49:32.534 [DEBUG] betty.server: diffing replication state: local_amount=3 remote_amount=3 +49:32.534 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 +49:32.534 [DEBUG] betty.server: performing updates: updates=1 +49:32.535 [DEBUG] betty.server: finished updates +49:32.535 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=20 +49:32.538 [DEBUG] betty: Skipping remote check since it is managed automatically: check=serfHealth +49:32.538 [INFO] betty: Synced node info +49:32.538 [DEBUG] betty: Node info in sync +49:32.758 [DEBUG] betty: Node info in sync +49:32.778 [INFO] betty: Synced service: service=mesh-gateway +49:32.789 [INFO] betty.server.serf.wan: serf: EventMemberJoin: bonnie.dc3 127.0.0.1 +49:32.789 [INFO] betty.server: Handled event for server in area: event=member-join server=bonnie.dc3 area=wan +49:32.841 [ERROR] betty.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:23748: Remote DC has no server currently reachable +49:33.148 [DEBUG] betty: Skipping remote check since it is managed automatically: check=serfHealth +49:33.148 [DEBUG] betty: Node info in sync +49:33.148 [DEBUG] betty: Service in sync: service=mesh-gateway +49:33.148 [DEBUG] betty: Node info in sync +49:33.148 [DEBUG] betty: Service in sync: service=mesh-gateway +49:33.237 [DEBUG] betty.server.serf.wan: serf: messageJoinType: betty.dc2 +49:33.237 [DEBUG] betty.server.serf.wan: serf: messageJoinType: bonnie.dc3 +49:33.304 [ERROR] betty.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:23748: Remote DC has no server currently reachable +49:33.397 [DEBUG] betty.server: finished fetching remote objects: amount=3 +49:33.397 [DEBUG] betty.server: diffing replication state: local_amount=3 remote_amount=3 +49:33.397 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 +49:33.397 [DEBUG] betty.server: performing updates: updates=1 +49:33.398 [DEBUG] betty.server: finished updates +49:33.398 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=22 +49:33.737 [DEBUG] betty.server.serf.wan: serf: messageJoinType: betty.dc2 +49:33.737 [DEBUG] betty.server.serf.wan: serf: messageJoinType: bonnie.dc3 +49:33.810 [ERROR] betty.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:23748: Remote DC has no server currently reachable +49:34.306 [INFO] betty.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:23412] local=[] +49:37.304 [ERROR] betty.server.memberlist.wan: memberlist: Failed to send ping: Remote DC has no server currently reachable +49:37.399 [DEBUG] betty.server: finished fetching remote objects: amount=3 +49:37.399 [DEBUG] betty.server: diffing replication state: local_amount=3 remote_amount=3 +49:37.399 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 +49:37.399 [DEBUG] betty.server: performing updates: updates=1 +49:37.399 [INFO] betty.server: federation state anti-entropy synced +49:37.399 [DEBUG] betty.server: finished updates +49:37.399 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=23 +49:37.399 [INFO] betty.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:23412] local=[127.0.0.1:23412] +49:37.978 [INFO] betty: Requesting shutdown +49:37.978 [INFO] betty.server: shutting down server +49:37.978 [DEBUG] betty.leader: stopping routine: routine="config entry replication" +49:37.978 [DEBUG] betty.leader: stopping routine: routine="federation state replication" +49:37.978 [DEBUG] betty.leader: stopping routine: routine="federation state anti-entropy" +49:37.978 [DEBUG] betty.leader: stopping routine: routine="secondary CA roots watch" +49:37.978 [DEBUG] betty.leader: stopping routine: routine="intention replication" +49:37.978 [DEBUG] betty.leader: stopping routine: routine="intermediate cert renew watch" +49:37.978 [DEBUG] betty.leader: stopping routine: routine="CA root pruning" +49:37.978 [WARN] betty.server.serf.lan: serf: Shutdown without a Leave +49:37.978 [DEBUG] betty.leader: stopped routine: routine="federation state anti-entropy" +49:37.978 [DEBUG] betty.leader: stopped routine: routine="intermediate cert renew watch" +49:37.978 [DEBUG] betty.leader: stopped routine: routine="CA root pruning" +49:37.978 [WARN] betty.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:37.979Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.979Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.979Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:37.979Z [INFO] agent.router.manager: shutting down +49:37.979 [INFO] betty: consul server down +49:37.979 [INFO] betty: shutdown complete +49:37.979 [INFO] betty: Stopping server: protocol=DNS address=127.0.0.1:23723 network=tcp +49:37.979 [INFO] betty: Stopping server: protocol=DNS address=127.0.0.1:23723 network=udp +49:37.979 [INFO] betty: Stopping server: protocol=HTTP address=127.0.0.1:23710 network=tcp +49:37.979 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:23497 datacenter=dc1 method=Intention.List error="rpc error making call: EOF" +49:37.979 [ERROR] betty.server.connect: error replicating intentions: routine="intention replication" error="rpc error making call: EOF" +49:37.979 [DEBUG] betty.leader: stopped routine: routine="intention replication" +49:37.979 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:23497 datacenter=dc1 method=ConnectCA.Roots error="rpc error making call: EOF" +49:37.979 [ERROR] betty.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: rpc error making call: EOF" +49:37.979 [DEBUG] betty.leader: stopped routine: routine="secondary CA roots watch" +49:37.979 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:23497 datacenter=dc1 method=FederationState.List error="rpc error making call: EOF" +49:37.979 [WARN] betty.replication.federation_state: replication error (will retry if still leader): error="failed to retrieve federation states: rpc error making call: EOF" +49:37.979 [INFO] betty.replication.federation_state: stopped replication +49:37.979 [DEBUG] betty.leader: stopped routine: routine="federation state replication" +49:37.979 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:23497 datacenter=dc1 method=ConfigEntry.ListAll error="rpc error making call: EOF" +49:37.979 [INFO] betty.server.replication.config_entry: stopped replication +49:37.979 [DEBUG] betty.leader: stopped routine: routine="config entry replication" +49:37.979 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:23497 datacenter=dc1 method=Internal.ServiceDump error="rpc error making call: EOF" +49:38.479 [INFO] betty: Waiting for endpoints to shut down +49:38.479 [INFO] betty: Endpoints down +2022-12-04T05:49:30.219Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:30.233Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:30.235 [INFO] bob.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c76cc722-c0d1-cbbd-30e9-80e967bbe1d6 Address:127.0.0.1:23497}]" +49:30.235 [INFO] bob.server.raft: entering follower state: follower="Node at 127.0.0.1:23497 [Follower]" leader= +49:30.235 [INFO] bob.server.serf.wan: serf: EventMemberJoin: bob.dc1 127.0.0.1 +49:30.236 [INFO] bob.server.serf.lan: serf: EventMemberJoin: bob 127.0.0.1 +2022-12-04T05:49:30.236Z [INFO] agent.router: Initializing LAN area manager +49:30.236 [INFO] bob.server: Adding LAN server: server="bob (Addr: tcp/127.0.0.1:23497) (DC: dc1)" +49:30.236 [INFO] bob: Started DNS server: address=127.0.0.1:23450 network=udp +49:30.236 [INFO] bob.server: Handled event for server in area: event=member-join server=bob.dc1 area=wan +49:30.236 [INFO] bob: Started DNS server: address=127.0.0.1:23450 network=tcp +49:30.237 [INFO] bob: Started HTTP server: address=127.0.0.1:23493 network=tcp +49:30.237 [INFO] bob: started state syncer +49:30.237 [INFO] bob: Started gRPC server: address=127.0.0.1:23512 network=tcp +49:30.284 [WARN] bob.server.raft: heartbeat timeout reached, starting election: last-leader= +49:30.284 [INFO] bob.server.raft: entering candidate state: node="Node at 127.0.0.1:23497 [Candidate]" term=2 +49:30.285 [DEBUG] bob.server.raft: votes: needed=1 +49:30.285 [DEBUG] bob.server.raft: vote granted: from=c76cc722-c0d1-cbbd-30e9-80e967bbe1d6 term=2 tally=1 +49:30.285 [INFO] bob.server.raft: election won: tally=1 +49:30.285 [INFO] bob.server.raft: entering leader state: leader="Node at 127.0.0.1:23497 [Leader]" +49:30.285 [INFO] bob.server: cluster leadership acquired +49:30.285 [DEBUG] bob.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23497 +49:30.286 [INFO] bob.server: New leader elected: payload=bob +49:30.286 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:30.288 [INFO] bob.server.connect: initialized primary datacenter CA with provider: provider=consul +49:30.288 [INFO] bob.leader: started routine: routine="federation state anti-entropy" +49:30.288 [INFO] bob.leader: started routine: routine="federation state pruning" +49:30.288 [INFO] bob.leader: started routine: routine="intermediate cert renew watch" +49:30.288 [INFO] bob.leader: started routine: routine="CA root pruning" +49:30.288 [DEBUG] bob.server: Skipping self join check for node since the cluster is too small: node=bob +49:30.288 [INFO] bob.server: member joined, marking health alive: member=bob +49:30.290 [INFO] bob.server: federation state anti-entropy synced +49:30.446 [DEBUG] bob: Skipping remote check since it is managed automatically: check=serfHealth +49:30.446 [INFO] bob: Synced node info +49:30.562 [DEBUG] bob: Node info in sync +49:30.563 [INFO] bob: Synced service: service=mesh-gateway +49:30.564 [INFO] bob.server: federation state anti-entropy synced +49:32.235 [INFO] bob.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:23412] local=[127.0.0.1:23412] +49:32.334 [DEBUG] bob.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:57480 +49:32.334 [INFO] bob.server.serf.wan: serf: EventMemberJoin: betty.dc2 127.0.0.1 +49:32.334 [INFO] bob.server: Handled event for server in area: event=member-join server=betty.dc2 area=wan +49:32.476 [DEBUG] bob.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:57484 +49:32.476 [INFO] bob.server.serf.wan: serf: EventMemberJoin: bonnie.dc3 127.0.0.1 +49:32.476 [INFO] bob.server: Handled event for server in area: event=member-join server=bonnie.dc3 area=wan +49:32.483 [DEBUG] bob: Skipping remote check since it is managed automatically: check=serfHealth +49:32.483 [DEBUG] bob: Node info in sync +49:32.483 [DEBUG] bob: Service in sync: service=mesh-gateway +49:32.483 [DEBUG] bob: Node info in sync +49:32.483 [DEBUG] bob: Service in sync: service=mesh-gateway +49:32.843 [DEBUG] bob.server.serf.wan: serf: messageJoinType: betty.dc2 +49:32.985 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 +49:33.305 [DEBUG] bob.server.serf.wan: serf: messageJoinType: betty.dc2 +49:33.305 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 +49:33.461 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 +49:33.461 [DEBUG] bob.server.serf.wan: serf: messageJoinType: betty.dc2 +49:33.814 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 +49:33.961 [DEBUG] bob.server.serf.wan: serf: messageJoinType: betty.dc2 +49:38.479 [INFO] bob: Requesting shutdown +49:38.479 [INFO] bob.server: shutting down server +49:38.479 [DEBUG] bob.leader: stopping routine: routine="federation state anti-entropy" +49:38.479 [DEBUG] bob.leader: stopping routine: routine="federation state pruning" +49:38.479 [DEBUG] bob.leader: stopping routine: routine="intermediate cert renew watch" +49:38.479 [DEBUG] bob.leader: stopping routine: routine="CA root pruning" +49:38.479 [WARN] bob.server.serf.lan: serf: Shutdown without a Leave +49:38.479 [ERROR] bob.server: error performing anti-entropy sync of federation state: error="context canceled" +49:38.479 [DEBUG] bob.leader: stopped routine: routine="federation state anti-entropy" +49:38.479 [DEBUG] bob.leader: stopped routine: routine="federation state pruning" +49:38.479 [DEBUG] bob.leader: stopped routine: routine="intermediate cert renew watch" +49:38.479 [DEBUG] bob.leader: stopped routine: routine="CA root pruning" +49:38.480 [WARN] bob.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:38.480Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.480Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.480Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.480Z [INFO] agent.router.manager: shutting down +49:38.480 [INFO] bob: consul server down +49:38.480 [INFO] bob: shutdown complete +49:38.480 [INFO] bob: Stopping server: protocol=DNS address=127.0.0.1:23450 network=tcp +49:38.480 [INFO] bob: Stopping server: protocol=DNS address=127.0.0.1:23450 network=udp +49:38.480 [INFO] bob: Stopping server: protocol=HTTP address=127.0.0.1:23493 network=tcp +49:38.980 [INFO] bob: Waiting for endpoints to shut down +49:38.980 [INFO] bob: Endpoints down +--- PASS: TestAgent_JoinWAN_viaMeshGateway (8.77s) + --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc1_to_dc2 (0.02s) + --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc1_to_dc3 (0.02s) + --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc2_to_dc1 (0.00s) + --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc2_to_dc3 (0.02s) + --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc3_to_dc1 (0.00s) + --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc3_to_dc2 (0.00s) === CONT TestAgent_Metrics_ACLDeny ---- PASS: TestAgentConnectAuthorize_allow (0.97s) +49:38.035 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-de4a228b-ec61-a225-d53d-4b99dbf09b67 127.0.0.1 +2022-12-04T05:49:38.035Z [INFO] agent.router: Initializing LAN area manager +49:38.035 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23420 network=udp +49:38.035 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23420 network=tcp +49:38.035 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23337 network=tcp +49:38.036 [INFO] TestAgent: started state syncer +49:38.036 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23328 network=tcp +2022-12-04T05:49:38.036Z [WARN] agent.router.manager: No servers available +49:38.036 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +49:38.036 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Stream connection from=127.0.0.1:35700 +49:38.037 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-e15af33b-fbb8-a81c-51a1-d8230768c918 127.0.0.1 +49:38.037 [INFO] TestAgent: Requesting shutdown +49:38.037 [INFO] TestAgent.client: adding server: server="Node-e15af33b-fbb8-a81c-51a1-d8230768c918 (Addr: tcp/127.0.0.1:23376) (DC: dc1)" +49:38.037 [INFO] TestAgent.client: New leader elected: payload=Node-e15af33b-fbb8-a81c-51a1-d8230768c918 +49:38.037 [INFO] TestAgent.client: shutting down client +49:38.037 [WARN] TestAgent.client.serf.lan: serf: Shutdown without a Leave +49:38.037 [INFO] TestAgent: consul client down +49:38.037 [INFO] TestAgent: shutdown complete +49:38.037 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23420 network=tcp +49:38.037 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23420 network=udp +49:38.037 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23337 network=tcp +49:38.537 [INFO] TestAgent: Waiting for endpoints to shut down +49:38.537 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:37.855Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.867Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.869 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e15af33b-fbb8-a81c-51a1-d8230768c918 Address:127.0.0.1:23376}]" +49:37.869 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23376 [Follower]" leader= +49:37.869 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e15af33b-fbb8-a81c-51a1-d8230768c918.dc1 127.0.0.1 +49:37.870 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e15af33b-fbb8-a81c-51a1-d8230768c918 127.0.0.1 +2022-12-04T05:49:37.870Z [INFO] agent.router: Initializing LAN area manager +49:37.870 [INFO] TestAgent.server: Adding LAN server: server="Node-e15af33b-fbb8-a81c-51a1-d8230768c918 (Addr: tcp/127.0.0.1:23376) (DC: dc1)" +49:37.870 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e15af33b-fbb8-a81c-51a1-d8230768c918.dc1 area=wan +49:37.870 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23336 network=udp +49:37.870 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23336 network=tcp +49:37.870 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23351 network=tcp +49:37.870 [INFO] TestAgent: started state syncer +49:37.870 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23286 network=tcp +49:37.933 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:37.933 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23376 [Candidate]" term=2 +49:37.933 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:37.933 [DEBUG] TestAgent.server.raft: vote granted: from=e15af33b-fbb8-a81c-51a1-d8230768c918 term=2 tally=1 +49:37.933 [INFO] TestAgent.server.raft: election won: tally=1 +49:37.933 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23376 [Leader]" +49:37.934 [INFO] TestAgent.server: cluster leadership acquired +49:37.934 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23376 +49:37.934 [INFO] TestAgent.server: New leader elected: payload=Node-e15af33b-fbb8-a81c-51a1-d8230768c918 +49:37.935 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:37.937 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:37.937 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:37.937 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:37.937 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:37.937 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:37.937 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e15af33b-fbb8-a81c-51a1-d8230768c918 +49:37.937 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e15af33b-fbb8-a81c-51a1-d8230768c918 +49:37.938 [INFO] TestAgent.server: federation state anti-entropy synced +49:38.036 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:23353] +49:38.036 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:23353 +49:38.037 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-de4a228b-ec61-a225-d53d-4b99dbf09b67 127.0.0.1 +49:38.037 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +49:38.037 [INFO] TestAgent.server: member joined, marking health alive: member=Node-de4a228b-ec61-a225-d53d-4b99dbf09b67 +49:38.304 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:38.305 [INFO] TestAgent: Synced node info +49:38.537 [INFO] TestAgent: Requesting shutdown +49:38.537 [INFO] TestAgent.server: shutting down server +49:38.537 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:38.537 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:38.537 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:38.537 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:38.537 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:38.537 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:38.537 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:38.537 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:38.537 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:38.537 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:38.538 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:38.538Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.538Z [INFO] agent.router.manager: shutting down +49:38.538 [INFO] TestAgent: consul server down +49:38.538 [INFO] TestAgent: shutdown complete +49:38.538 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23336 network=tcp +49:38.538 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23336 network=udp +49:38.538 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23351 network=tcp +49:39.038 [INFO] TestAgent: Waiting for endpoints to shut down +49:39.038 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_JoinLANNotify (1.20s) === CONT TestAgent_Self_ACLDeny -2021-10-31T23:13:30.218Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:30.249Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:30.251 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:26fbb898-6ec2-6449-8982-b9db3c7141ca Address:127.0.0.1:16699}]" -13:30.252 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-26fbb898-6ec2-6449-8982-b9db3c7141ca.dc1 127.0.0.1 -13:30.252 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-26fbb898-6ec2-6449-8982-b9db3c7141ca 127.0.0.1 -2021-10-31T23:13:30.252Z [INFO] agent.router: Initializing LAN area manager -13:30.253 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16666 network=udp -13:30.253 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16699 [Follower]" leader= -13:30.253 [INFO] TestAgent.server: Adding LAN server: server="Node-26fbb898-6ec2-6449-8982-b9db3c7141ca (Addr: tcp/127.0.0.1:16699) (DC: dc1)" -13:30.253 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-26fbb898-6ec2-6449-8982-b9db3c7141ca.dc1 area=wan -13:30.253 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16666 network=tcp -13:30.278 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16695 network=tcp -13:30.278 [INFO] TestAgent: started state syncer -13:30.279 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16651 network=tcp -13:30.308 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:30.308 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16699 [Candidate]" term=2 -13:30.309 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:30.309 [DEBUG] TestAgent.server.raft: vote granted: from=26fbb898-6ec2-6449-8982-b9db3c7141ca term=2 tally=1 -13:30.309 [INFO] TestAgent.server.raft: election won: tally=1 -13:30.309 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16699 [Leader]" -13:30.309 [INFO] TestAgent.server: cluster leadership acquired -13:30.310 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16699 -13:30.311 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:30.311 [INFO] TestAgent.server: New leader elected: payload=Node-26fbb898-6ec2-6449-8982-b9db3c7141ca -13:30.321 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:30.321 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:30.321 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:30.321 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:30.321 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:30.322 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-26fbb898-6ec2-6449-8982-b9db3c7141ca -13:30.322 [INFO] TestAgent.server: member joined, marking health alive: member=Node-26fbb898-6ec2-6449-8982-b9db3c7141ca -13:30.323 [INFO] TestAgent.server: federation state anti-entropy synced -13:30.494 [INFO] TestAgent: Requesting shutdown -13:30.494 [INFO] TestAgent.server: shutting down server -13:30.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:30.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:30.494 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:30.494 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:30.494 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:30.494 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:30.494 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:30.494 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:30.494 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:30.494 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:30.494 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:30.494 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:30.494Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.494Z [INFO] agent.router.manager: shutting down -13:30.494 [INFO] TestAgent: consul server down -13:30.494 [INFO] TestAgent: shutdown complete -13:30.494 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16666 network=tcp -13:30.495 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16666 network=udp -13:30.495 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16695 network=tcp -13:31.002 [INFO] TestAgent: Waiting for endpoints to shut down -13:31.002 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_idInvalidFormat (0.90s) -=== CONT TestAgent_Self -=== RUN TestAgent_Self/normal -2021-10-31T23:13:29.784Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:29.843Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:29.866 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4bc60095-5354-0612-6340-9796fab544c1 Address:127.0.0.1:16657}]" -13:29.867 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16657 [Follower]" leader= -13:29.867 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4bc60095-5354-0612-6340-9796fab544c1.dc1 127.0.0.1 -13:29.868 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4bc60095-5354-0612-6340-9796fab544c1 127.0.0.1 -2021-10-31T23:13:29.868Z [INFO] agent.router: Initializing LAN area manager -13:29.869 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16624 network=udp -13:29.869 [INFO] TestAgent.server: Adding LAN server: server="Node-4bc60095-5354-0612-6340-9796fab544c1 (Addr: tcp/127.0.0.1:16657) (DC: dc1)" -13:29.869 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4bc60095-5354-0612-6340-9796fab544c1.dc1 area=wan -13:29.869 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16624 network=tcp -13:29.871 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16653 network=tcp -13:29.871 [INFO] TestAgent: started state syncer -13:29.871 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16630 network=tcp -13:29.936 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:29.936 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16657 [Candidate]" term=2 -13:29.937 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:29.937 [DEBUG] TestAgent.server.raft: vote granted: from=4bc60095-5354-0612-6340-9796fab544c1 term=2 tally=1 -13:29.937 [INFO] TestAgent.server.raft: election won: tally=1 -13:29.937 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16657 [Leader]" -13:29.937 [INFO] TestAgent.server: cluster leadership acquired -13:29.937 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16657 -13:29.937 [INFO] TestAgent.server: New leader elected: payload=Node-4bc60095-5354-0612-6340-9796fab544c1 -13:29.942 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:29.944 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:29.944 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:29.944 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:29.944 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:29.944 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:29.944 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4bc60095-5354-0612-6340-9796fab544c1 -13:29.944 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4bc60095-5354-0612-6340-9796fab544c1 -13:29.945 [INFO] TestAgent.server: federation state anti-entropy synced -13:29.991 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:29.991 [INFO] TestAgent: Synced node info -13:29.992 [INFO] TestAgent: Synced service: service=redis -13:29.992 [DEBUG] TestAgent: Check in sync: check=redis-check -13:29.992 [INFO] TestAgent: Synced check: check=node-check -13:30.151 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:30.151 [DEBUG] TestAgent: Node info in sync -13:30.151 [DEBUG] TestAgent: Service in sync: service=redis -13:30.151 [DEBUG] TestAgent: Check in sync: check=redis-check -13:30.151 [DEBUG] TestAgent: Check in sync: check=node-check -13:30.493 [DEBUG] TestAgent: Registered node: node="Node-level check" -13:30.493 [DEBUG] TestAgent: Registered node: node="Serf Health Status" -13:30.493 [DEBUG] TestAgent: Registered node: node="Service-level check" -13:30.493 [INFO] TestAgent: Sync in progress: iteration=1 -13:30.493 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:30.493 [DEBUG] TestAgent: Node info in sync -13:30.493 [DEBUG] TestAgent: Service in sync: service=redis -13:30.493 [DEBUG] TestAgent: Check in sync: check=redis-check -13:30.493 [DEBUG] TestAgent: Check in sync: check=node-check -13:30.493 [INFO] TestAgent: Sync in progress: iteration=2 -13:30.494 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:30.494 [DEBUG] TestAgent: Node info in sync -13:30.494 [DEBUG] TestAgent: Service in sync: service=redis -13:30.494 [DEBUG] TestAgent: Check in sync: check=redis-check -13:30.494 [DEBUG] TestAgent: Check in sync: check=node-check -13:30.494 [INFO] TestAgent: Sync in progress: iteration=3 -13:30.494 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:30.494 [DEBUG] TestAgent: Node info in sync -13:30.494 [DEBUG] TestAgent: Service in sync: service=redis -13:30.494 [DEBUG] TestAgent: Check in sync: check=redis-check -13:30.494 [DEBUG] TestAgent: Check in sync: check=node-check -13:30.494 [INFO] TestAgent: Sync in progress: iteration=4 -13:30.494 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:30.494 [DEBUG] TestAgent: Node info in sync -13:30.494 [DEBUG] TestAgent: Service in sync: service=redis -13:30.494 [DEBUG] TestAgent: Check in sync: check=redis-check -13:30.494 [DEBUG] TestAgent: Check in sync: check=node-check -13:30.494 [INFO] TestAgent: Sync in progress: iteration=5 -13:30.494 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:30.494 [DEBUG] TestAgent: Node info in sync -13:30.494 [DEBUG] TestAgent: Service in sync: service=redis -13:30.494 [DEBUG] TestAgent: Check in sync: check=redis-check -13:30.494 [DEBUG] TestAgent: Check in sync: check=node-check -13:30.494 [INFO] TestAgent: Sync in progress: iteration=6 -13:30.494 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:30.494 [DEBUG] TestAgent: Node info in sync -13:30.494 [DEBUG] TestAgent: Service in sync: service=redis -13:30.494 [DEBUG] TestAgent: Check in sync: check=node-check -13:30.494 [DEBUG] TestAgent: Check in sync: check=redis-check -13:30.494 [INFO] TestAgent: Sync in progress: iteration=7 -13:30.494 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:30.494 [DEBUG] TestAgent: Node info in sync -13:30.494 [DEBUG] TestAgent: Service in sync: service=redis -13:30.494 [DEBUG] TestAgent: Check in sync: check=redis-check -13:30.494 [DEBUG] TestAgent: Check in sync: check=node-check -13:30.494 [INFO] TestAgent: Sync in progress: iteration=8 -13:30.494 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:30.494 [DEBUG] TestAgent: Node info in sync -13:30.494 [DEBUG] TestAgent: Service in sync: service=redis -13:30.494 [DEBUG] TestAgent: Check in sync: check=redis-check -13:30.494 [DEBUG] TestAgent: Check in sync: check=node-check -13:30.494 [INFO] TestAgent: Sync in progress: iteration=9 -13:30.494 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:30.494 [DEBUG] TestAgent: Node info in sync -13:30.494 [DEBUG] TestAgent: Service in sync: service=redis -13:30.494 [DEBUG] TestAgent: Check in sync: check=redis-check -13:30.494 [DEBUG] TestAgent: Check in sync: check=node-check -13:30.494 [INFO] TestAgent: Sync in progress: iteration=10 -13:30.494 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:30.494 [DEBUG] TestAgent: Node info in sync -13:30.494 [DEBUG] TestAgent: Service in sync: service=redis -13:30.494 [DEBUG] TestAgent: Check in sync: check=redis-check -13:30.494 [DEBUG] TestAgent: Check in sync: check=node-check -13:30.494 [INFO] TestAgent: Requesting shutdown -13:30.494 [INFO] TestAgent.server: shutting down server -13:30.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:30.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:30.494 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:30.494 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:30.494 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:30.494 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:30.494 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:30.494 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:30.494 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:30.494 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:30.494 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:30.495Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.495Z [INFO] agent.router.manager: shutting down -13:30.495 [INFO] TestAgent: consul server down -13:30.495 [INFO] TestAgent: shutdown complete -13:30.495 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16624 network=tcp -13:30.495 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16624 network=udp -13:30.495 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16653 network=tcp -13:31.005 [INFO] TestAgent: Waiting for endpoints to shut down -13:31.005 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_IndexChurn (2.66s) - --- PASS: TestAgent_IndexChurn/no_tags (1.41s) - --- PASS: TestAgent_IndexChurn/with_tags (1.25s) +[WARN] freeport: 10 out of 48 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:38.231Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:38.243Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:38.245 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1e228491-781e-45b5-0cc3-317d70b2536e Address:127.0.0.1:23320}]" +49:38.245 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23320 [Follower]" leader= +49:38.246 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1e228491-781e-45b5-0cc3-317d70b2536e.dc1 127.0.0.1 +49:38.246 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1e228491-781e-45b5-0cc3-317d70b2536e 127.0.0.1 +2022-12-04T05:49:38.246Z [INFO] agent.router: Initializing LAN area manager +49:38.246 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23371 network=udp +49:38.246 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1e228491-781e-45b5-0cc3-317d70b2536e.dc1 area=wan +49:38.246 [INFO] TestAgent.server: Adding LAN server: server="Node-1e228491-781e-45b5-0cc3-317d70b2536e (Addr: tcp/127.0.0.1:23320) (DC: dc1)" +49:38.246 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23371 network=tcp +49:38.247 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23393 network=tcp +49:38.247 [INFO] TestAgent: started state syncer +49:38.247 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23321 network=tcp +49:38.308 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:38.308 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23320 [Candidate]" term=2 +49:38.308 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:38.308 [DEBUG] TestAgent.server.raft: vote granted: from=1e228491-781e-45b5-0cc3-317d70b2536e term=2 tally=1 +49:38.308 [INFO] TestAgent.server.raft: election won: tally=1 +49:38.308 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23320 [Leader]" +49:38.308 [INFO] TestAgent.server: cluster leadership acquired +49:38.308 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23320 +49:38.308 [INFO] TestAgent.server: New leader elected: payload=Node-1e228491-781e-45b5-0cc3-317d70b2536e +49:38.309 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:38.310 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:38.310 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:38.310 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:38.310 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:38.310 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:38.310 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1e228491-781e-45b5-0cc3-317d70b2536e +49:38.310 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1e228491-781e-45b5-0cc3-317d70b2536e +49:38.311 [INFO] TestAgent.server: federation state anti-entropy synced +49:38.614 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:38.614 [INFO] TestAgent: Synced node info +49:38.614 [DEBUG] TestAgent: Node info in sync +49:38.659 [INFO] TestAgent: Requesting shutdown +49:38.659 [INFO] TestAgent.server: shutting down server +49:38.659 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:38.659 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:38.659 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:38.659 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:38.659 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:38.659 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:38.659 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:38.659 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:38.659 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:38.659 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:38.659 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:38.659Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.659Z [INFO] agent.router.manager: shutting down +49:38.659 [INFO] TestAgent: consul server down +49:38.659 [INFO] TestAgent: shutdown complete +49:38.660 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23371 network=tcp +49:38.660 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23371 network=udp +49:38.660 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23393 network=tcp +49:39.160 [INFO] TestAgent: Waiting for endpoints to shut down +49:39.160 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Members_WAN (0.94s) +=== CONT TestACL_vetServiceUpdate +2022-12-04T05:49:39.171Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:39.171Z [WARN] agent: bootstrap = true: do not enable unless necessary +--- PASS: TestACL_vetServiceUpdate (0.01s) === CONT TestAgent_Checks_ACLFilter -=== RUN TestAgent_Reload_ACLDeny/no_token -=== RUN TestAgent_Reload_ACLDeny/read-only_token +2022-12-04T05:49:38.698Z [WARN] TestAgent_Reload: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:38.595Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:38.609Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:38.610 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b0c98fae-dc09-e706-c66e-bbd93c2a00d7 Address:127.0.0.1:23411}]" +49:38.611 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b0c98fae-dc09-e706-c66e-bbd93c2a00d7.dc1 127.0.0.1 +49:38.612 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b0c98fae-dc09-e706-c66e-bbd93c2a00d7 127.0.0.1 +2022-12-04T05:49:38.612Z [INFO] agent.router: Initializing LAN area manager +49:38.612 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23411 [Follower]" leader= +49:38.612 [INFO] TestAgent.server: Adding LAN server: server="Node-b0c98fae-dc09-e706-c66e-bbd93c2a00d7 (Addr: tcp/127.0.0.1:23411) (DC: dc1)" +49:38.612 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b0c98fae-dc09-e706-c66e-bbd93c2a00d7.dc1 area=wan +49:38.612 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23434 network=tcp +49:38.612 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23434 network=udp +49:38.613 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23463 network=tcp +49:38.613 [WARN] TestAgent: The 'handler' field in watches has been deprecated and replaced with the 'args' field. See https://www.consul.io/docs/agent/watches.html +49:38.613 [INFO] TestAgent: started state syncer +49:38.613 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23426 network=tcp +49:38.676 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:38.676 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23411 [Candidate]" term=2 +49:38.676 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:38.676 [DEBUG] TestAgent.server.raft: vote granted: from=b0c98fae-dc09-e706-c66e-bbd93c2a00d7 term=2 tally=1 +49:38.676 [INFO] TestAgent.server.raft: election won: tally=1 +49:38.676 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23411 [Leader]" +49:38.676 [INFO] TestAgent.server: cluster leadership acquired +49:38.676 [INFO] TestAgent.server: New leader elected: payload=Node-b0c98fae-dc09-e706-c66e-bbd93c2a00d7 +49:38.676 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23411 +49:38.677 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:38.679 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:38.679 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:38.679 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:38.679 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:38.679 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:38.679 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b0c98fae-dc09-e706-c66e-bbd93c2a00d7 +49:38.679 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b0c98fae-dc09-e706-c66e-bbd93c2a00d7 +49:38.680 [INFO] TestAgent.server: federation state anti-entropy synced +49:38.684 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:38.684 [INFO] TestAgent: Synced node info +49:38.684 [INFO] TestAgent: Synced service: service=redis +49:38.698 [INFO] TestAgent: Requesting shutdown +49:38.698 [INFO] TestAgent.server: shutting down server +49:38.698 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:38.698 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:38.698 [ERROR] TestAgent.http: Request error: method=GET url=/v1/kv/test?dc=dc1 from=127.0.0.1:44432 error="No cluster leader" +49:38.698 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:38.698Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.698Z [INFO] agent.router.manager: shutting down +49:38.698 [INFO] TestAgent: consul server down +49:38.698 [INFO] TestAgent: shutdown complete +49:38.698 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23434 network=tcp +49:38.698 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23434 network=udp +49:38.699 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23463 network=tcp +49:39.199 [INFO] TestAgent: Waiting for endpoints to shut down +49:39.199 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Reload (0.62s) +=== CONT TestAgent_HealthServicesACLEnforcement +2022-12-04T05:49:38.582Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:38.582Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:38.609Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:38.609Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:38.610 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fc3acac3-2171-3204-d16c-7213502e81b1 Address:127.0.0.1:23446}]" +49:38.610 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23446 [Follower]" leader= +49:38.610 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fc3acac3-2171-3204-d16c-7213502e81b1.dc1 127.0.0.1 +49:38.611 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fc3acac3-2171-3204-d16c-7213502e81b1 127.0.0.1 +2022-12-04T05:49:38.611Z [INFO] agent.router: Initializing LAN area manager +49:38.611 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23462 network=udp +49:38.611 [INFO] TestAgent.server: Adding LAN server: server="Node-fc3acac3-2171-3204-d16c-7213502e81b1 (Addr: tcp/127.0.0.1:23446) (DC: dc1)" +49:38.611 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fc3acac3-2171-3204-d16c-7213502e81b1.dc1 area=wan +49:38.611 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23462 network=tcp +49:38.612 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23407 network=tcp +49:38.612 [INFO] TestAgent: started state syncer +49:38.612 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23433 network=tcp +49:38.661 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:38.665 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:38.665 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23446 [Candidate]" term=2 +49:38.665 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:38.665 [DEBUG] TestAgent.server.raft: vote granted: from=fc3acac3-2171-3204-d16c-7213502e81b1 term=2 tally=1 +49:38.665 [INFO] TestAgent.server.raft: election won: tally=1 +49:38.665 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23446 [Leader]" +49:38.665 [INFO] TestAgent.server: cluster leadership acquired +49:38.666 [INFO] TestAgent.server: New leader elected: payload=Node-fc3acac3-2171-3204-d16c-7213502e81b1 +49:38.666 [INFO] TestAgent.server: initializing acls +49:38.666 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:38.666 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:38.666 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:38.667 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:38.667 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:38.667 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:38.667 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-fc3acac3-2171-3204-d16c-7213502e81b1 +49:38.667 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-fc3acac3-2171-3204-d16c-7213502e81b1.dc1 +49:38.667 [INFO] TestAgent.server: Updating LAN server: server="Node-fc3acac3-2171-3204-d16c-7213502e81b1 (Addr: tcp/127.0.0.1:23446) (DC: dc1)" +49:38.667 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-fc3acac3-2171-3204-d16c-7213502e81b1.dc1 area=wan +49:38.667 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:38.669 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:38.669 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:38.669 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:38.669 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:38.669 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:38.669 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fc3acac3-2171-3204-d16c-7213502e81b1 +49:38.669 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fc3acac3-2171-3204-d16c-7213502e81b1 +49:38.669 [INFO] TestAgent.server: federation state anti-entropy synced +49:38.670 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fc3acac3-2171-3204-d16c-7213502e81b1 +49:38.717 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fc3acac3-2171-3204-d16c-7213502e81b1 +49:38.717 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fc3acac3-2171-3204-d16c-7213502e81b1 +49:38.718 [INFO] TestAgent: Requesting shutdown +49:38.718 [INFO] TestAgent.server: shutting down server +49:38.718 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:38.718 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:38.718 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:38.718 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:38.718 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:38.718 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:38.718 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:38.718 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:38.718 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:38.718 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:38.718 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:38.718 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:38.718 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:38.718 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:38.718 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:38.718 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:38.719Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.719Z [INFO] agent.router.manager: shutting down +49:38.719 [INFO] TestAgent: consul server down +49:38.719 [INFO] TestAgent: shutdown complete +49:38.719 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23462 network=tcp +49:38.719 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23462 network=udp +49:38.719 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23407 network=tcp +49:39.219 [INFO] TestAgent: Waiting for endpoints to shut down +49:39.219 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Reload_ACLDeny (0.65s) + --- PASS: TestAgent_Reload_ACLDeny/no_token (0.00s) + --- PASS: TestAgent_Reload_ACLDeny/read-only_token (0.00s) +=== CONT TestAgent_HealthServiceByName +2022-12-04T05:49:38.369Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:38.379Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:38.380 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b41604b0-d2ba-0a86-c40b-02529b5ce2a7 Address:127.0.0.1:23439}]" +49:38.380 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23439 [Follower]" leader= +49:38.381 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b41604b0-d2ba-0a86-c40b-02529b5ce2a7.dc1 127.0.0.1 +49:38.381 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b41604b0-d2ba-0a86-c40b-02529b5ce2a7 127.0.0.1 +2022-12-04T05:49:38.381Z [INFO] agent.router: Initializing LAN area manager +49:38.382 [INFO] TestAgent.server: Adding LAN server: server="Node-b41604b0-d2ba-0a86-c40b-02529b5ce2a7 (Addr: tcp/127.0.0.1:23439) (DC: dc1)" +49:38.382 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b41604b0-d2ba-0a86-c40b-02529b5ce2a7.dc1 area=wan +49:38.382 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23322 network=udp +49:38.382 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23322 network=tcp +49:38.382 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23442 network=tcp +49:38.382 [INFO] TestAgent: started state syncer +49:38.382 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23513 network=tcp +49:38.452 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:38.452 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23439 [Candidate]" term=2 +49:38.452 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:38.452 [DEBUG] TestAgent.server.raft: vote granted: from=b41604b0-d2ba-0a86-c40b-02529b5ce2a7 term=2 tally=1 +49:38.452 [INFO] TestAgent.server.raft: election won: tally=1 +49:38.452 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23439 [Leader]" +49:38.452 [INFO] TestAgent.server: cluster leadership acquired +49:38.453 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23439 +49:38.453 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:38.455 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:38.455 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:38.455 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:38.455 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:38.455 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:38.455 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b41604b0-d2ba-0a86-c40b-02529b5ce2a7 +49:38.455 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b41604b0-d2ba-0a86-c40b-02529b5ce2a7 +49:38.455 [INFO] TestAgent.server: New leader elected: payload=Node-b41604b0-d2ba-0a86-c40b-02529b5ce2a7 +49:38.456 [INFO] TestAgent.server: federation state anti-entropy synced +49:38.497 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:38.498 [INFO] TestAgent: Synced node info +49:38.774 [INFO] TestAgent: Requesting shutdown +49:38.774 [INFO] TestAgent.server: shutting down server +49:38.774 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:38.774 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:38.774 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:38.774 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:38.774 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:38.774 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:38.774 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:38.774 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:38.774 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:38.774 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:38.774 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:38.774Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.774Z [INFO] agent.router.manager: shutting down +49:38.774 [INFO] TestAgent: consul server down +49:38.774 [INFO] TestAgent: shutdown complete +49:38.774 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23322 network=tcp +49:38.774 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23322 network=udp +49:38.774 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23442 network=tcp +49:39.275 [INFO] TestAgent: Waiting for endpoints to shut down +49:39.275 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Members (0.92s) +=== CONT TestAgent_HealthServiceByID +2022-12-04T05:49:37.512Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:37.512Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.543Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:37.543Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.545 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2aa54b21-3018-2b5e-be15-2117e0a57c1e Address:127.0.0.1:24235}]" +49:37.545 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24235 [Follower]" leader= +49:37.546 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2aa54b21-3018-2b5e-be15-2117e0a57c1e.dc1 127.0.0.1 +49:37.547 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2aa54b21-3018-2b5e-be15-2117e0a57c1e 127.0.0.1 +2022-12-04T05:49:37.547Z [INFO] agent.router: Initializing LAN area manager +49:37.547 [INFO] TestAgent.server: Adding LAN server: server="Node-2aa54b21-3018-2b5e-be15-2117e0a57c1e (Addr: tcp/127.0.0.1:24235) (DC: dc1)" +49:37.547 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2aa54b21-3018-2b5e-be15-2117e0a57c1e.dc1 area=wan +49:37.547 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24195 network=tcp +49:37.547 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24195 network=udp +49:37.548 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24231 network=tcp +49:37.548 [INFO] TestAgent: started state syncer +49:37.548 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24131 network=tcp +49:37.597 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:37.600 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:37.600 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24235 [Candidate]" term=2 +49:37.600 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:37.600 [DEBUG] TestAgent.server.raft: vote granted: from=2aa54b21-3018-2b5e-be15-2117e0a57c1e term=2 tally=1 +49:37.600 [INFO] TestAgent.server.raft: election won: tally=1 +49:37.600 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24235 [Leader]" +49:37.600 [INFO] TestAgent.server: cluster leadership acquired +49:37.600 [INFO] TestAgent.server: initializing acls +49:37.601 [INFO] TestAgent.server: New leader elected: payload=Node-2aa54b21-3018-2b5e-be15-2117e0a57c1e +49:37.601 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:37.601 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:37.601 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:37.601 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:37.601 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:37.601 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:37.601 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-2aa54b21-3018-2b5e-be15-2117e0a57c1e +49:37.602 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-2aa54b21-3018-2b5e-be15-2117e0a57c1e.dc1 +49:37.602 [INFO] TestAgent.server: Updating LAN server: server="Node-2aa54b21-3018-2b5e-be15-2117e0a57c1e (Addr: tcp/127.0.0.1:24235) (DC: dc1)" +49:37.602 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-2aa54b21-3018-2b5e-be15-2117e0a57c1e.dc1 area=wan +49:37.602 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:37.603 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:37.603 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:37.603 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:37.603 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:37.603 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:37.603 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2aa54b21-3018-2b5e-be15-2117e0a57c1e +49:37.603 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2aa54b21-3018-2b5e-be15-2117e0a57c1e +49:37.604 [INFO] TestAgent.server: federation state anti-entropy synced +49:37.604 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2aa54b21-3018-2b5e-be15-2117e0a57c1e +49:37.660 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +49:37.660 [WARN] TestAgent: Node info update blocked by ACLs: node=2aa54b21-3018-2b5e-be15-2117e0a57c1e accessorID=00000000-0000-0000-0000-000000000002 +49:37.660 [DEBUG] TestAgent: Node info in sync +49:37.747 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-2aa54b21-3018-2b5e-be15-2117e0a57c1e +49:37.747 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-2aa54b21-3018-2b5e-be15-2117e0a57c1e +49:37.748 [INFO] TestAgent: Updated agent's ACL token: token=default +49:38.796 [DEBUG] TestAgent.acl: dropping service from result due to ACLs: service="{consul {}}" +49:38.796 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:38.797 [INFO] TestAgent: Synced node info +49:38.800 [INFO] TestAgent: Requesting shutdown +49:38.800 [INFO] TestAgent.server: shutting down server +49:38.800 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:38.800 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:38.800 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:38.800 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:38.800 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:38.800 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:38.800 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:38.800 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:38.800 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:38.800 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:38.800 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:38.800 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:38.800 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:38.800 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:38.800 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:38.800Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.800Z [INFO] agent.router.manager: shutting down +49:38.800 [INFO] TestAgent: consul server down +49:38.800 [INFO] TestAgent: shutdown complete +49:38.800 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24195 network=tcp +49:38.801 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24195 network=udp +49:38.801 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24231 network=tcp +49:39.301 [INFO] TestAgent: Waiting for endpoints to shut down +49:39.301 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_TokenTriggersFullSync (4.44s) + --- PASS: TestAgent_TokenTriggersFullSync/acl_agent_token (0.92s) + --- PASS: TestAgent_TokenTriggersFullSync/agent (0.72s) + --- PASS: TestAgent_TokenTriggersFullSync/acl_token (0.99s) + --- PASS: TestAgent_TokenTriggersFullSync/default (1.81s) +=== CONT TestAgent_ChecksWithFilter +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestAgent_Checks_ACLFilter/no_token +=== RUN TestAgent_Checks_ACLFilter/root_token +2022-12-04T05:49:38.061Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:38.071Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:38.072 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b5b96bfd-f100-46fc-5ffa-a54e133e8333 Address:127.0.0.1:23390}]" +49:38.072 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23390 [Follower]" leader= +49:38.073 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b5b96bfd-f100-46fc-5ffa-a54e133e8333.dc1 127.0.0.1 +49:38.073 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b5b96bfd-f100-46fc-5ffa-a54e133e8333 127.0.0.1 +2022-12-04T05:49:38.073Z [INFO] agent.router: Initializing LAN area manager +49:38.073 [INFO] TestAgent.server: Adding LAN server: server="Node-b5b96bfd-f100-46fc-5ffa-a54e133e8333 (Addr: tcp/127.0.0.1:23390) (DC: dc1)" +49:38.073 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b5b96bfd-f100-46fc-5ffa-a54e133e8333.dc1 area=wan +49:38.073 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23329 network=udp +49:38.073 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23329 network=tcp +49:38.074 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23295 network=tcp +49:38.074 [INFO] TestAgent: started state syncer +49:38.074 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23377 network=tcp +49:38.127 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:38.127 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23390 [Candidate]" term=2 +49:38.127 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:38.127 [DEBUG] TestAgent.server.raft: vote granted: from=b5b96bfd-f100-46fc-5ffa-a54e133e8333 term=2 tally=1 +49:38.127 [INFO] TestAgent.server.raft: election won: tally=1 +49:38.127 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23390 [Leader]" +49:38.127 [INFO] TestAgent.server: cluster leadership acquired +49:38.127 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23390 +49:38.128 [INFO] TestAgent.server: New leader elected: payload=Node-b5b96bfd-f100-46fc-5ffa-a54e133e8333 +49:38.128 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:38.129 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:38.129 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:38.129 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:38.129 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:38.129 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:38.129 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b5b96bfd-f100-46fc-5ffa-a54e133e8333 +49:38.129 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b5b96bfd-f100-46fc-5ffa-a54e133e8333 +49:38.130 [INFO] TestAgent.server: federation state anti-entropy synced +49:38.215 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:38.215 [INFO] TestAgent: Synced node info +49:38.372 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:46598 +49:38.373 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4e682b07-e126-4198-d230-4b6e4cbb49ac 127.0.0.1 +49:38.373 [INFO] TestAgent.server: Adding LAN server: server="Node-4e682b07-e126-4198-d230-4b6e4cbb49ac (Addr: tcp/127.0.0.1:23299) (DC: dc1)" +49:38.373 [INFO] TestAgent.server: New leader elected: payload=Node-4e682b07-e126-4198-d230-4b6e4cbb49ac +49:38.373 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-4e682b07-e126-4198-d230-4b6e4cbb49ac other=Node-b5b96bfd-f100-46fc-5ffa-a54e133e8333 +49:38.373 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4e682b07-e126-4198-d230-4b6e4cbb49ac +49:38.374 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-4e682b07-e126-4198-d230-4b6e4cbb49ac.dc1 127.0.0.1:23298 +49:38.374 [INFO] TestAgent: Requesting shutdown +49:38.374 [INFO] TestAgent.server: shutting down server +49:38.374 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:38.374 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:38.374 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:38.374 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:38.374 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:38.374 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:38.374 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:38.374 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:38.374 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:38.374 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:38.374 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:38.374Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.374Z [INFO] agent.router.manager: shutting down +49:38.374 [INFO] TestAgent: consul server down +49:38.374 [INFO] TestAgent: shutdown complete +49:38.374 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23329 network=tcp +49:38.374 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23329 network=udp +49:38.374 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23295 network=tcp +49:38.374 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4e682b07-e126-4198-d230-4b6e4cbb49ac.dc1 127.0.0.1 +49:38.374 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-4e682b07-e126-4198-d230-4b6e4cbb49ac.dc1 address=127.0.0.1:23298 +49:38.874 [INFO] TestAgent: Waiting for endpoints to shut down +49:38.874 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:37.857Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:37.857Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.867Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:37.867Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.868 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4e682b07-e126-4198-d230-4b6e4cbb49ac Address:127.0.0.1:23299}]" +49:37.869 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23299 [Follower]" leader= +49:37.869 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4e682b07-e126-4198-d230-4b6e4cbb49ac.dc1 127.0.0.1 +49:37.869 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4e682b07-e126-4198-d230-4b6e4cbb49ac 127.0.0.1 +2022-12-04T05:49:37.869Z [INFO] agent.router: Initializing LAN area manager +49:37.869 [INFO] TestAgent.server: Adding LAN server: server="Node-4e682b07-e126-4198-d230-4b6e4cbb49ac (Addr: tcp/127.0.0.1:23299) (DC: dc1)" +49:37.869 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4e682b07-e126-4198-d230-4b6e4cbb49ac.dc1 area=wan +49:37.870 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23287 network=udp +49:37.870 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23287 network=tcp +49:37.870 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23330 network=tcp +49:37.870 [INFO] TestAgent: started state syncer +49:37.870 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23307 network=tcp +49:37.919 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:37.919 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23299 [Candidate]" term=2 +49:37.919 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:37.919 [DEBUG] TestAgent.server.raft: vote granted: from=4e682b07-e126-4198-d230-4b6e4cbb49ac term=2 tally=1 +49:37.919 [INFO] TestAgent.server.raft: election won: tally=1 +49:37.919 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23299 [Leader]" +49:37.919 [INFO] TestAgent.server: cluster leadership acquired +49:37.919 [INFO] TestAgent.server: initializing acls +49:37.920 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:37.920 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:37.920 [INFO] TestAgent.server: New leader elected: payload=Node-4e682b07-e126-4198-d230-4b6e4cbb49ac +49:37.920 [INFO] TestAgent.server: initializing acls +49:37.920 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:37.924 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:37.924 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:37.925 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:37.925 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:37.925 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:37.925 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +49:37.925 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:37.925 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4e682b07-e126-4198-d230-4b6e4cbb49ac +49:37.925 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4e682b07-e126-4198-d230-4b6e4cbb49ac +49:37.925 [INFO] TestAgent.server: Updating LAN server: server="Node-4e682b07-e126-4198-d230-4b6e4cbb49ac (Addr: tcp/127.0.0.1:23299) (DC: dc1)" +49:37.926 [INFO] TestAgent.server: Updating LAN server: server="Node-4e682b07-e126-4198-d230-4b6e4cbb49ac (Addr: tcp/127.0.0.1:23299) (DC: dc1)" +49:37.926 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4e682b07-e126-4198-d230-4b6e4cbb49ac.dc1 +49:37.926 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4e682b07-e126-4198-d230-4b6e4cbb49ac.dc1 +49:37.926 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4e682b07-e126-4198-d230-4b6e4cbb49ac.dc1 area=wan +49:37.926 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4e682b07-e126-4198-d230-4b6e4cbb49ac.dc1 area=wan +49:37.927 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:37.929 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:37.929 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:37.929 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:37.929 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:37.929 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:37.929 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4e682b07-e126-4198-d230-4b6e4cbb49ac +49:37.929 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4e682b07-e126-4198-d230-4b6e4cbb49ac +49:37.929 [INFO] TestAgent.server: federation state anti-entropy synced +49:37.930 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4e682b07-e126-4198-d230-4b6e4cbb49ac +49:37.930 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4e682b07-e126-4198-d230-4b6e4cbb49ac +49:38.052 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4e682b07-e126-4198-d230-4b6e4cbb49ac +49:38.116 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:38.117 [INFO] TestAgent: Synced node info +49:38.369 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4e682b07-e126-4198-d230-4b6e4cbb49ac +49:38.369 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:23388] +49:38.372 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:23388 +49:38.373 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b5b96bfd-f100-46fc-5ffa-a54e133e8333 127.0.0.1 +49:38.373 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +49:38.373 [DEBUG] TestAgent: systemd notify failed: error="No socket" +49:38.374 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:57694 +49:38.374 [INFO] TestAgent.server: Adding LAN server: server="Node-b5b96bfd-f100-46fc-5ffa-a54e133e8333 (Addr: tcp/127.0.0.1:23390) (DC: dc1)" +49:38.374 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-b5b96bfd-f100-46fc-5ffa-a54e133e8333 other=Node-4e682b07-e126-4198-d230-4b6e4cbb49ac +49:38.374 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b5b96bfd-f100-46fc-5ffa-a54e133e8333 +49:38.374 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:23389: connect: connection refused +49:38.374 [DEBUG] TestAgent.server: Failed to flood-join server at address: server=Node-b5b96bfd-f100-46fc-5ffa-a54e133e8333.dc1 address=127.0.0.1:23389 error="1 error occurred: + * Failed to join 127.0.0.1: dial tcp 127.0.0.1:23389: connect: connection refused + +" +49:38.374 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b5b96bfd-f100-46fc-5ffa-a54e133e8333.dc1 127.0.0.1 +49:38.374 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b5b96bfd-f100-46fc-5ffa-a54e133e8333.dc1 area=wan +49:38.874 [INFO] TestAgent: Requesting shutdown +49:38.874 [INFO] TestAgent.server: shutting down server +49:38.874 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:38.874 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:38.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:38.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:38.874 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:38.874 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:38.874 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:38.875 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:38.875 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:38.875 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:38.875 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:38.875 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:38.875 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:38.875 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:38.875 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:38.875Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.875Z [INFO] agent.router.manager: shutting down +49:38.875 [INFO] TestAgent: consul server down +49:38.875 [INFO] TestAgent: shutdown complete +49:38.875 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23287 network=tcp +49:38.875 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23287 network=udp +49:38.875 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23330 network=tcp +49:39.373 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-b5b96bfd-f100-46fc-5ffa-a54e133e8333 (timeout reached) +49:39.375 [INFO] TestAgent: Waiting for endpoints to shut down +49:39.375 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Join_ACLDeny (1.53s) + --- PASS: TestAgent_Join_ACLDeny/no_token (0.00s) + --- PASS: TestAgent_Join_ACLDeny/agent_master_token (0.00s) + --- PASS: TestAgent_Join_ACLDeny/read-only_token (0.00s) +=== CONT TestAgent_Checks === RUN TestAgent_Metrics_ACLDeny/no_token === RUN TestAgent_Metrics_ACLDeny/agent_master_token === RUN TestAgent_Metrics_ACLDeny/read-only_token -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestAgent_Checks_ACLFilter/no_token -=== RUN TestAgent_Checks_ACLFilter/root_token -2021-10-31T23:13:30.260Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:30.319Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:30.323 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:83e552d5-63ca-1f40-22a6-64d531d91095 Address:127.0.0.1:16629}]" -13:30.323 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16629 [Follower]" leader= -13:30.324 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-83e552d5-63ca-1f40-22a6-64d531d91095.dc1 127.0.0.1 -13:30.324 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-83e552d5-63ca-1f40-22a6-64d531d91095 127.0.0.1 -2021-10-31T23:13:30.324Z [INFO] agent.router: Initializing LAN area manager -13:30.325 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16638 network=udp -13:30.325 [INFO] TestAgent.server: Adding LAN server: server="Node-83e552d5-63ca-1f40-22a6-64d531d91095 (Addr: tcp/127.0.0.1:16629) (DC: dc1)" -13:30.325 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-83e552d5-63ca-1f40-22a6-64d531d91095.dc1 area=wan -13:30.325 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16638 network=tcp -13:30.334 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16625 network=tcp -13:30.334 [INFO] TestAgent: started state syncer -13:30.334 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16505 network=tcp -13:30.413 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:30.413 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16629 [Candidate]" term=2 -13:30.414 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:30.414 [DEBUG] TestAgent.server.raft: vote granted: from=83e552d5-63ca-1f40-22a6-64d531d91095 term=2 tally=1 -13:30.414 [INFO] TestAgent.server.raft: election won: tally=1 -13:30.414 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16629 [Leader]" -13:30.414 [INFO] TestAgent.server: cluster leadership acquired -13:30.414 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16629 -13:30.415 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:30.417 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:30.417 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:30.417 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:30.417 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:30.417 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:30.417 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-83e552d5-63ca-1f40-22a6-64d531d91095 -13:30.417 [INFO] TestAgent.server: member joined, marking health alive: member=Node-83e552d5-63ca-1f40-22a6-64d531d91095 -13:30.418 [INFO] TestAgent.server: New leader elected: payload=Node-83e552d5-63ca-1f40-22a6-64d531d91095 -13:30.419 [INFO] TestAgent.server: federation state anti-entropy synced -13:30.568 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:30.569 [INFO] TestAgent: Synced node info -13:30.569 [DEBUG] TestAgent: Node info in sync -13:30.757 [INFO] TestAgent: Requesting shutdown -13:30.757 [INFO] TestAgent.server: shutting down server -13:30.757 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:30.757 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:30.757 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:30.757 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:30.757 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:30.757 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:30.757 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:30.757 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:30.757 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:30.757 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:30.757 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:30.757Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.757Z [INFO] agent.router.manager: shutting down -13:30.757 [INFO] TestAgent: consul server down -13:30.757 [INFO] TestAgent: shutdown complete -13:30.757 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16638 network=tcp -13:30.758 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16638 network=udp -13:30.758 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16625 network=tcp -13:31.263 [INFO] TestAgent: Waiting for endpoints to shut down -13:31.263 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentConnectAuthorize_noTarget (1.03s) -=== CONT TestAgent_HealthServicesACLEnforcement +2022-12-04T05:49:38.183Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:38.192Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:38.194 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2dfb1bdd-87d8-e1af-d454-08d5769556c7 Address:127.0.0.1:23425}]" +49:38.194 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23425 [Follower]" leader= +49:38.194 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2dfb1bdd-87d8-e1af-d454-08d5769556c7.dc1 127.0.0.1 +49:38.195 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2dfb1bdd-87d8-e1af-d454-08d5769556c7 127.0.0.1 +2022-12-04T05:49:38.195Z [INFO] agent.router: Initializing LAN area manager +49:38.195 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23441 network=udp +49:38.195 [INFO] TestAgent.server: Adding LAN server: server="Node-2dfb1bdd-87d8-e1af-d454-08d5769556c7 (Addr: tcp/127.0.0.1:23425) (DC: dc1)" +49:38.195 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2dfb1bdd-87d8-e1af-d454-08d5769556c7.dc1 area=wan +49:38.195 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23441 network=tcp +49:38.196 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23421 network=tcp +49:38.196 [INFO] TestAgent: started state syncer +49:38.196 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23370 network=tcp +49:38.246 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:38.246 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23425 [Candidate]" term=2 +49:38.246 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:38.246 [DEBUG] TestAgent.server.raft: vote granted: from=2dfb1bdd-87d8-e1af-d454-08d5769556c7 term=2 tally=1 +49:38.246 [INFO] TestAgent.server.raft: election won: tally=1 +49:38.246 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23425 [Leader]" +49:38.246 [INFO] TestAgent.server: cluster leadership acquired +49:38.246 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23425 +49:38.246 [INFO] TestAgent.server: New leader elected: payload=Node-2dfb1bdd-87d8-e1af-d454-08d5769556c7 +49:38.247 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:38.248 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:38.248 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:38.248 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:38.248 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:38.248 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:38.248 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2dfb1bdd-87d8-e1af-d454-08d5769556c7 +49:38.248 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2dfb1bdd-87d8-e1af-d454-08d5769556c7 +49:38.248 [INFO] TestAgent.server: federation state anti-entropy synced +49:38.464 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:40148 +49:38.464 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a1a39e67-cbfc-0138-c221-c59bfc380e32.dc1 127.0.0.1 +49:38.465 [INFO] TestAgent: Requesting shutdown +49:38.465 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a1a39e67-cbfc-0138-c221-c59bfc380e32.dc1 area=wan +49:38.465 [INFO] TestAgent.server: shutting down server +49:38.465 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:38.465 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:38.465 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:38.465 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:38.465 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:38.465 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:38.465 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:38.465 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:38.465 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:38.465 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:38.465 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:38.465 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:38.465Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.465Z [INFO] agent.router.manager: shutting down +49:38.465 [INFO] TestAgent: consul server down +49:38.465 [INFO] TestAgent: shutdown complete +49:38.465 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23441 network=tcp +49:38.465 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23441 network=udp +49:38.465 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23421 network=tcp +49:38.965 [INFO] TestAgent: Waiting for endpoints to shut down +49:38.965 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:37.927Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.944Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.946 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a1a39e67-cbfc-0138-c221-c59bfc380e32 Address:127.0.0.1:24284}]" +49:37.946 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24284 [Follower]" leader= +49:37.946 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a1a39e67-cbfc-0138-c221-c59bfc380e32.dc1 127.0.0.1 +49:37.947 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a1a39e67-cbfc-0138-c221-c59bfc380e32 127.0.0.1 +2022-12-04T05:49:37.947Z [INFO] agent.router: Initializing LAN area manager +49:37.947 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a1a39e67-cbfc-0138-c221-c59bfc380e32.dc1 area=wan +49:37.947 [INFO] TestAgent.server: Adding LAN server: server="Node-a1a39e67-cbfc-0138-c221-c59bfc380e32 (Addr: tcp/127.0.0.1:24284) (DC: dc1)" +49:37.947 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23364 network=udp +49:37.947 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23364 network=tcp +49:37.948 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24280 network=tcp +49:37.948 [INFO] TestAgent: started state syncer +49:37.948 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24271 network=tcp +49:37.988 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:37.988 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24284 [Candidate]" term=2 +49:37.989 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:37.989 [DEBUG] TestAgent.server.raft: vote granted: from=a1a39e67-cbfc-0138-c221-c59bfc380e32 term=2 tally=1 +49:37.989 [INFO] TestAgent.server.raft: election won: tally=1 +49:37.989 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24284 [Leader]" +49:37.989 [INFO] TestAgent.server: cluster leadership acquired +49:37.989 [INFO] TestAgent.server: New leader elected: payload=Node-a1a39e67-cbfc-0138-c221-c59bfc380e32 +49:37.989 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24284 +49:37.990 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:37.991 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:37.991 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:37.991 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:37.991 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:37.991 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:37.991 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1a39e67-cbfc-0138-c221-c59bfc380e32 +49:37.991 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a1a39e67-cbfc-0138-c221-c59bfc380e32 +49:37.992 [INFO] TestAgent.server: federation state anti-entropy synced +49:38.262 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:38.262 [INFO] TestAgent: Synced node info +49:38.463 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:23424] +49:38.464 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:23424 +49:38.465 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2dfb1bdd-87d8-e1af-d454-08d5769556c7.dc1 127.0.0.1 +49:38.465 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +49:38.465 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2dfb1bdd-87d8-e1af-d454-08d5769556c7.dc1 area=wan +49:38.965 [INFO] TestAgent: Requesting shutdown +49:38.965 [INFO] TestAgent.server: shutting down server +49:38.965 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:38.965 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:38.965 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:38.965 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:38.965 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:38.965 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:38.965 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:38.965 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:38.965 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:38.965 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:38.966 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:38.966Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.966Z [INFO] agent.router.manager: shutting down +49:38.966 [INFO] TestAgent: consul server down +49:38.966 [INFO] TestAgent: shutdown complete +49:38.966 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23364 network=tcp +49:38.966 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23364 network=udp +49:38.966 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24280 network=tcp +49:39.466 [INFO] TestAgent: Waiting for endpoints to shut down +49:39.466 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Join_WAN (1.57s) +=== CONT TestAgent_Services_ACLFilter === RUN TestAgent_Self_ACLDeny/no_token === RUN TestAgent_Self_ACLDeny/agent_master_token === RUN TestAgent_Self_ACLDeny/read-only_token -2021-10-31T23:13:30.655Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:30.670Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:30.672 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:48f390e2-156b-2aaa-49b7-4c23f9642f94 Address:127.0.0.1:16685}]" -13:30.672 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16685 [Follower]" leader= -13:30.673 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-48f390e2-156b-2aaa-49b7-4c23f9642f94.dc1 127.0.0.1 -13:30.673 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-48f390e2-156b-2aaa-49b7-4c23f9642f94 127.0.0.1 -2021-10-31T23:13:30.673Z [INFO] agent.router: Initializing LAN area manager -13:30.673 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16715 network=udp -13:30.674 [INFO] TestAgent.server: Adding LAN server: server="Node-48f390e2-156b-2aaa-49b7-4c23f9642f94 (Addr: tcp/127.0.0.1:16685) (DC: dc1)" -13:30.674 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-48f390e2-156b-2aaa-49b7-4c23f9642f94.dc1 area=wan -13:30.674 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16715 network=tcp -13:30.674 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16681 network=tcp -13:30.674 [INFO] TestAgent: started state syncer -13:30.674 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16162 network=tcp -13:30.713 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:30.739 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16685 [Candidate]" term=2 -13:30.739 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:30.739 [DEBUG] TestAgent.server.raft: vote granted: from=48f390e2-156b-2aaa-49b7-4c23f9642f94 term=2 tally=1 -13:30.739 [INFO] TestAgent.server.raft: election won: tally=1 -13:30.739 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16685 [Leader]" -13:30.739 [INFO] TestAgent.server: cluster leadership acquired -13:30.740 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16685 -13:30.741 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:30.741 [INFO] TestAgent.server: New leader elected: payload=Node-48f390e2-156b-2aaa-49b7-4c23f9642f94 -13:30.743 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:30.743 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:30.743 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:30.743 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:30.743 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:30.743 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-48f390e2-156b-2aaa-49b7-4c23f9642f94 -13:30.743 [INFO] TestAgent.server: member joined, marking health alive: member=Node-48f390e2-156b-2aaa-49b7-4c23f9642f94 -13:30.745 [INFO] TestAgent.server: federation state anti-entropy synced -13:30.794 [INFO] TestAgent: Requesting shutdown -13:30.794 [INFO] TestAgent.server: shutting down server -13:30.794 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:30.794 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:30.794 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:30.794 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:30.794 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:30.794 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:30.794 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:30.794 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:30.794 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:30.794 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:30.794 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:30.794 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:30.795Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.795Z [INFO] agent.router.manager: shutting down -13:30.795 [INFO] TestAgent: consul server down -13:30.795 [INFO] TestAgent: shutdown complete -13:30.795 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16715 network=tcp -13:30.795 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16715 network=udp -13:30.795 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16681 network=tcp -13:31.297 [INFO] TestAgent: Waiting for endpoints to shut down -13:31.297 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Members (0.73s) -=== CONT TestAgent_HealthServiceByName -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:30.525Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:30.544Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:30.547 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aca00aa4-cc7e-cd8b-49cc-2880aba52691 Address:127.0.0.1:16252}]" -13:30.556 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aca00aa4-cc7e-cd8b-49cc-2880aba52691.dc1 127.0.0.1 -13:30.557 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aca00aa4-cc7e-cd8b-49cc-2880aba52691 127.0.0.1 -2021-10-31T23:13:30.557Z [INFO] agent.router: Initializing LAN area manager -13:30.566 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16252 [Follower]" leader= -13:30.567 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aca00aa4-cc7e-cd8b-49cc-2880aba52691.dc1 area=wan -13:30.567 [INFO] TestAgent.server: Adding LAN server: server="Node-aca00aa4-cc7e-cd8b-49cc-2880aba52691 (Addr: tcp/127.0.0.1:16252) (DC: dc1)" -13:30.567 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16701 network=tcp -13:30.570 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16701 network=udp -13:30.571 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16248 network=tcp -13:30.571 [INFO] TestAgent: started state syncer -13:30.571 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16735 network=tcp -13:30.617 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:30.617 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16252 [Candidate]" term=2 -13:30.618 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:30.618 [DEBUG] TestAgent.server.raft: vote granted: from=aca00aa4-cc7e-cd8b-49cc-2880aba52691 term=2 tally=1 -13:30.618 [INFO] TestAgent.server.raft: election won: tally=1 -13:30.618 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16252 [Leader]" -13:30.618 [INFO] TestAgent.server: cluster leadership acquired -13:30.619 [INFO] TestAgent.server: New leader elected: payload=Node-aca00aa4-cc7e-cd8b-49cc-2880aba52691 -13:30.621 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16252 -13:30.623 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:30.625 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:30.625 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:30.625 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:30.625 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:30.625 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:30.625 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aca00aa4-cc7e-cd8b-49cc-2880aba52691 -13:30.626 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aca00aa4-cc7e-cd8b-49cc-2880aba52691 -13:30.626 [INFO] TestAgent.server: federation state anti-entropy synced -13:30.846 [INFO] TestAgent: Requesting shutdown -13:30.846 [INFO] TestAgent.server: shutting down server -13:30.846 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:30.846 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:30.846 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:30.846 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:30.846 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:30.846 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:30.846 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:30.846 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:30.846 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:30.846 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:30.846 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:30.846 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:30.846Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.846Z [INFO] agent.router.manager: shutting down -13:30.847 [INFO] TestAgent: consul server down -13:30.847 [INFO] TestAgent: shutdown complete -13:30.847 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16701 network=tcp -13:30.847 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16701 network=udp -13:30.847 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16248 network=tcp -13:31.366 [INFO] TestAgent: Waiting for endpoints to shut down -13:31.366 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Members_WAN (0.87s) -=== CONT TestAgent_HealthServiceByID -2021-10-31T23:13:30.512Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:30.512Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:30.542Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:30.542Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:30.544 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:59fc187d-f568-26c8-07f2-50d742a23a37 Address:127.0.0.1:16574}]" -13:30.545 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-59fc187d-f568-26c8-07f2-50d742a23a37.dc1 127.0.0.1 -13:30.545 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16574 [Follower]" leader= -13:30.562 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-59fc187d-f568-26c8-07f2-50d742a23a37 127.0.0.1 -2021-10-31T23:13:30.562Z [INFO] agent.router: Initializing LAN area manager -13:30.563 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16506 network=udp -13:30.564 [INFO] TestAgent.server: Adding LAN server: server="Node-59fc187d-f568-26c8-07f2-50d742a23a37 (Addr: tcp/127.0.0.1:16574) (DC: dc1)" -13:30.564 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-59fc187d-f568-26c8-07f2-50d742a23a37.dc1 area=wan -13:30.565 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16506 network=tcp -13:30.569 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16570 network=tcp -13:30.569 [INFO] TestAgent: started state syncer -13:30.570 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16700 network=tcp -13:30.617 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:30.617 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:30.618 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16574 [Candidate]" term=2 -13:30.618 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:30.618 [DEBUG] TestAgent.server.raft: vote granted: from=59fc187d-f568-26c8-07f2-50d742a23a37 term=2 tally=1 -13:30.618 [INFO] TestAgent.server.raft: election won: tally=1 -13:30.618 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16574 [Leader]" -13:30.619 [INFO] TestAgent.server: cluster leadership acquired -13:30.619 [INFO] TestAgent.server: New leader elected: payload=Node-59fc187d-f568-26c8-07f2-50d742a23a37 -13:30.619 [INFO] TestAgent.server: initializing acls -13:30.619 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:30.619 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:30.620 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:30.621 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:30.621 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:30.621 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:30.624 [INFO] TestAgent: Synced node info -13:30.626 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-59fc187d-f568-26c8-07f2-50d742a23a37 -13:30.627 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-59fc187d-f568-26c8-07f2-50d742a23a37.dc1 -13:30.627 [INFO] TestAgent.server: Updating LAN server: server="Node-59fc187d-f568-26c8-07f2-50d742a23a37 (Addr: tcp/127.0.0.1:16574) (DC: dc1)" -13:30.627 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-59fc187d-f568-26c8-07f2-50d742a23a37.dc1 area=wan -13:30.628 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:30.631 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:30.631 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:30.631 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:30.631 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:30.631 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:30.646 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-59fc187d-f568-26c8-07f2-50d742a23a37 -13:30.646 [INFO] TestAgent.server: member joined, marking health alive: member=Node-59fc187d-f568-26c8-07f2-50d742a23a37 -13:30.647 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-59fc187d-f568-26c8-07f2-50d742a23a37 -13:30.651 [INFO] TestAgent.server: federation state anti-entropy synced -13:30.894 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-59fc187d-f568-26c8-07f2-50d742a23a37 -13:30.894 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-59fc187d-f568-26c8-07f2-50d742a23a37 -13:30.894 [DEBUG] TestAgent: dropping node from result due to ACLs: node=Node-59fc187d-f568-26c8-07f2-50d742a23a37 accessorID=00000000-0000-0000-0000-000000000002 -13:30.894 [INFO] TestAgent: Requesting shutdown -13:30.894 [INFO] TestAgent.server: shutting down server -13:30.894 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:30.894 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:30.894 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:30.894 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:30.894 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:30.894 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:30.894 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:30.895 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:30.895 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:30.895 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:30.895 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:30.895 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:30.895 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:30.895 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:30.895 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:30.895Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.895Z [INFO] agent.router.manager: shutting down -13:30.895 [INFO] TestAgent: consul server down -13:30.895 [INFO] TestAgent: shutdown complete -13:30.895 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16506 network=tcp -13:30.895 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16506 network=udp -13:30.895 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16570 network=tcp -13:31.401 [INFO] TestAgent: Waiting for endpoints to shut down -13:31.401 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Members_ACLFilter (0.95s) - --- PASS: TestAgent_Members_ACLFilter/no_token (0.00s) - --- PASS: TestAgent_Members_ACLFilter/root_token (0.00s) -=== CONT TestAgent_ChecksWithFilter +2022-12-04T05:49:38.431Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:38.454Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:38.455 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e2281077-cb8b-78fd-e157-b0469f37d620 Address:127.0.0.1:23498}]" +49:38.456 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e2281077-cb8b-78fd-e157-b0469f37d620.dc1 127.0.0.1 +49:38.456 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23498 [Follower]" leader= +49:38.456 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e2281077-cb8b-78fd-e157-b0469f37d620 127.0.0.1 +2022-12-04T05:49:38.456Z [INFO] agent.router: Initializing LAN area manager +49:38.457 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23521 network=udp +49:38.457 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e2281077-cb8b-78fd-e157-b0469f37d620.dc1 area=wan +49:38.457 [INFO] TestAgent.server: Adding LAN server: server="Node-e2281077-cb8b-78fd-e157-b0469f37d620 (Addr: tcp/127.0.0.1:23498) (DC: dc1)" +49:38.457 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23521 network=tcp +49:38.457 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23522 network=tcp +49:38.457 [INFO] TestAgent: started state syncer +49:38.457 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23461 network=tcp +49:38.501 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:38.501 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23498 [Candidate]" term=2 +49:38.501 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:38.501 [DEBUG] TestAgent.server.raft: vote granted: from=e2281077-cb8b-78fd-e157-b0469f37d620 term=2 tally=1 +49:38.501 [INFO] TestAgent.server.raft: election won: tally=1 +49:38.501 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23498 [Leader]" +49:38.501 [INFO] TestAgent.server: cluster leadership acquired +49:38.502 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23498 +49:38.502 [INFO] TestAgent.server: New leader elected: payload=Node-e2281077-cb8b-78fd-e157-b0469f37d620 +49:38.502 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:38.504 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:38.504 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:38.504 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:38.504 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:38.504 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:38.504 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e2281077-cb8b-78fd-e157-b0469f37d620 +49:38.504 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e2281077-cb8b-78fd-e157-b0469f37d620 +49:38.504 [INFO] TestAgent.server: federation state anti-entropy synced +49:38.533 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:36780 +49:38.534 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-497f1641-37d2-6aa6-7683-15b0e22318ca 127.0.0.1 +49:38.534 [INFO] TestAgent.server: Adding LAN server: server="Node-497f1641-37d2-6aa6-7683-15b0e22318ca (Addr: tcp/127.0.0.1:23397) (DC: dc1)" +49:38.534 [INFO] TestAgent.server: New leader elected: payload=Node-497f1641-37d2-6aa6-7683-15b0e22318ca +49:38.534 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-497f1641-37d2-6aa6-7683-15b0e22318ca other=Node-e2281077-cb8b-78fd-e157-b0469f37d620 +49:38.534 [INFO] TestAgent.server: member joined, marking health alive: member=Node-497f1641-37d2-6aa6-7683-15b0e22318ca +49:38.534 [INFO] TestAgent: Requesting shutdown +49:38.534 [INFO] TestAgent.server: shutting down server +49:38.534 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:38.534 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:38.534 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:38.534 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:38.534 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:38.534 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:38.534 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:38.534 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:38.534 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:38.534 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:38.534 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:38.534 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-497f1641-37d2-6aa6-7683-15b0e22318ca.dc1 127.0.0.1:23396 +49:38.534 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:38.534Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.534Z [INFO] agent.router.manager: shutting down +49:38.534 [INFO] TestAgent: consul server down +49:38.534 [INFO] TestAgent: shutdown complete +49:38.534 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23521 network=tcp +49:38.534 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23521 network=udp +49:38.535 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23522 network=tcp +49:38.535 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-497f1641-37d2-6aa6-7683-15b0e22318ca.dc1 127.0.0.1 +49:38.535 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-497f1641-37d2-6aa6-7683-15b0e22318ca.dc1 address=127.0.0.1:23396 +49:39.035 [INFO] TestAgent: Waiting for endpoints to shut down +49:39.035 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:37.958Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.970Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.972 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:497f1641-37d2-6aa6-7683-15b0e22318ca Address:127.0.0.1:23397}]" +49:37.972 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23397 [Follower]" leader= +49:37.972 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-497f1641-37d2-6aa6-7683-15b0e22318ca.dc1 127.0.0.1 +49:37.973 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-497f1641-37d2-6aa6-7683-15b0e22318ca 127.0.0.1 +2022-12-04T05:49:37.973Z [INFO] agent.router: Initializing LAN area manager +49:37.973 [INFO] TestAgent.server: Adding LAN server: server="Node-497f1641-37d2-6aa6-7683-15b0e22318ca (Addr: tcp/127.0.0.1:23397) (DC: dc1)" +49:37.973 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-497f1641-37d2-6aa6-7683-15b0e22318ca.dc1 area=wan +49:37.973 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24244 network=udp +49:37.973 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24244 network=tcp +49:37.974 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23372 network=tcp +49:37.974 [INFO] TestAgent: started state syncer +49:37.974 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23391 network=tcp +49:38.032 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:38.032 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23397 [Candidate]" term=2 +49:38.032 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:38.032 [DEBUG] TestAgent.server.raft: vote granted: from=497f1641-37d2-6aa6-7683-15b0e22318ca term=2 tally=1 +49:38.032 [INFO] TestAgent.server.raft: election won: tally=1 +49:38.032 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23397 [Leader]" +49:38.032 [INFO] TestAgent.server: cluster leadership acquired +49:38.032 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23397 +49:38.033 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:38.034 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:38.034 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:38.034 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:38.034 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:38.034 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:38.034 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-497f1641-37d2-6aa6-7683-15b0e22318ca +49:38.034 [INFO] TestAgent.server: member joined, marking health alive: member=Node-497f1641-37d2-6aa6-7683-15b0e22318ca +49:38.035 [INFO] TestAgent.server: New leader elected: payload=Node-497f1641-37d2-6aa6-7683-15b0e22318ca +49:38.035 [INFO] TestAgent.server: federation state anti-entropy synced +49:38.326 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:38.327 [INFO] TestAgent: Synced node info +49:38.343 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:38.343 [DEBUG] TestAgent: Node info in sync +49:38.343 [DEBUG] TestAgent: Node info in sync +49:38.533 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:23524] +49:38.533 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:23524 +49:38.534 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e2281077-cb8b-78fd-e157-b0469f37d620 127.0.0.1 +49:38.534 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +49:38.534 [DEBUG] TestAgent: systemd notify failed: error="No socket" +49:38.534 [INFO] TestAgent.server: Adding LAN server: server="Node-e2281077-cb8b-78fd-e157-b0469f37d620 (Addr: tcp/127.0.0.1:23498) (DC: dc1)" +49:38.534 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-e2281077-cb8b-78fd-e157-b0469f37d620 other=Node-497f1641-37d2-6aa6-7683-15b0e22318ca +49:38.534 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e2281077-cb8b-78fd-e157-b0469f37d620 +49:38.534 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:49568 +49:38.535 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e2281077-cb8b-78fd-e157-b0469f37d620.dc1 127.0.0.1 +49:38.535 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e2281077-cb8b-78fd-e157-b0469f37d620.dc1 area=wan +49:38.535 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:23525: connect: connection refused +49:38.535 [DEBUG] TestAgent.server: Failed to flood-join server at address: server=Node-e2281077-cb8b-78fd-e157-b0469f37d620.dc1 address=127.0.0.1:23525 error="1 error occurred: + * Failed to join 127.0.0.1: dial tcp 127.0.0.1:23525: connect: connection refused + +" +49:39.035 [INFO] TestAgent: Requesting shutdown +49:39.035 [INFO] TestAgent.server: shutting down server +49:39.035 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:39.035 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:39.035 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:39.035 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:39.035 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:39.035 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:39.035 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:39.035 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:39.035 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:39.035 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:39.035 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:39.035Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:39.035Z [INFO] agent.router.manager: shutting down +49:39.035 [INFO] TestAgent: consul server down +49:39.035 [INFO] TestAgent: shutdown complete +49:39.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24244 network=tcp +49:39.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24244 network=udp +49:39.035 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23372 network=tcp +49:39.475 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-e2281077-cb8b-78fd-e157-b0469f37d620 (timeout reached) +49:39.535 [INFO] TestAgent: Waiting for endpoints to shut down +49:39.535 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Join (1.59s) +=== CONT TestAgent_Services_TerminatingGateway +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestAgent_HealthServiceByName/passing_checks === RUN TestAgent_HealthServiceByName/passing_checks/format=text === RUN TestAgent_HealthServiceByName/passing_checks/format=json @@ -44375,232 +50856,14 @@ === RUN TestAgent_HealthServiceByName/maintenance_check_on_node === RUN TestAgent_HealthServiceByName/maintenance_check_on_node/format=text === RUN TestAgent_HealthServiceByName/maintenance_check_on_node/format=json -2021-10-31T23:13:30.629Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:30.629Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:30.698Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:30.698Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:30.700 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d9e9dbe9-92f5-312c-f888-c08b60039864 Address:127.0.0.1:16734}]" -13:30.701 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d9e9dbe9-92f5-312c-f888-c08b60039864.dc1 127.0.0.1 -13:30.701 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16734 [Follower]" leader= -13:30.706 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d9e9dbe9-92f5-312c-f888-c08b60039864 127.0.0.1 -2021-10-31T23:13:30.706Z [INFO] agent.router: Initializing LAN area manager -13:30.706 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16163 network=udp -13:30.706 [INFO] TestAgent.server: Adding LAN server: server="Node-d9e9dbe9-92f5-312c-f888-c08b60039864 (Addr: tcp/127.0.0.1:16734) (DC: dc1)" -13:30.706 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d9e9dbe9-92f5-312c-f888-c08b60039864.dc1 area=wan -13:30.706 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16163 network=tcp -13:30.707 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16730 network=tcp -13:30.707 [INFO] TestAgent: started state syncer -13:30.707 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16253 network=tcp -13:30.758 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:30.764 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:30.764 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16734 [Candidate]" term=2 -13:30.764 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:30.764 [DEBUG] TestAgent.server.raft: vote granted: from=d9e9dbe9-92f5-312c-f888-c08b60039864 term=2 tally=1 -13:30.764 [INFO] TestAgent.server.raft: election won: tally=1 -13:30.764 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16734 [Leader]" -13:30.764 [INFO] TestAgent.server: cluster leadership acquired -13:30.764 [INFO] TestAgent.server: initializing acls -13:30.765 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:30.765 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:30.766 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:30.766 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:30.766 [INFO] TestAgent.server: New leader elected: payload=Node-d9e9dbe9-92f5-312c-f888-c08b60039864 -13:30.766 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:30.766 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:30.766 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d9e9dbe9-92f5-312c-f888-c08b60039864 -13:30.767 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d9e9dbe9-92f5-312c-f888-c08b60039864.dc1 -13:30.767 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:30.769 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d9e9dbe9-92f5-312c-f888-c08b60039864.dc1 area=wan -13:30.770 [INFO] TestAgent.server: Updating LAN server: server="Node-d9e9dbe9-92f5-312c-f888-c08b60039864 (Addr: tcp/127.0.0.1:16734) (DC: dc1)" -13:30.782 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:30.782 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:30.782 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:30.782 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:30.782 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:30.782 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d9e9dbe9-92f5-312c-f888-c08b60039864 -13:30.782 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d9e9dbe9-92f5-312c-f888-c08b60039864 -13:30.784 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d9e9dbe9-92f5-312c-f888-c08b60039864 -13:30.784 [INFO] TestAgent.server: federation state anti-entropy synced -13:30.954 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:30.955 [INFO] TestAgent: Synced node info -13:31.062 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d9e9dbe9-92f5-312c-f888-c08b60039864 -13:31.062 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d9e9dbe9-92f5-312c-f888-c08b60039864 -13:31.063 [INFO] TestAgent: Requesting shutdown -13:31.064 [INFO] TestAgent.server: shutting down server -13:31.064 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:31.064 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:31.064 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:31.064 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:31.064 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:31.064 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:31.064 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:31.064 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:31.066 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:31.066 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:31.066 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:31.066 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:31.066 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:31.066 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:31.066 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2021-10-31T23:13:31.066Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:31.066Z [INFO] agent.router.manager: shutting down -13:31.067 [INFO] TestAgent: consul server down -13:31.067 [INFO] TestAgent: shutdown complete -13:31.067 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16163 network=tcp -13:31.067 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16163 network=udp -13:31.067 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16730 network=tcp -13:31.569 [INFO] TestAgent: Waiting for endpoints to shut down -13:31.569 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Reload_ACLDeny (1.00s) - --- PASS: TestAgent_Reload_ACLDeny/no_token (0.00s) - --- PASS: TestAgent_Reload_ACLDeny/read-only_token (0.00s) -=== CONT TestAgent_Checks -2021-10-31T23:13:30.933Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:30.933Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:30.963Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:30.963Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:30.970 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4f820238-39b3-e8f3-dbed-ea3dade74db4 Address:127.0.0.1:16678}]" -13:30.970 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4f820238-39b3-e8f3-dbed-ea3dade74db4.dc1 127.0.0.1 -13:30.971 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4f820238-39b3-e8f3-dbed-ea3dade74db4 127.0.0.1 -2021-10-31T23:13:30.971Z [INFO] agent.router: Initializing LAN area manager -13:30.971 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16673 network=udp -13:30.971 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16678 [Follower]" leader= -13:30.971 [INFO] TestAgent.server: Adding LAN server: server="Node-4f820238-39b3-e8f3-dbed-ea3dade74db4 (Addr: tcp/127.0.0.1:16678) (DC: dc1)" -13:30.972 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4f820238-39b3-e8f3-dbed-ea3dade74db4.dc1 area=wan -13:30.972 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16673 network=tcp -13:30.972 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16674 network=tcp -13:30.972 [INFO] TestAgent: started state syncer -13:30.972 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16686 network=tcp -13:31.017 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:31.017 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16678 [Candidate]" term=2 -13:31.018 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:31.018 [DEBUG] TestAgent.server.raft: vote granted: from=4f820238-39b3-e8f3-dbed-ea3dade74db4 term=2 tally=1 -13:31.018 [INFO] TestAgent.server.raft: election won: tally=1 -13:31.018 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16678 [Leader]" -13:31.018 [INFO] TestAgent.server: cluster leadership acquired -13:31.018 [INFO] TestAgent.server: initializing acls -13:31.018 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:31.018 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:31.019 [INFO] TestAgent.server: New leader elected: payload=Node-4f820238-39b3-e8f3-dbed-ea3dade74db4 -13:31.019 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:31.019 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:31.019 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:31.019 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:31.019 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4f820238-39b3-e8f3-dbed-ea3dade74db4 -13:31.019 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4f820238-39b3-e8f3-dbed-ea3dade74db4.dc1 -13:31.020 [INFO] TestAgent.server: Updating LAN server: server="Node-4f820238-39b3-e8f3-dbed-ea3dade74db4 (Addr: tcp/127.0.0.1:16678) (DC: dc1)" -13:31.020 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4f820238-39b3-e8f3-dbed-ea3dade74db4.dc1 area=wan -13:31.020 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:31.030 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:31.030 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:31.030 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:31.030 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:31.030 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:31.030 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4f820238-39b3-e8f3-dbed-ea3dade74db4 -13:31.030 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4f820238-39b3-e8f3-dbed-ea3dade74db4 -13:31.031 [INFO] TestAgent.server: federation state anti-entropy synced -13:31.031 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4f820238-39b3-e8f3-dbed-ea3dade74db4 -13:31.073 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4f820238-39b3-e8f3-dbed-ea3dade74db4 -13:31.075 [INFO] TestAgent: Requesting shutdown -13:31.075 [INFO] TestAgent.server: shutting down server -13:31.075 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:31.075 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:31.075 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:31.075 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:31.075 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:31.075 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:31.075 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:31.075 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:31.075 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:31.075 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:31.075 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:31.075 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:31.075 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:31.075 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:31.075 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:31.075 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:31.076Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:31.076Z [INFO] agent.router.manager: shutting down -13:31.076 [INFO] TestAgent: consul server down -13:31.076 [INFO] TestAgent: shutdown complete -13:31.076 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16673 network=tcp -13:31.076 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16673 network=udp -13:31.076 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16674 network=tcp -13:31.577 [INFO] TestAgent: Waiting for endpoints to shut down -13:31.577 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Metrics_ACLDeny (0.67s) - --- PASS: TestAgent_Metrics_ACLDeny/no_token (0.00s) - --- PASS: TestAgent_Metrics_ACLDeny/agent_master_token (0.00s) - --- PASS: TestAgent_Metrics_ACLDeny/read-only_token (0.00s) -=== CONT TestAgent_Services_ACLFilter -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:31.135Z [WARN] TestAgent_Reload: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:30.658Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:30.693Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:30.703 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:50645eb0-2c43-0ddc-c8d4-41de7d30c4d6 Address:127.0.0.1:16175}]" -13:30.704 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-50645eb0-2c43-0ddc-c8d4-41de7d30c4d6.dc1 127.0.0.1 -13:30.704 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-50645eb0-2c43-0ddc-c8d4-41de7d30c4d6 127.0.0.1 -2021-10-31T23:13:30.704Z [INFO] agent.router: Initializing LAN area manager -13:30.704 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16254 network=udp -13:30.705 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16175 [Follower]" leader= -13:30.705 [INFO] TestAgent.server: Adding LAN server: server="Node-50645eb0-2c43-0ddc-c8d4-41de7d30c4d6 (Addr: tcp/127.0.0.1:16175) (DC: dc1)" -13:30.705 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-50645eb0-2c43-0ddc-c8d4-41de7d30c4d6.dc1 area=wan -13:30.705 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16254 network=tcp -13:30.738 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16171 network=tcp -13:30.738 [WARN] TestAgent: The 'handler' field in watches has been deprecated and replaced with the 'args' field. See https://www.consul.io/docs/agent/watches.html -13:30.738 [INFO] TestAgent: started state syncer -13:30.739 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16672 network=tcp -13:30.741 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:30.741 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16175 [Candidate]" term=2 -13:30.742 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:30.742 [DEBUG] TestAgent.server.raft: vote granted: from=50645eb0-2c43-0ddc-c8d4-41de7d30c4d6 term=2 tally=1 -13:30.742 [INFO] TestAgent.server.raft: election won: tally=1 -13:30.742 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16175 [Leader]" -13:30.742 [INFO] TestAgent.server: cluster leadership acquired -13:30.742 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test?dc=dc1 from=127.0.0.1:56038 latency=101.7µs -13:30.742 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16175 -13:30.743 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:30.745 [INFO] TestAgent.server: New leader elected: payload=Node-50645eb0-2c43-0ddc-c8d4-41de7d30c4d6 -13:30.755 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:30.755 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:30.755 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:30.755 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:30.755 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:30.755 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-50645eb0-2c43-0ddc-c8d4-41de7d30c4d6 -13:30.755 [INFO] TestAgent.server: member joined, marking health alive: member=Node-50645eb0-2c43-0ddc-c8d4-41de7d30c4d6 -13:30.756 [INFO] TestAgent.server: federation state anti-entropy synced -13:30.766 [DEBUG] TestAgent: watch handler output: watch_handler=true output= -13:30.966 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:30.966 [INFO] TestAgent: Synced node info -13:30.966 [INFO] TestAgent: Synced service: service=redis -13:31.135 [INFO] TestAgent: Requesting shutdown -13:31.135 [INFO] TestAgent.server: shutting down server -13:31.135 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:31.136 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:31.136 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -2021-10-31T23:13:31.136Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:31.136Z [INFO] agent.router.manager: shutting down -13:31.136 [INFO] TestAgent: consul server down -13:31.136 [INFO] TestAgent: shutdown complete -13:31.136 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16254 network=tcp -13:31.136 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16254 network=udp -13:31.136 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16171 network=tcp -13:31.637 [INFO] TestAgent: Waiting for endpoints to shut down -13:31.637 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Reload (1.04s) -=== CONT TestAgent_Services_TerminatingGateway === RUN TestAgent_HealthServiceByID/passing_checks -=== RUN TestAgent_HealthServicesACLEnforcement/no-token-health-by-id === RUN TestAgent_HealthServiceByID/passing_checks/format=text === RUN TestAgent_HealthServiceByID/passing_checks/format=json -=== RUN TestAgent_HealthServicesACLEnforcement/no-token-health-by-name === RUN TestAgent_HealthServiceByID/warning_checks === RUN TestAgent_HealthServiceByID/warning_checks/format=text === RUN TestAgent_HealthServiceByID/warning_checks/format=json -=== RUN TestAgent_HealthServicesACLEnforcement/root-token-health-by-id === RUN TestAgent_HealthServiceByID/critical_checks === RUN TestAgent_HealthServiceByID/critical_checks/format=text -=== RUN TestAgent_HealthServicesACLEnforcement/root-token-health-by-name === RUN TestAgent_HealthServiceByID/critical_checks/format=json === RUN TestAgent_HealthServiceByID/unknown_serviceid === RUN TestAgent_HealthServiceByID/unknown_serviceid/format=text @@ -44611,705 +50874,368 @@ === RUN TestAgent_HealthServiceByID/maintenance_check_on_node === RUN TestAgent_HealthServiceByID/maintenance_check_on_node/format=text === RUN TestAgent_HealthServiceByID/maintenance_check_on_node/format=json -2021-10-31T23:13:31.025Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:31.025Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:31.130Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:31.130Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:31.132 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0280bd37-9083-357f-3261-821bae56c7e2 Address:127.0.0.1:16720}]" -13:31.133 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16720 [Follower]" leader= -13:31.137 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0280bd37-9083-357f-3261-821bae56c7e2.dc1 127.0.0.1 -13:31.139 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0280bd37-9083-357f-3261-821bae56c7e2 127.0.0.1 -2021-10-31T23:13:31.139Z [INFO] agent.router: Initializing LAN area manager -13:31.139 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16736 network=udp -13:31.139 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16736 network=tcp -13:31.139 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0280bd37-9083-357f-3261-821bae56c7e2.dc1 area=wan -13:31.176 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16716 network=tcp -13:31.176 [INFO] TestAgent: started state syncer -13:31.177 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16728 network=tcp -13:31.182 [INFO] TestAgent.server: Adding LAN server: server="Node-0280bd37-9083-357f-3261-821bae56c7e2 (Addr: tcp/127.0.0.1:16720) (DC: dc1)" -13:31.189 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:31.190 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16720 [Candidate]" term=2 -13:31.190 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:31.190 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:31.190 [DEBUG] TestAgent.server.raft: vote granted: from=0280bd37-9083-357f-3261-821bae56c7e2 term=2 tally=1 -13:31.190 [INFO] TestAgent.server.raft: election won: tally=1 -13:31.190 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16720 [Leader]" -13:31.190 [INFO] TestAgent.server: cluster leadership acquired -13:31.190 [INFO] TestAgent.server: initializing acls -13:31.190 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:31.190 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:31.190 [INFO] TestAgent.server: New leader elected: payload=Node-0280bd37-9083-357f-3261-821bae56c7e2 -13:31.191 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:31.191 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:31.191 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:31.191 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:31.191 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0280bd37-9083-357f-3261-821bae56c7e2 -13:31.191 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0280bd37-9083-357f-3261-821bae56c7e2.dc1 -13:31.192 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:31.193 [INFO] TestAgent.server: Updating LAN server: server="Node-0280bd37-9083-357f-3261-821bae56c7e2 (Addr: tcp/127.0.0.1:16720) (DC: dc1)" -13:31.194 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0280bd37-9083-357f-3261-821bae56c7e2.dc1 area=wan -13:31.198 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:31.198 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:31.198 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:31.198 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:31.198 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:31.198 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0280bd37-9083-357f-3261-821bae56c7e2 -13:31.198 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0280bd37-9083-357f-3261-821bae56c7e2 -13:31.199 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0280bd37-9083-357f-3261-821bae56c7e2 -13:31.199 [INFO] TestAgent.server: federation state anti-entropy synced -13:31.206 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0280bd37-9083-357f-3261-821bae56c7e2 -13:31.206 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0280bd37-9083-357f-3261-821bae56c7e2 -13:31.206 [DEBUG] TestAgent: dropping check from result due to ACLs: check=mysql -13:31.207 [INFO] TestAgent: Requesting shutdown -13:31.207 [INFO] TestAgent.server: shutting down server -13:31.207 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:31.207 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:31.207 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:31.207 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:31.207 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:31.207 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:31.207 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:31.207 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:31.207 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:31.207 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:31.207 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:31.207 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:31.207 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:31.207 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:31.207 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:31.207 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:31.207Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:31.207Z [INFO] agent.router.manager: shutting down -13:31.207 [INFO] TestAgent: consul server down -13:31.207 [INFO] TestAgent: shutdown complete -13:31.207 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16736 network=tcp -13:31.207 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16736 network=udp -13:31.207 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16716 network=tcp -13:31.709 [INFO] TestAgent: Waiting for endpoints to shut down -13:31.710 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Checks_ACLFilter (0.70s) +=== RUN TestAgent_HealthServicesACLEnforcement/no-token-health-by-id +=== RUN TestAgent_HealthServicesACLEnforcement/no-token-health-by-name +=== RUN TestAgent_HealthServicesACLEnforcement/root-token-health-by-id +=== RUN TestAgent_HealthServicesACLEnforcement/root-token-health-by-name +=== RUN TestAgent_Services_ACLFilter/no_token +=== RUN TestAgent_Services_ACLFilter/root_token +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:39.185Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:39.185Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:39.197Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:39.197Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:39.199 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0c69ec06-49bb-031c-d185-9f1ad9c7fab5 Address:127.0.0.1:23561}]" +49:39.199 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23561 [Follower]" leader= +49:39.200 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0c69ec06-49bb-031c-d185-9f1ad9c7fab5.dc1 127.0.0.1 +49:39.200 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0c69ec06-49bb-031c-d185-9f1ad9c7fab5 127.0.0.1 +2022-12-04T05:49:39.200Z [INFO] agent.router: Initializing LAN area manager +49:39.200 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23535 network=udp +49:39.201 [INFO] TestAgent.server: Adding LAN server: server="Node-0c69ec06-49bb-031c-d185-9f1ad9c7fab5 (Addr: tcp/127.0.0.1:23561) (DC: dc1)" +49:39.201 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0c69ec06-49bb-031c-d185-9f1ad9c7fab5.dc1 area=wan +49:39.201 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23535 network=tcp +49:39.201 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23536 network=tcp +49:39.201 [INFO] TestAgent: started state syncer +49:39.201 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23447 network=tcp +49:39.237 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:39.237 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23561 [Candidate]" term=2 +49:39.237 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:39.237 [DEBUG] TestAgent.server.raft: vote granted: from=0c69ec06-49bb-031c-d185-9f1ad9c7fab5 term=2 tally=1 +49:39.237 [INFO] TestAgent.server.raft: election won: tally=1 +49:39.237 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23561 [Leader]" +49:39.237 [INFO] TestAgent.server: cluster leadership acquired +49:39.238 [INFO] TestAgent.server: New leader elected: payload=Node-0c69ec06-49bb-031c-d185-9f1ad9c7fab5 +49:39.238 [INFO] TestAgent.server: initializing acls +49:39.238 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:39.238 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:39.238 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:39.239 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:39.239 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:39.239 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:39.239 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0c69ec06-49bb-031c-d185-9f1ad9c7fab5 +49:39.239 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0c69ec06-49bb-031c-d185-9f1ad9c7fab5.dc1 +49:39.239 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:39.239 [INFO] TestAgent.server: Updating LAN server: server="Node-0c69ec06-49bb-031c-d185-9f1ad9c7fab5 (Addr: tcp/127.0.0.1:23561) (DC: dc1)" +49:39.239 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0c69ec06-49bb-031c-d185-9f1ad9c7fab5.dc1 area=wan +49:39.241 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:39.241 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:39.241 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:39.241 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:39.241 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:39.241 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0c69ec06-49bb-031c-d185-9f1ad9c7fab5 +49:39.241 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0c69ec06-49bb-031c-d185-9f1ad9c7fab5 +49:39.241 [INFO] TestAgent.server: federation state anti-entropy synced +49:39.241 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0c69ec06-49bb-031c-d185-9f1ad9c7fab5 +49:39.312 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0c69ec06-49bb-031c-d185-9f1ad9c7fab5 +49:39.312 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-0c69ec06-49bb-031c-d185-9f1ad9c7fab5 +49:39.312 [DEBUG] TestAgent: dropping check from result due to ACLs: check=mysql +49:39.313 [INFO] TestAgent: Requesting shutdown +49:39.313 [INFO] TestAgent.server: shutting down server +49:39.313 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:39.313 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:39.313 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:39.313 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:39.313 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:39.313 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:39.313 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:39.313 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:39.313 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:39.313 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:39.313 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:39.313 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:39.313 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:39.313 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:39.313 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:39.313 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:39.313Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:39.313Z [INFO] agent.router.manager: shutting down +49:39.313 [INFO] TestAgent: consul server down +49:39.313 [INFO] TestAgent: shutdown complete +49:39.313 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23535 network=tcp +49:39.313 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23535 network=udp +49:39.313 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23536 network=tcp +49:39.813 [INFO] TestAgent: Waiting for endpoints to shut down +49:39.813 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Checks_ACLFilter (0.64s) --- PASS: TestAgent_Checks_ACLFilter/no_token (0.00s) --- PASS: TestAgent_Checks_ACLFilter/root_token (0.00s) === CONT TestAgent_Services_MeshGateway -2021-10-31T23:13:30.935Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:30.935Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:30.958Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:30.958Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:30.960 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:31e2c8a5-fba4-1320-1f07-0bbf863f8c22 Address:127.0.0.1:16671}]" -13:30.961 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-31e2c8a5-fba4-1320-1f07-0bbf863f8c22.dc1 127.0.0.1 -13:30.961 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16671 [Follower]" leader= -13:30.962 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-31e2c8a5-fba4-1320-1f07-0bbf863f8c22 127.0.0.1 -2021-10-31T23:13:30.962Z [INFO] agent.router: Initializing LAN area manager -13:30.962 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16694 network=udp -13:30.962 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-31e2c8a5-fba4-1320-1f07-0bbf863f8c22.dc1 area=wan -13:30.962 [INFO] TestAgent.server: Adding LAN server: server="Node-31e2c8a5-fba4-1320-1f07-0bbf863f8c22 (Addr: tcp/127.0.0.1:16671) (DC: dc1)" -13:30.963 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16694 network=tcp -13:30.963 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16667 network=tcp -13:30.963 [INFO] TestAgent: started state syncer -13:30.963 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16714 network=tcp -13:31.005 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:31.006 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16671 [Candidate]" term=2 -13:31.006 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:31.006 [DEBUG] TestAgent.server.raft: vote granted: from=31e2c8a5-fba4-1320-1f07-0bbf863f8c22 term=2 tally=1 -13:31.006 [INFO] TestAgent.server.raft: election won: tally=1 -13:31.006 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16671 [Leader]" -13:31.006 [INFO] TestAgent.server: cluster leadership acquired -13:31.006 [INFO] TestAgent.server: initializing acls -13:31.006 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:31.006 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:31.007 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:31.007 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:31.007 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:31.007 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:31.007 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-31e2c8a5-fba4-1320-1f07-0bbf863f8c22 -13:31.007 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-31e2c8a5-fba4-1320-1f07-0bbf863f8c22.dc1 -13:31.008 [INFO] TestAgent.server: New leader elected: payload=Node-31e2c8a5-fba4-1320-1f07-0bbf863f8c22 -13:31.008 [INFO] TestAgent.server: Updating LAN server: server="Node-31e2c8a5-fba4-1320-1f07-0bbf863f8c22 (Addr: tcp/127.0.0.1:16671) (DC: dc1)" -13:31.008 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:31.008 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-31e2c8a5-fba4-1320-1f07-0bbf863f8c22.dc1 area=wan -13:31.009 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:31.009 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:31.009 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:31.009 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:31.009 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:31.009 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-31e2c8a5-fba4-1320-1f07-0bbf863f8c22 -13:31.009 [INFO] TestAgent.server: member joined, marking health alive: member=Node-31e2c8a5-fba4-1320-1f07-0bbf863f8c22 -13:31.010 [INFO] TestAgent.server: federation state anti-entropy synced -13:31.010 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-31e2c8a5-fba4-1320-1f07-0bbf863f8c22 -13:31.018 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:31.019 [INFO] TestAgent: Synced node info -13:31.019 [DEBUG] TestAgent: Node info in sync -13:31.294 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-31e2c8a5-fba4-1320-1f07-0bbf863f8c22 -13:31.294 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-31e2c8a5-fba4-1320-1f07-0bbf863f8c22 -13:31.296 [INFO] TestAgent: Requesting shutdown -13:31.297 [INFO] TestAgent.server: shutting down server -13:31.297 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:31.297 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:31.297 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:31.297 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:31.297 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:31.297 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:31.297 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:31.297 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:31.297 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:31.297 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:31.297 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:31.297 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:31.297 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:31.297 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:31.297 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:31.297Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:31.297Z [INFO] agent.router.manager: shutting down -13:31.297 [INFO] TestAgent: consul server down -13:31.297 [INFO] TestAgent: shutdown complete -13:31.297 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16694 network=tcp -13:31.300 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16694 network=udp -13:31.300 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16667 network=tcp -13:31.801 [INFO] TestAgent: Waiting for endpoints to shut down -13:31.801 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Self_ACLDeny (0.89s) +2022-12-04T05:49:39.312Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:39.322Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:39.323 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:847c8302-b9f1-81e1-57d3-10cc6b3904a7 Address:127.0.0.1:23456}]" +49:39.323 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23456 [Follower]" leader= +49:39.324 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-847c8302-b9f1-81e1-57d3-10cc6b3904a7.dc1 127.0.0.1 +49:39.324 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-847c8302-b9f1-81e1-57d3-10cc6b3904a7 127.0.0.1 +2022-12-04T05:49:39.324Z [INFO] agent.router: Initializing LAN area manager +49:39.324 [INFO] TestAgent.server: Adding LAN server: server="Node-847c8302-b9f1-81e1-57d3-10cc6b3904a7 (Addr: tcp/127.0.0.1:23456) (DC: dc1)" +49:39.324 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-847c8302-b9f1-81e1-57d3-10cc6b3904a7.dc1 area=wan +49:39.324 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23479 network=udp +49:39.325 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23479 network=tcp +49:39.325 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23480 network=tcp +49:39.325 [INFO] TestAgent: started state syncer +49:39.325 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23492 network=tcp +49:39.369 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:39.369 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23456 [Candidate]" term=2 +49:39.370 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:39.370 [DEBUG] TestAgent.server.raft: vote granted: from=847c8302-b9f1-81e1-57d3-10cc6b3904a7 term=2 tally=1 +49:39.370 [INFO] TestAgent.server.raft: election won: tally=1 +49:39.370 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23456 [Leader]" +49:39.370 [INFO] TestAgent.server: cluster leadership acquired +49:39.370 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23456 +49:39.371 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:39.372 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:39.372 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:39.372 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:39.372 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:39.372 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:39.372 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-847c8302-b9f1-81e1-57d3-10cc6b3904a7 +49:39.372 [INFO] TestAgent.server: member joined, marking health alive: member=Node-847c8302-b9f1-81e1-57d3-10cc6b3904a7 +49:39.372 [INFO] TestAgent.server: New leader elected: payload=Node-847c8302-b9f1-81e1-57d3-10cc6b3904a7 +49:39.373 [INFO] TestAgent.server: federation state anti-entropy synced +49:39.396 [INFO] TestAgent: Requesting shutdown +49:39.396 [INFO] TestAgent.server: shutting down server +49:39.396 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:39.396 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:39.396 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:39.396 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:39.396 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:39.396 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:39.397 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:39.396 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:39.396 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:39.396 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:39.396 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:39.397 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:39.397Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:39.397Z [INFO] agent.router.manager: shutting down +49:39.397 [INFO] TestAgent: consul server down +49:39.397 [INFO] TestAgent: shutdown complete +49:39.397 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23479 network=tcp +49:39.397 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23479 network=udp +49:39.397 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23480 network=tcp +49:39.897 [INFO] TestAgent: Waiting for endpoints to shut down +49:39.897 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ChecksWithFilter (0.60s) +=== CONT TestAgent_Services_Sidecar +2022-12-04T05:49:38.991Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:38.991Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:39.001Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:39.001Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:39.003 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:159ce85b-1421-bb20-8e6b-21b72a11bce5 Address:127.0.0.1:23432}]" +49:39.003 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23432 [Follower]" leader= +49:39.003 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-159ce85b-1421-bb20-8e6b-21b72a11bce5.dc1 127.0.0.1 +49:39.003 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-159ce85b-1421-bb20-8e6b-21b72a11bce5 127.0.0.1 +2022-12-04T05:49:39.003Z [INFO] agent.router: Initializing LAN area manager +49:39.004 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23427 network=udp +49:39.004 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-159ce85b-1421-bb20-8e6b-21b72a11bce5.dc1 area=wan +49:39.004 [INFO] TestAgent.server: Adding LAN server: server="Node-159ce85b-1421-bb20-8e6b-21b72a11bce5 (Addr: tcp/127.0.0.1:23432) (DC: dc1)" +49:39.004 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23427 network=tcp +49:39.004 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23428 network=tcp +49:39.004 [INFO] TestAgent: started state syncer +49:39.004 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23562 network=tcp +49:39.054 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:39.069 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:39.069 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23432 [Candidate]" term=2 +49:39.070 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:39.070 [DEBUG] TestAgent.server.raft: vote granted: from=159ce85b-1421-bb20-8e6b-21b72a11bce5 term=2 tally=1 +49:39.070 [INFO] TestAgent.server.raft: election won: tally=1 +49:39.070 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23432 [Leader]" +49:39.070 [INFO] TestAgent.server: cluster leadership acquired +49:39.070 [INFO] TestAgent.server: New leader elected: payload=Node-159ce85b-1421-bb20-8e6b-21b72a11bce5 +49:39.070 [INFO] TestAgent.server: initializing acls +49:39.070 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:39.070 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:39.071 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:39.071 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:39.071 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:39.071 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:39.071 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-159ce85b-1421-bb20-8e6b-21b72a11bce5 +49:39.071 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-159ce85b-1421-bb20-8e6b-21b72a11bce5.dc1 +49:39.071 [INFO] TestAgent.server: Updating LAN server: server="Node-159ce85b-1421-bb20-8e6b-21b72a11bce5 (Addr: tcp/127.0.0.1:23432) (DC: dc1)" +49:39.071 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-159ce85b-1421-bb20-8e6b-21b72a11bce5.dc1 area=wan +49:39.072 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:39.073 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:39.073 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:39.073 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:39.073 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:39.073 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:39.073 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-159ce85b-1421-bb20-8e6b-21b72a11bce5 +49:39.073 [INFO] TestAgent.server: member joined, marking health alive: member=Node-159ce85b-1421-bb20-8e6b-21b72a11bce5 +49:39.074 [INFO] TestAgent.server: federation state anti-entropy synced +49:39.074 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-159ce85b-1421-bb20-8e6b-21b72a11bce5 +49:39.280 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:39.281 [INFO] TestAgent: Synced node info +49:39.410 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-159ce85b-1421-bb20-8e6b-21b72a11bce5 +49:39.411 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-159ce85b-1421-bb20-8e6b-21b72a11bce5 +49:39.411 [INFO] TestAgent: Requesting shutdown +49:39.411 [INFO] TestAgent.server: shutting down server +49:39.411 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:39.412 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:39.412 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:39.412 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:39.412 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:39.412 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:39.412 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:39.412 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:39.412 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:39.412 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:39.412 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:39.412 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:39.412 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:39.412 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:39.412 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:39.412Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:39.412Z [INFO] agent.router.manager: shutting down +49:39.412 [INFO] TestAgent: consul server down +49:39.412 [INFO] TestAgent: shutdown complete +49:39.412 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23427 network=tcp +49:39.412 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23427 network=udp +49:39.412 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23428 network=tcp +49:39.913 [INFO] TestAgent: Waiting for endpoints to shut down +49:39.913 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Metrics_ACLDeny (0.93s) + --- PASS: TestAgent_Metrics_ACLDeny/no_token (0.00s) + --- PASS: TestAgent_Metrics_ACLDeny/agent_master_token (0.00s) + --- PASS: TestAgent_Metrics_ACLDeny/read-only_token (0.00s) +=== CONT TestAgent_Services_ExternalConnectProxy +2022-12-04T05:49:39.048Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:39.048Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:39.058Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:39.058Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:39.059 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:949a5a0c-da81-37b7-d5c4-e27030ca02d2 Address:127.0.0.1:23491}]" +49:39.060 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23491 [Follower]" leader= +49:39.060 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-949a5a0c-da81-37b7-d5c4-e27030ca02d2.dc1 127.0.0.1 +49:39.060 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-949a5a0c-da81-37b7-d5c4-e27030ca02d2 127.0.0.1 +2022-12-04T05:49:39.060Z [INFO] agent.router: Initializing LAN area manager +49:39.061 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-949a5a0c-da81-37b7-d5c4-e27030ca02d2.dc1 area=wan +49:39.061 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23549 network=udp +49:39.061 [INFO] TestAgent.server: Adding LAN server: server="Node-949a5a0c-da81-37b7-d5c4-e27030ca02d2 (Addr: tcp/127.0.0.1:23491) (DC: dc1)" +49:39.061 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23549 network=tcp +49:39.061 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23550 network=tcp +49:39.061 [INFO] TestAgent: started state syncer +49:39.061 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23555 network=tcp +49:39.106 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:39.106 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23491 [Candidate]" term=2 +49:39.106 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:39.106 [DEBUG] TestAgent.server.raft: vote granted: from=949a5a0c-da81-37b7-d5c4-e27030ca02d2 term=2 tally=1 +49:39.106 [INFO] TestAgent.server.raft: election won: tally=1 +49:39.106 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23491 [Leader]" +49:39.106 [INFO] TestAgent.server: cluster leadership acquired +49:39.107 [INFO] TestAgent.server: New leader elected: payload=Node-949a5a0c-da81-37b7-d5c4-e27030ca02d2 +49:39.107 [INFO] TestAgent.server: initializing acls +49:39.107 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:39.107 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:39.107 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:39.108 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:39.108 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:39.108 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:39.108 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-949a5a0c-da81-37b7-d5c4-e27030ca02d2 +49:39.108 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-949a5a0c-da81-37b7-d5c4-e27030ca02d2.dc1 +49:39.108 [INFO] TestAgent.server: Updating LAN server: server="Node-949a5a0c-da81-37b7-d5c4-e27030ca02d2 (Addr: tcp/127.0.0.1:23491) (DC: dc1)" +49:39.108 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-949a5a0c-da81-37b7-d5c4-e27030ca02d2.dc1 area=wan +49:39.108 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:39.110 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:39.110 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:39.110 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:39.110 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:39.110 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:39.110 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-949a5a0c-da81-37b7-d5c4-e27030ca02d2 +49:39.110 [INFO] TestAgent.server: member joined, marking health alive: member=Node-949a5a0c-da81-37b7-d5c4-e27030ca02d2 +49:39.110 [INFO] TestAgent.server: federation state anti-entropy synced +49:39.110 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-949a5a0c-da81-37b7-d5c4-e27030ca02d2 +49:39.248 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:39.256 [INFO] TestAgent: Synced node info +49:39.502 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-949a5a0c-da81-37b7-d5c4-e27030ca02d2 +49:39.502 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-949a5a0c-da81-37b7-d5c4-e27030ca02d2 +49:39.504 [INFO] TestAgent: Requesting shutdown +49:39.504 [INFO] TestAgent.server: shutting down server +49:39.504 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:39.504 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:39.504 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:39.504 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:39.504 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:39.504 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:39.504 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:39.504 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:39.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:39.504 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:39.504 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:39.504 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:39.504 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:39.504 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:39.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:49:39.504Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:39.504Z [INFO] agent.router.manager: shutting down +49:39.504 [INFO] TestAgent: consul server down +49:39.504 [INFO] TestAgent: shutdown complete +49:39.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23549 network=tcp +49:39.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23549 network=udp +49:39.504 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23550 network=tcp +49:40.005 [INFO] TestAgent: Waiting for endpoints to shut down +49:40.005 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Self_ACLDeny (0.97s) --- PASS: TestAgent_Self_ACLDeny/no_token (0.00s) --- PASS: TestAgent_Self_ACLDeny/agent_master_token (0.00s) --- PASS: TestAgent_Self_ACLDeny/read-only_token (0.00s) -=== CONT TestAgent_Services_Sidecar -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=true -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestAgent_Services_ACLFilter/no_token -=== RUN TestAgent_Services_ACLFilter/root_token -2021-10-31T23:13:25.003Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:25.155Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:25.156 [INFO] bonnie.server.gateway_locator: will dial the primary datacenter through its mesh gateways -13:25.167 [INFO] bonnie.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ab5ef13d-175e-3f8c-d264-984ac532a13a Address:127.0.0.1:16919}]" -13:25.167 [INFO] bonnie.server.serf.wan: serf: EventMemberJoin: bonnie.dc3 127.0.0.1 -13:25.168 [INFO] bonnie.server.serf.lan: serf: EventMemberJoin: bonnie 127.0.0.1 -2021-10-31T23:13:25.168Z [INFO] agent.router: Initializing LAN area manager -13:25.168 [INFO] bonnie: Started DNS server: address=127.0.0.1:16268 network=udp -13:25.168 [INFO] bonnie.server.raft: entering follower state: follower="Node at 127.0.0.1:16919 [Follower]" leader= -13:25.169 [INFO] bonnie.server: Adding LAN server: server="bonnie (Addr: tcp/127.0.0.1:16919) (DC: dc3)" -13:25.169 [INFO] bonnie.server: Handled event for server in area: event=member-join server=bonnie.dc3 area=wan -13:25.169 [INFO] bonnie: Started DNS server: address=127.0.0.1:16268 network=tcp -13:25.182 [INFO] bonnie: Started HTTP server: address=127.0.0.1:16157 network=tcp -13:25.182 [INFO] bonnie: started state syncer -13:25.182 [INFO] bonnie: Started gRPC server: address=127.0.0.1:16281 network=tcp -13:25.182 [INFO] bonnie: Refreshing mesh gateways is supported for the following discovery methods: discovery_methods="aliyun aws gce mdns os packet" -13:25.182 [INFO] bonnie: Refreshing mesh gateways... -13:25.182 [INFO] bonnie.server.gateway_locator: updated fallback list of primary mesh gateways: mesh_gateways=[127.0.0.1:16970] -13:25.182 [INFO] bonnie: Refreshing mesh gateways completed -13:25.182 [INFO] bonnie: Retry join is supported for the following discovery methods: cluster=WAN discovery_methods="aliyun aws gce mdns os packet" -13:25.182 [INFO] bonnie: Joining cluster...: cluster=WAN -13:25.182 [INFO] bonnie: (WAN) joining: wan_addresses=[*.dc1/192.0.2.2] -13:25.225 [WARN] bonnie.server.raft: heartbeat timeout reached, starting election: last-leader= -13:25.226 [INFO] bonnie.server.raft: entering candidate state: node="Node at 127.0.0.1:16919 [Candidate]" term=2 -13:25.226 [DEBUG] bonnie.server.raft: votes: needed=1 -13:25.226 [DEBUG] bonnie.server.raft: vote granted: from=ab5ef13d-175e-3f8c-d264-984ac532a13a term=2 tally=1 -13:25.226 [INFO] bonnie.server.raft: election won: tally=1 -13:25.226 [INFO] bonnie.server.raft: entering leader state: leader="Node at 127.0.0.1:16919 [Leader]" -13:25.226 [INFO] bonnie.server: cluster leadership acquired -13:25.226 [DEBUG] bonnie.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false -13:25.227 [WARN] bonnie.server.connect: primary datacenter is configured but unreachable - deferring initialization of the secondary datacenter CA -13:25.227 [INFO] bonnie.leader: started routine: routine="config entry replication" -13:25.227 [INFO] bonnie.leader: started routine: routine="federation state replication" -13:25.227 [INFO] bonnie.leader: started routine: routine="federation state anti-entropy" -13:25.227 [INFO] bonnie.leader: started routine: routine="secondary CA roots watch" -13:25.227 [INFO] bonnie.leader: started routine: routine="intention replication" -13:25.227 [INFO] bonnie.leader: started routine: routine="intermediate cert renew watch" -13:25.227 [INFO] bonnie.leader: started routine: routine="CA root pruning" -13:25.227 [DEBUG] bonnie.server: Skipping self join check for node since the cluster is too small: node=bonnie -13:25.227 [INFO] bonnie.server: member joined, marking health alive: member=bonnie -13:25.228 [INFO] bonnie.server: New leader elected: payload=bonnie -13:25.228 [WARN] bonnie.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConfigEntry.ListAll -13:25.228 [DEBUG] bonnie.server: federation states are not enabled in the primary dc -13:25.228 [DEBUG] bonnie.server: federation states are not enabled in the primary dc -13:25.228 [DEBUG] bonnie.server: federation states are not enabled in the primary dc -13:25.228 [DEBUG] bonnie.server: federation states are not enabled in the primary dc -13:25.228 [DEBUG] bonnie.server: federation states are not enabled in the primary dc -13:25.228 [DEBUG] bonnie.server: federation states are not enabled in the primary dc -13:25.228 [DEBUG] bonnie.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 -13:25.228 [WARN] bonnie.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConnectCA.Roots -13:25.228 [ERROR] bonnie.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: No path to datacenter" -13:25.229 [DEBUG] bonnie.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 -13:25.229 [WARN] bonnie.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=Intention.List -13:25.229 [ERROR] bonnie.server.connect: error replicating intentions: routine="intention replication" error="No path to datacenter" -13:25.256 [DEBUG] bonnie.server.memberlist.wan: memberlist: Initiating push/pull sync with: *.dc1 127.0.0.1:16970 -13:25.262 [INFO] bonnie.server.serf.wan: serf: EventMemberJoin: betty.dc2 127.0.0.1 -13:25.262 [INFO] bonnie.server.serf.wan: serf: EventMemberJoin: bob.dc1 127.0.0.1 -13:25.262 [INFO] bonnie: (WAN) joined: number_of_nodes=1 -13:25.262 [INFO] bonnie: Join cluster completed. Synced with initial agents: cluster=WAN num_agents=1 -13:25.262 [INFO] bonnie.server: Handled event for server in area: event=member-join server=betty.dc2 area=wan -13:25.262 [INFO] bonnie.server: Handled event for server in area: event=member-join server=bob.dc1 area=wan -13:25.358 [DEBUG] bonnie: Skipping remote check since it is managed automatically: check=serfHealth -13:25.359 [INFO] bonnie: Synced node info -13:25.589 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: betty.dc2 -13:25.670 [ERROR] bonnie.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:16202: Remote DC has no server currently reachable -13:25.705 [DEBUG] bonnie: Node info in sync -13:25.721 [INFO] bonnie: Synced service: service=mesh-gateway -13:25.722 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.722 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.722 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.723 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:26.010 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: betty.dc2 -13:26.010 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: bonnie.dc3 -13:26.170 [ERROR] bonnie.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:16202: Remote DC has no server currently reachable -13:26.282 [DEBUG] bonnie.server.replication.config_entry: finished fetching config entries: amount=0 -13:26.282 [DEBUG] bonnie.server.replication.config_entry: Config Entry replication: local=0 remote=0 -13:26.282 [DEBUG] bonnie.server.replication.config_entry: Config Entry replication: deletions=0 updates=0 -13:26.282 [DEBUG] bonnie.server.replication.config_entry: replication completed through remote index: index=1 -13:26.303 [INFO] bonnie.server.gateway_locator: will dial the primary datacenter using our local mesh gateways if possible -13:26.303 [DEBUG] bonnie.server: finished fetching remote objects: amount=1 -13:26.303 [DEBUG] bonnie.server: diffing replication state: local_amount=0 remote_amount=1 -13:26.303 [DEBUG] bonnie.server: diffed replication state: deletions=0 updates=1 -13:26.303 [DEBUG] bonnie.server: performing updates: updates=1 -13:26.303 [DEBUG] bonnie.server: finished updates -13:26.303 [DEBUG] bonnie.replication.federation_state: replication completed through remote index: index=14 -13:26.518 [DEBUG] bonnie.server.serf.wan: serf: messageJoinType: bonnie.dc3 -13:27.170 [INFO] bonnie.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:16970] local=[] -13:27.254 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:4a:c6:ab:ef:63:c9:60:1a:51:7f:19:62:e3:e9:d9:0e:76:55:10:6e:74:24:69:28:a1:6c:b8:b9:8f:fd:89 is_primary=false -13:27.254 [DEBUG] bonnie: Skipping remote check since it is managed automatically: check=serfHealth -13:27.254 [DEBUG] bonnie: Node info in sync -13:27.254 [DEBUG] bonnie: Service in sync: service=mesh-gateway -13:27.254 [DEBUG] bonnie: Node info in sync -13:27.254 [DEBUG] bonnie: Service in sync: service=mesh-gateway -13:27.266 [INFO] bonnie.server.connect: received new intermediate certificate from primary datacenter -13:27.274 [INFO] bonnie.server.connect: updated root certificates from primary datacenter -13:27.906 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:28.119 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:28.334 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:28.359 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:28.359 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:28.359 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:28.359 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:28.359 [ERROR] bonnie.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc2 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:29.232 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16971 datacenter=dc2 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:29.662 [DEBUG] bonnie.server: finished fetching remote objects: amount=2 -13:29.662 [DEBUG] bonnie.server: diffing replication state: local_amount=1 remote_amount=2 -13:29.662 [DEBUG] bonnie.server: diffed replication state: deletions=0 updates=1 -13:29.662 [DEBUG] bonnie.server: performing updates: updates=1 -13:29.662 [DEBUG] bonnie.server: finished updates -13:29.662 [DEBUG] bonnie.replication.federation_state: replication completed through remote index: index=17 -13:30.234 [DEBUG] bonnie.server: finished fetching remote objects: amount=3 -13:30.234 [DEBUG] bonnie.server: diffing replication state: local_amount=2 remote_amount=3 -13:30.234 [DEBUG] bonnie.server: diffed replication state: deletions=0 updates=1 -13:30.234 [DEBUG] bonnie.server: performing updates: updates=1 -13:30.234 [INFO] bonnie.server: federation state anti-entropy synced -13:30.234 [DEBUG] bonnie.server: finished updates -13:30.234 [DEBUG] bonnie.replication.federation_state: replication completed through remote index: index=18 -13:30.235 [INFO] bonnie.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:16970] local=[127.0.0.1:16970] -13:30.451 [INFO] bonnie: Requesting shutdown -13:30.451 [INFO] bonnie.server: shutting down server -13:30.451 [DEBUG] bonnie.leader: stopping routine: routine="config entry replication" -13:30.451 [DEBUG] bonnie.leader: stopping routine: routine="federation state replication" -13:30.451 [DEBUG] bonnie.leader: stopping routine: routine="federation state anti-entropy" -13:30.451 [DEBUG] bonnie.leader: stopping routine: routine="secondary CA roots watch" -13:30.451 [DEBUG] bonnie.leader: stopping routine: routine="intention replication" -13:30.451 [DEBUG] bonnie.leader: stopping routine: routine="intermediate cert renew watch" -13:30.451 [DEBUG] bonnie.leader: stopping routine: routine="CA root pruning" -13:30.451 [WARN] bonnie.server.serf.lan: serf: Shutdown without a Leave -13:30.452 [DEBUG] bonnie.leader: stopped routine: routine="federation state anti-entropy" -13:30.452 [DEBUG] bonnie.leader: stopped routine: routine="intermediate cert renew watch" -13:30.452 [DEBUG] bonnie.leader: stopped routine: routine="CA root pruning" -13:30.452 [WARN] bonnie.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:30.452Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.452Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.452Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.452Z [INFO] agent.router.manager: shutting down -13:30.452 [INFO] bonnie: consul server down -13:30.452 [INFO] bonnie: shutdown complete -13:30.452 [INFO] bonnie: Stopping server: protocol=DNS address=127.0.0.1:16268 network=tcp -13:30.452 [INFO] bonnie: Stopping server: protocol=DNS address=127.0.0.1:16268 network=udp -13:30.452 [INFO] bonnie: Stopping server: protocol=HTTP address=127.0.0.1:16157 network=tcp -13:30.453 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16941 datacenter=dc1 method=Internal.ServiceDump error="rpc error making call: EOF" -13:30.453 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16941 datacenter=dc1 method=ConfigEntry.ListAll error="rpc error making call: EOF" -13:30.453 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16941 datacenter=dc1 method=FederationState.List error="rpc error making call: EOF" -13:30.453 [WARN] bonnie.replication.federation_state: replication error (will retry if still leader): error="failed to retrieve federation states: rpc error making call: EOF" -13:30.453 [INFO] bonnie.replication.federation_state: stopped replication -13:30.453 [DEBUG] bonnie.leader: stopped routine: routine="federation state replication" -13:30.453 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16941 datacenter=dc1 method=Intention.List error="rpc error making call: EOF" -13:30.453 [ERROR] bonnie.server.connect: error replicating intentions: routine="intention replication" error="rpc error making call: EOF" -13:30.453 [DEBUG] bonnie.leader: stopped routine: routine="intention replication" -13:30.453 [ERROR] bonnie.server.rpc: RPC failed to server in DC: server=127.0.0.1:16941 datacenter=dc1 method=ConnectCA.Roots error="rpc error making call: EOF" -13:30.453 [ERROR] bonnie.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: rpc error making call: EOF" -13:30.453 [DEBUG] bonnie.leader: stopped routine: routine="secondary CA roots watch" -13:30.453 [INFO] bonnie.server.replication.config_entry: stopped replication -13:30.453 [DEBUG] bonnie.leader: stopped routine: routine="config entry replication" -13:30.953 [INFO] bonnie: Waiting for endpoints to shut down -13:30.953 [INFO] bonnie: Endpoints down -2021-10-31T23:13:24.548Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:24.593Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:24.598 [INFO] betty.server.gateway_locator: will dial the primary datacenter through its mesh gateways -13:24.600 [INFO] betty.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d0009807-ce6b-4f19-d3d4-99dfe6cba4e6 Address:127.0.0.1:16971}]" -13:24.601 [INFO] betty.server.serf.wan: serf: EventMemberJoin: betty.dc2 127.0.0.1 -13:24.601 [INFO] betty.server.serf.lan: serf: EventMemberJoin: betty 127.0.0.1 -2021-10-31T23:13:24.601Z [INFO] agent.router: Initializing LAN area manager -13:24.602 [INFO] betty.server.raft: entering follower state: follower="Node at 127.0.0.1:16971 [Follower]" leader= -13:24.602 [INFO] betty.server: Adding LAN server: server="betty (Addr: tcp/127.0.0.1:16971) (DC: dc2)" -13:24.602 [INFO] betty.server: Handled event for server in area: event=member-join server=betty.dc2 area=wan -13:24.610 [INFO] betty: Started DNS server: address=127.0.0.1:16149 network=udp -13:24.610 [INFO] betty: Started DNS server: address=127.0.0.1:16149 network=tcp -13:24.610 [INFO] betty: Started HTTP server: address=127.0.0.1:16199 network=tcp -13:24.610 [INFO] betty: started state syncer -13:24.611 [INFO] betty: Started gRPC server: address=127.0.0.1:16176 network=tcp -13:24.611 [INFO] betty: Refreshing mesh gateways is supported for the following discovery methods: discovery_methods="aliyun aws gce mdns os packet" -13:24.611 [INFO] betty: Refreshing mesh gateways... -13:24.611 [INFO] betty.server.gateway_locator: updated fallback list of primary mesh gateways: mesh_gateways=[127.0.0.1:16970] -13:24.611 [INFO] betty: Refreshing mesh gateways completed -13:24.611 [INFO] betty: Retry join is supported for the following discovery methods: cluster=WAN discovery_methods="aliyun aws gce mdns os packet" -13:24.611 [INFO] betty: Joining cluster...: cluster=WAN -13:24.611 [INFO] betty: (WAN) joining: wan_addresses=[*.dc1/192.0.2.2] -13:24.651 [WARN] betty.server.raft: heartbeat timeout reached, starting election: last-leader= -13:24.651 [INFO] betty.server.raft: entering candidate state: node="Node at 127.0.0.1:16971 [Candidate]" term=2 -13:24.651 [DEBUG] betty.server.raft: votes: needed=1 -13:24.651 [DEBUG] betty.server.raft: vote granted: from=d0009807-ce6b-4f19-d3d4-99dfe6cba4e6 term=2 tally=1 -13:24.651 [INFO] betty.server.raft: election won: tally=1 -13:24.651 [INFO] betty.server.raft: entering leader state: leader="Node at 127.0.0.1:16971 [Leader]" -13:24.651 [INFO] betty.server: cluster leadership acquired -13:24.651 [DEBUG] betty.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false -13:24.652 [WARN] betty.server.connect: primary datacenter is configured but unreachable - deferring initialization of the secondary datacenter CA -13:24.652 [INFO] betty.leader: started routine: routine="config entry replication" -13:24.652 [INFO] betty.leader: started routine: routine="federation state replication" -13:24.652 [INFO] betty.leader: started routine: routine="federation state anti-entropy" -13:24.652 [INFO] betty.leader: started routine: routine="secondary CA roots watch" -13:24.652 [INFO] betty.leader: started routine: routine="intention replication" -13:24.652 [INFO] betty.leader: started routine: routine="intermediate cert renew watch" -13:24.652 [INFO] betty.leader: started routine: routine="CA root pruning" -13:24.652 [DEBUG] betty.server: Skipping self join check for node since the cluster is too small: node=betty -13:24.652 [INFO] betty.server: member joined, marking health alive: member=betty -13:24.653 [INFO] betty.server: New leader elected: payload=betty -13:24.653 [DEBUG] betty.server: federation states are not enabled in the primary dc -13:24.666 [DEBUG] betty.server: federation states are not enabled in the primary dc -13:24.666 [DEBUG] betty.server: federation states are not enabled in the primary dc -13:24.666 [DEBUG] betty.server: federation states are not enabled in the primary dc -13:24.666 [DEBUG] betty.server: federation states are not enabled in the primary dc -13:24.653 [DEBUG] betty.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 -13:24.666 [WARN] betty.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConnectCA.Roots -13:24.666 [ERROR] betty.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: No path to datacenter" -13:24.653 [DEBUG] betty.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 -13:24.666 [WARN] betty.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=Intention.List -13:24.666 [ERROR] betty.server.connect: error replicating intentions: routine="intention replication" error="No path to datacenter" -13:24.653 [DEBUG] betty.server: federation states are not enabled in the primary dc -13:24.653 [WARN] betty.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConfigEntry.ListAll -13:24.655 [DEBUG] betty.server.memberlist.wan: memberlist: Initiating push/pull sync with: *.dc1 127.0.0.1:16970 -13:24.668 [INFO] betty.server.serf.wan: serf: EventMemberJoin: bob.dc1 127.0.0.1 -13:24.668 [INFO] betty: (WAN) joined: number_of_nodes=1 -13:24.668 [INFO] betty: Join cluster completed. Synced with initial agents: cluster=WAN num_agents=1 -13:24.668 [INFO] betty.server: Handled event for server in area: event=member-join server=bob.dc1 area=wan -13:24.858 [DEBUG] betty: Skipping remote check since it is managed automatically: check=serfHealth -13:24.859 [INFO] betty: Synced node info -13:24.859 [DEBUG] betty: Node info in sync -13:25.517 [INFO] betty.server.serf.wan: serf: EventMemberJoin: bonnie.dc3 127.0.0.1 -13:25.517 [DEBUG] betty.server.serf.wan: serf: messageJoinType: betty.dc2 -13:25.517 [INFO] betty.server: Handled event for server in area: event=member-join server=bonnie.dc3 area=wan -13:25.603 [ERROR] betty.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:16160: Remote DC has no server currently reachable -13:25.704 [DEBUG] betty: Node info in sync -13:25.705 [INFO] betty: Synced service: service=mesh-gateway -13:25.707 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:16919 datacenter=dc3 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.707 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:16919 datacenter=dc3 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.707 [ERROR] betty.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc3 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.707 [ERROR] betty.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc3 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.707 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:16919 datacenter=dc3 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.707 [ERROR] betty.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc3 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.707 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:16919 datacenter=dc3 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.707 [ERROR] betty.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc3 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:25.718 [DEBUG] betty.server.replication.config_entry: finished fetching config entries: amount=0 -13:25.718 [DEBUG] betty.server.replication.config_entry: Config Entry replication: local=0 remote=0 -13:25.718 [DEBUG] betty.server.replication.config_entry: Config Entry replication: deletions=0 updates=0 -13:25.718 [DEBUG] betty.server.replication.config_entry: replication completed through remote index: index=1 -13:25.718 [INFO] betty.server.gateway_locator: will dial the primary datacenter using our local mesh gateways if possible -13:25.718 [DEBUG] betty.server: finished fetching remote objects: amount=1 -13:25.718 [DEBUG] betty.server: diffing replication state: local_amount=0 remote_amount=1 -13:25.718 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 -13:25.718 [DEBUG] betty.server: performing updates: updates=1 -13:25.719 [DEBUG] betty.server: finished updates -13:25.719 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=14 -13:26.010 [DEBUG] betty.server.serf.wan: serf: messageJoinType: betty.dc2 -13:26.010 [DEBUG] betty.server.serf.wan: serf: messageJoinType: bonnie.dc3 -13:26.102 [ERROR] betty.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:16160: Remote DC has no server currently reachable -13:26.518 [DEBUG] betty.server.serf.wan: serf: messageJoinType: bonnie.dc3 -13:26.601 [ERROR] betty.server.memberlist.wan: memberlist: Failed to send gossip to 127.0.0.1:16160: Remote DC has no server currently reachable -13:26.603 [INFO] betty.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:16970] local=[] -13:26.675 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:4a:c6:ab:ef:63:c9:60:1a:51:7f:19:62:e3:e9:d9:0e:76:55:10:6e:74:24:69:28:a1:6c:b8:b9:8f:fd:89 is_primary=false -13:26.684 [INFO] betty.server.connect: received new intermediate certificate from primary datacenter -13:26.684 [INFO] betty.server.connect: updated root certificates from primary datacenter -13:26.686 [DEBUG] betty: Skipping remote check since it is managed automatically: check=serfHealth -13:26.686 [DEBUG] betty: Node info in sync -13:26.686 [DEBUG] betty: Service in sync: service=mesh-gateway -13:26.686 [DEBUG] betty: Node info in sync -13:26.686 [DEBUG] betty: Service in sync: service=mesh-gateway -13:28.894 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:16919 datacenter=dc3 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:29.610 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:16919 datacenter=dc3 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:29.610 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:16919 datacenter=dc3 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:29.610 [ERROR] betty.proxycfg: watch error: service_id=mesh-gateway id=mesh-gateway:dc3 error="error filling agent cache: rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:29.610 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:16919 datacenter=dc3 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:29.610 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:16919 datacenter=dc3 method=Internal.ServiceDump error="rpc error getting client: failed to get conn: Remote DC has no server currently reachable" -13:29.662 [DEBUG] betty.server: finished fetching remote objects: amount=2 -13:29.662 [DEBUG] betty.server: diffing replication state: local_amount=1 remote_amount=2 -13:29.662 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 -13:29.662 [DEBUG] betty.server: performing updates: updates=1 -13:29.663 [DEBUG] betty.server: finished updates -13:29.663 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=17 -13:29.663 [INFO] betty.server: federation state anti-entropy synced -13:29.663 [INFO] betty.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:16970] local=[127.0.0.1:16970] -13:30.234 [DEBUG] betty.server: finished fetching remote objects: amount=3 -13:30.234 [DEBUG] betty.server: diffing replication state: local_amount=2 remote_amount=3 -13:30.234 [DEBUG] betty.server: diffed replication state: deletions=0 updates=1 -13:30.234 [DEBUG] betty.server: performing updates: updates=1 -13:30.234 [DEBUG] betty.server: finished updates -13:30.234 [DEBUG] betty.replication.federation_state: replication completed through remote index: index=18 -13:30.953 [INFO] betty: Requesting shutdown -13:30.954 [INFO] betty.server: shutting down server -13:30.954 [DEBUG] betty.leader: stopping routine: routine="federation state anti-entropy" -13:30.954 [DEBUG] betty.leader: stopping routine: routine="secondary CA roots watch" -13:30.954 [DEBUG] betty.leader: stopping routine: routine="intention replication" -13:30.954 [DEBUG] betty.leader: stopping routine: routine="intermediate cert renew watch" -13:30.954 [DEBUG] betty.leader: stopping routine: routine="CA root pruning" -13:30.954 [DEBUG] betty.leader: stopping routine: routine="config entry replication" -13:30.954 [DEBUG] betty.leader: stopping routine: routine="federation state replication" -13:30.954 [WARN] betty.server.serf.lan: serf: Shutdown without a Leave -13:30.954 [DEBUG] betty.leader: stopped routine: routine="federation state anti-entropy" -13:30.954 [DEBUG] betty.leader: stopped routine: routine="intermediate cert renew watch" -13:30.954 [DEBUG] betty.leader: stopped routine: routine="CA root pruning" -13:30.954 [WARN] betty.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:30.954Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.954Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.954Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:30.954Z [INFO] agent.router.manager: shutting down -13:30.955 [INFO] betty: consul server down -13:30.955 [INFO] betty: shutdown complete -13:30.955 [INFO] betty: Stopping server: protocol=DNS address=127.0.0.1:16149 network=tcp -13:30.955 [INFO] betty: Stopping server: protocol=DNS address=127.0.0.1:16149 network=udp -13:30.955 [INFO] betty: Stopping server: protocol=HTTP address=127.0.0.1:16199 network=tcp -13:30.955 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:16941 datacenter=dc1 method=FederationState.List error="rpc error making call: EOF" -13:30.955 [WARN] betty.replication.federation_state: replication error (will retry if still leader): error="failed to retrieve federation states: rpc error making call: EOF" -13:30.955 [INFO] betty.replication.federation_state: stopped replication -13:30.955 [DEBUG] betty.leader: stopped routine: routine="federation state replication" -13:30.955 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:16941 datacenter=dc1 method=Internal.ServiceDump error="rpc error making call: EOF" -13:30.955 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:16941 datacenter=dc1 method=Intention.List error="rpc error making call: EOF" -13:30.955 [ERROR] betty.server.connect: error replicating intentions: routine="intention replication" error="rpc error making call: EOF" -13:30.955 [DEBUG] betty.leader: stopped routine: routine="intention replication" -13:30.955 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:16941 datacenter=dc1 method=ConnectCA.Roots error="rpc error making call: EOF" -13:30.955 [ERROR] betty.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: rpc error making call: EOF" -13:30.955 [DEBUG] betty.leader: stopped routine: routine="secondary CA roots watch" -13:30.955 [ERROR] betty.server.rpc: RPC failed to server in DC: server=127.0.0.1:16941 datacenter=dc1 method=ConfigEntry.ListAll error="rpc error making call: EOF" -13:30.955 [INFO] betty.server.replication.config_entry: stopped replication -13:30.955 [DEBUG] betty.leader: stopped routine: routine="config entry replication" -13:31.462 [INFO] betty: Waiting for endpoints to shut down -13:31.462 [INFO] betty: Endpoints down -2021-10-31T23:13:22.398Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:22.488Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:22.507 [INFO] bob.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5bb346df-8dfc-19bc-e25f-ebfd6f0a3014 Address:127.0.0.1:16941}]" -13:22.508 [INFO] bob.server.serf.wan: serf: EventMemberJoin: bob.dc1 127.0.0.1 -13:22.509 [INFO] bob.server.serf.lan: serf: EventMemberJoin: bob 127.0.0.1 -2021-10-31T23:13:22.509Z [INFO] agent.router: Initializing LAN area manager -13:22.509 [INFO] bob: Started DNS server: address=127.0.0.1:16920 network=udp -13:22.509 [INFO] bob.server.raft: entering follower state: follower="Node at 127.0.0.1:16941 [Follower]" leader= -13:22.510 [INFO] bob.server: Adding LAN server: server="bob (Addr: tcp/127.0.0.1:16941) (DC: dc1)" -13:22.510 [INFO] bob.server: Handled event for server in area: event=member-join server=bob.dc1 area=wan -13:22.510 [INFO] bob: Started DNS server: address=127.0.0.1:16920 network=tcp -13:22.511 [INFO] bob: Started HTTP server: address=127.0.0.1:16921 network=tcp -13:22.511 [INFO] bob: started state syncer -13:22.511 [INFO] bob: Started gRPC server: address=127.0.0.1:17000 network=tcp -13:22.564 [WARN] bob.server.raft: heartbeat timeout reached, starting election: last-leader= -13:22.565 [INFO] bob.server.raft: entering candidate state: node="Node at 127.0.0.1:16941 [Candidate]" term=2 -13:22.565 [DEBUG] bob.server.raft: votes: needed=1 -13:22.565 [DEBUG] bob.server.raft: vote granted: from=5bb346df-8dfc-19bc-e25f-ebfd6f0a3014 term=2 tally=1 -13:22.565 [INFO] bob.server.raft: election won: tally=1 -13:22.565 [INFO] bob.server.raft: entering leader state: leader="Node at 127.0.0.1:16941 [Leader]" -13:22.565 [INFO] bob.server: cluster leadership acquired -13:22.565 [DEBUG] bob.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16941 -13:22.565 [INFO] bob.server: New leader elected: payload=bob -13:22.566 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:22.568 [INFO] bob.server.connect: initialized primary datacenter CA with provider: provider=consul -13:22.568 [INFO] bob.leader: started routine: routine="federation state anti-entropy" -13:22.568 [INFO] bob.leader: started routine: routine="federation state pruning" -13:22.568 [INFO] bob.leader: started routine: routine="intermediate cert renew watch" -13:22.568 [INFO] bob.leader: started routine: routine="CA root pruning" -13:22.568 [DEBUG] bob.server: Skipping self join check for node since the cluster is too small: node=bob -13:22.568 [INFO] bob.server: member joined, marking health alive: member=bob -13:22.569 [INFO] bob.server: federation state anti-entropy synced -13:22.666 [DEBUG] bob: Skipping remote check since it is managed automatically: check=serfHealth -13:22.666 [INFO] bob: Synced node info -13:22.910 [DEBUG] bob: Node info in sync -13:22.911 [INFO] bob: Synced service: service=mesh-gateway -13:22.912 [INFO] bob.server: federation state anti-entropy synced -13:24.510 [INFO] bob.server.gateway_locator: new cached locations of mesh gateways: primary=[127.0.0.1:16970] local=[127.0.0.1:16970] -13:24.656 [DEBUG] bob.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:55598 -13:24.667 [INFO] bob.server.serf.wan: serf: EventMemberJoin: betty.dc2 127.0.0.1 -13:24.668 [INFO] bob.server: Handled event for server in area: event=member-join server=betty.dc2 area=wan -13:25.212 [DEBUG] bob.server.serf.wan: serf: messageJoinType: betty.dc2 -13:25.259 [DEBUG] bob.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:55618 -13:25.261 [INFO] bob.server.serf.wan: serf: EventMemberJoin: bonnie.dc3 127.0.0.1 -13:25.261 [INFO] bob.server: Handled event for server in area: event=member-join server=bonnie.dc3 area=wan -13:25.410 [DEBUG] bob: Skipping remote check since it is managed automatically: check=serfHealth -13:25.410 [DEBUG] bob: Node info in sync -13:25.410 [DEBUG] bob: Service in sync: service=mesh-gateway -13:25.410 [DEBUG] bob: Node info in sync -13:25.410 [DEBUG] bob: Service in sync: service=mesh-gateway -13:25.603 [DEBUG] bob.server.serf.wan: serf: messageJoinType: betty.dc2 -13:25.707 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 -13:26.102 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 -13:26.182 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 -13:26.601 [DEBUG] bob.server.serf.wan: serf: messageJoinType: bonnie.dc3 -13:31.462 [INFO] bob: Requesting shutdown -13:31.462 [INFO] bob.server: shutting down server -13:31.462 [DEBUG] bob.leader: stopping routine: routine="federation state anti-entropy" -13:31.462 [DEBUG] bob.leader: stopping routine: routine="federation state pruning" -13:31.462 [DEBUG] bob.leader: stopping routine: routine="intermediate cert renew watch" -13:31.462 [DEBUG] bob.leader: stopping routine: routine="CA root pruning" -13:31.462 [WARN] bob.server.serf.lan: serf: Shutdown without a Leave -13:31.464 [ERROR] bob.server: error performing anti-entropy sync of federation state: error="context canceled" -13:31.464 [DEBUG] bob.leader: stopped routine: routine="federation state anti-entropy" -13:31.464 [DEBUG] bob.leader: stopped routine: routine="federation state pruning" -13:31.464 [DEBUG] bob.leader: stopped routine: routine="intermediate cert renew watch" -13:31.464 [DEBUG] bob.leader: stopped routine: routine="CA root pruning" -13:31.464 [WARN] bob.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:31.464Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:31.464Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:31.464Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:31.464Z [INFO] agent.router.manager: shutting down -13:31.465 [INFO] bob: consul server down -13:31.465 [INFO] bob: shutdown complete -13:31.465 [INFO] bob: Stopping server: protocol=DNS address=127.0.0.1:16920 network=tcp -13:31.465 [INFO] bob: Stopping server: protocol=DNS address=127.0.0.1:16920 network=udp -13:31.465 [INFO] bob: Stopping server: protocol=HTTP address=127.0.0.1:16921 network=tcp -13:31.965 [INFO] bob: Waiting for endpoints to shut down -13:31.965 [INFO] bob: Endpoints down ---- PASS: TestAgent_JoinWAN_viaMeshGateway (9.62s) - --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc1_to_dc2 (0.04s) - --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc1_to_dc3 (0.04s) - --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc2_to_dc1 (0.00s) - --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc2_to_dc3 (0.06s) - --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc3_to_dc1 (0.00s) - --- PASS: TestAgent_JoinWAN_viaMeshGateway/dc3_to_dc2 (0.05s) -=== CONT TestAgent_Services_ExternalConnectProxy -2021-10-31T23:13:31.068Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:31.138Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:31.175 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1ee81776-cb77-aa95-63ef-889f9479474b Address:127.0.0.1:16706}]" -13:31.175 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1ee81776-cb77-aa95-63ef-889f9479474b.dc1 127.0.0.1 -13:31.176 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16706 [Follower]" leader= -13:31.176 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1ee81776-cb77-aa95-63ef-889f9479474b 127.0.0.1 -2021-10-31T23:13:31.176Z [INFO] agent.router: Initializing LAN area manager -13:31.176 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16743 network=udp -13:31.177 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16743 network=tcp -13:31.177 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1ee81776-cb77-aa95-63ef-889f9479474b.dc1 area=wan -13:31.177 [INFO] TestAgent.server: Adding LAN server: server="Node-1ee81776-cb77-aa95-63ef-889f9479474b (Addr: tcp/127.0.0.1:16706) (DC: dc1)" -13:31.177 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16702 network=tcp -13:31.182 [INFO] TestAgent: started state syncer -13:31.182 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16749 network=tcp -13:31.213 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:31.213 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16706 [Candidate]" term=2 -13:31.214 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:31.214 [DEBUG] TestAgent.server.raft: vote granted: from=1ee81776-cb77-aa95-63ef-889f9479474b term=2 tally=1 -13:31.214 [INFO] TestAgent.server.raft: election won: tally=1 -13:31.214 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16706 [Leader]" -13:31.214 [INFO] TestAgent.server: cluster leadership acquired -13:31.214 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16706 -13:31.215 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:31.216 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:31.216 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:31.216 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:31.216 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:31.216 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:31.217 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1ee81776-cb77-aa95-63ef-889f9479474b -13:31.217 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1ee81776-cb77-aa95-63ef-889f9479474b -13:31.217 [INFO] TestAgent.server: New leader elected: payload=Node-1ee81776-cb77-aa95-63ef-889f9479474b -13:31.226 [INFO] TestAgent.server: federation state anti-entropy synced -13:31.370 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:31.370 [INFO] TestAgent: Synced node info -13:31.503 [INFO] TestAgent: Requesting shutdown -13:31.503 [INFO] TestAgent.server: shutting down server -13:31.503 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:31.503 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:31.503 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:31.503 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:31.503 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:31.503 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:31.503 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:31.503 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:31.503 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:31.503 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:31.503 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:31.503Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:31.504Z [INFO] agent.router.manager: shutting down -13:31.504 [INFO] TestAgent: consul server down -13:31.504 [INFO] TestAgent: shutdown complete -13:31.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16743 network=tcp -13:31.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16743 network=udp -13:31.506 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16702 network=tcp -13:32.009 [INFO] TestAgent: Waiting for endpoints to shut down -13:32.009 [INFO] TestAgent: Endpoints down -=== RUN TestAgent_Self/no_grpc -2021-10-31T23:13:31.378Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:31.400Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:31.410 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e0d42c28-5fa4-735f-8281-307a150daf28 Address:127.0.0.1:16748}]" -13:31.410 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e0d42c28-5fa4-735f-8281-307a150daf28.dc1 127.0.0.1 -13:31.411 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e0d42c28-5fa4-735f-8281-307a150daf28 127.0.0.1 -2021-10-31T23:13:31.411Z [INFO] agent.router: Initializing LAN area manager -13:31.411 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16750 network=udp -13:31.411 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16748 [Follower]" leader= -13:31.412 [INFO] TestAgent.server: Adding LAN server: server="Node-e0d42c28-5fa4-735f-8281-307a150daf28 (Addr: tcp/127.0.0.1:16748) (DC: dc1)" -13:31.412 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e0d42c28-5fa4-735f-8281-307a150daf28.dc1 area=wan -13:31.412 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16750 network=tcp -13:31.412 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16744 network=tcp -13:31.412 [INFO] TestAgent: started state syncer -13:31.412 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16582 network=tcp -13:31.458 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:31.458 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16748 [Candidate]" term=2 -13:31.458 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:31.458 [DEBUG] TestAgent.server.raft: vote granted: from=e0d42c28-5fa4-735f-8281-307a150daf28 term=2 tally=1 -13:31.458 [INFO] TestAgent.server.raft: election won: tally=1 -13:31.458 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16748 [Leader]" -13:31.462 [INFO] TestAgent.server: cluster leadership acquired -13:31.462 [INFO] TestAgent.server: New leader elected: payload=Node-e0d42c28-5fa4-735f-8281-307a150daf28 -13:31.466 [INFO] TestAgent: Synced node info -13:31.466 [DEBUG] TestAgent: Node info in sync -13:31.466 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16748 -13:31.468 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:31.470 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:31.470 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:31.470 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:31.470 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:31.470 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:31.470 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e0d42c28-5fa4-735f-8281-307a150daf28 -13:31.470 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e0d42c28-5fa4-735f-8281-307a150daf28 -13:31.472 [INFO] TestAgent.server: federation state anti-entropy synced -13:31.519 [INFO] TestAgent: Requesting shutdown -13:31.519 [INFO] TestAgent.server: shutting down server -13:31.519 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:31.519 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:31.519 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:31.519 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:31.519 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:31.519 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:31.519 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:31.519 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:31.519 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:31.519 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:31.520 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:31.520Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:31.520Z [INFO] agent.router.manager: shutting down -13:31.520 [INFO] TestAgent: consul server down -13:31.520 [INFO] TestAgent: shutdown complete -13:31.520 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16750 network=tcp -13:31.520 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16750 network=udp -13:31.520 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16744 network=tcp -13:32.022 [INFO] TestAgent: Waiting for endpoints to shut down -13:32.022 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_HealthServiceByName (0.72s) +=== CONT TestAgent_ServicesFiltered +[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:39.232Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:39.250Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:39.252 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:501a743d-7456-fb26-1bc9-7c645eddd881 Address:127.0.0.1:23327}]" +49:39.253 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-501a743d-7456-fb26-1bc9-7c645eddd881.dc1 127.0.0.1 +49:39.253 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-501a743d-7456-fb26-1bc9-7c645eddd881 127.0.0.1 +2022-12-04T05:49:39.253Z [INFO] agent.router: Initializing LAN area manager +49:39.253 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23399 network=udp +49:39.253 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23327 [Follower]" leader= +49:39.254 [INFO] TestAgent.server: Adding LAN server: server="Node-501a743d-7456-fb26-1bc9-7c645eddd881 (Addr: tcp/127.0.0.1:23327) (DC: dc1)" +49:39.254 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-501a743d-7456-fb26-1bc9-7c645eddd881.dc1 area=wan +49:39.254 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23399 network=tcp +49:39.255 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23365 network=tcp +49:39.255 [INFO] TestAgent: started state syncer +49:39.255 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23499 network=tcp +49:39.294 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:39.294 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23327 [Candidate]" term=2 +49:39.294 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:39.294 [DEBUG] TestAgent.server.raft: vote granted: from=501a743d-7456-fb26-1bc9-7c645eddd881 term=2 tally=1 +49:39.294 [INFO] TestAgent.server.raft: election won: tally=1 +49:39.294 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23327 [Leader]" +49:39.294 [INFO] TestAgent.server: cluster leadership acquired +49:39.294 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23327 +49:39.294 [INFO] TestAgent.server: New leader elected: payload=Node-501a743d-7456-fb26-1bc9-7c645eddd881 +49:39.295 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:39.296 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:39.296 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:39.296 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:39.296 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:39.296 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:39.296 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-501a743d-7456-fb26-1bc9-7c645eddd881 +49:39.296 [INFO] TestAgent.server: member joined, marking health alive: member=Node-501a743d-7456-fb26-1bc9-7c645eddd881 +49:39.296 [INFO] TestAgent.server: federation state anti-entropy synced +49:39.315 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:39.316 [INFO] TestAgent: Synced node info +49:39.591 [INFO] TestAgent: Requesting shutdown +49:39.591 [INFO] TestAgent.server: shutting down server +49:39.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:39.592 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:39.592 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:39.592 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:39.592 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:39.592 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:39.592 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:39.592 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:39.592 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:39.592 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:39.592 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:39.592Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:39.592Z [INFO] agent.router.manager: shutting down +49:39.592 [INFO] TestAgent: consul server down +49:39.592 [INFO] TestAgent: shutdown complete +49:39.592 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23399 network=tcp +49:39.592 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23399 network=udp +49:39.592 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23365 network=tcp +49:40.093 [INFO] TestAgent: Waiting for endpoints to shut down +49:40.093 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_HealthServiceByName (0.87s) --- PASS: TestAgent_HealthServiceByName/passing_checks (0.00s) --- PASS: TestAgent_HealthServiceByName/passing_checks/format=text (0.00s) --- PASS: TestAgent_HealthServiceByName/passing_checks/format=json (0.00s) @@ -45328,65 +51254,120 @@ --- PASS: TestAgent_HealthServiceByName/maintenance_check_on_node (0.00s) --- PASS: TestAgent_HealthServiceByName/maintenance_check_on_node/format=text (0.00s) --- PASS: TestAgent_HealthServiceByName/maintenance_check_on_node/format=json (0.00s) -=== CONT TestAgent_ServicesFiltered -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=false -[WARN] freeport: 10 out of 48 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:31.414Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:31.473Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:31.485 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5b172725-ba73-3e0c-ae30-fc3985455716 Address:127.0.0.1:16762}]" -13:31.485 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16762 [Follower]" leader= -13:31.487 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5b172725-ba73-3e0c-ae30-fc3985455716.dc1 127.0.0.1 -13:31.488 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5b172725-ba73-3e0c-ae30-fc3985455716 127.0.0.1 -2021-10-31T23:13:31.488Z [INFO] agent.router: Initializing LAN area manager -13:31.488 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16583 network=udp -13:31.488 [INFO] TestAgent.server: Adding LAN server: server="Node-5b172725-ba73-3e0c-ae30-fc3985455716 (Addr: tcp/127.0.0.1:16762) (DC: dc1)" -13:31.488 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5b172725-ba73-3e0c-ae30-fc3985455716.dc1 area=wan -13:31.489 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16583 network=tcp -13:31.502 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16758 network=tcp -13:31.503 [INFO] TestAgent: started state syncer -13:31.503 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16756 network=tcp -13:31.529 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:31.530 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16762 [Candidate]" term=2 -13:31.530 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:31.530 [DEBUG] TestAgent.server.raft: vote granted: from=5b172725-ba73-3e0c-ae30-fc3985455716 term=2 tally=1 -13:31.530 [INFO] TestAgent.server.raft: election won: tally=1 -13:31.530 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16762 [Leader]" -13:31.530 [INFO] TestAgent.server: cluster leadership acquired -13:31.530 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16762 -13:31.531 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:31.532 [INFO] TestAgent.server: New leader elected: payload=Node-5b172725-ba73-3e0c-ae30-fc3985455716 -13:31.538 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:31.538 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:31.538 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:31.538 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:31.538 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:31.539 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5b172725-ba73-3e0c-ae30-fc3985455716 -13:31.539 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5b172725-ba73-3e0c-ae30-fc3985455716 -13:31.540 [INFO] TestAgent.server: federation state anti-entropy synced -13:31.688 [INFO] TestAgent: Requesting shutdown -13:31.688 [INFO] TestAgent.server: shutting down server -13:31.688 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:31.688 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:31.688 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:31.688 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:31.689 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:31.689 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:31.689 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:31.689 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:31.689 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:31.689 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:31.689 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:31.689 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:31.689Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:31.689Z [INFO] agent.router.manager: shutting down -13:31.689 [INFO] TestAgent: consul server down -13:31.689 [INFO] TestAgent: shutdown complete -13:31.689 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16583 network=tcp -13:31.689 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16583 network=udp -13:31.689 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16758 network=tcp -13:32.190 [INFO] TestAgent: Waiting for endpoints to shut down -13:32.190 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_HealthServiceByID (0.82s) +=== CONT TestAgent_Services +2022-12-04T05:49:39.388Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:39.399Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:39.400 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c387cfcf-e91b-a39f-cad4-47cf8320e1a6 Address:127.0.0.1:23505}]" +49:39.400 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23505 [Follower]" leader= +49:39.401 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c387cfcf-e91b-a39f-cad4-47cf8320e1a6.dc1 127.0.0.1 +49:39.401 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c387cfcf-e91b-a39f-cad4-47cf8320e1a6 127.0.0.1 +2022-12-04T05:49:39.401Z [INFO] agent.router: Initializing LAN area manager +49:39.401 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23451 network=udp +49:39.401 [INFO] TestAgent.server: Adding LAN server: server="Node-c387cfcf-e91b-a39f-cad4-47cf8320e1a6 (Addr: tcp/127.0.0.1:23505) (DC: dc1)" +49:39.401 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c387cfcf-e91b-a39f-cad4-47cf8320e1a6.dc1 area=wan +49:39.401 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23451 network=tcp +49:39.402 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23452 network=tcp +49:39.402 [INFO] TestAgent: started state syncer +49:39.402 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23471 network=tcp +49:39.449 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:39.449 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23505 [Candidate]" term=2 +49:39.449 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:39.449 [DEBUG] TestAgent.server.raft: vote granted: from=c387cfcf-e91b-a39f-cad4-47cf8320e1a6 term=2 tally=1 +49:39.449 [INFO] TestAgent.server.raft: election won: tally=1 +49:39.449 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23505 [Leader]" +49:39.449 [INFO] TestAgent.server: cluster leadership acquired +49:39.449 [INFO] TestAgent.server: New leader elected: payload=Node-c387cfcf-e91b-a39f-cad4-47cf8320e1a6 +49:39.449 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23505 +49:39.450 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:39.451 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:39.451 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:39.451 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:39.451 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:39.451 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:39.451 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c387cfcf-e91b-a39f-cad4-47cf8320e1a6 +49:39.451 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c387cfcf-e91b-a39f-cad4-47cf8320e1a6 +49:39.452 [INFO] TestAgent.server: federation state anti-entropy synced +49:39.605 [INFO] TestAgent: Requesting shutdown +49:39.605 [INFO] TestAgent.server: shutting down server +49:39.605 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:39.605 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:39.605 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:39.605 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:39.605 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:39.605 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:39.605 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:39.605 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:39.605 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:39.605 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:39.606 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:39.606 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +2022-12-04T05:49:39.606Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:39.606Z [INFO] agent.router.manager: shutting down +49:39.606 [INFO] TestAgent: consul server down +49:39.606 [INFO] TestAgent: shutdown complete +49:39.606 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23451 network=tcp +49:39.606 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23451 network=udp +49:39.606 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23452 network=tcp +49:40.108 [INFO] TestAgent: Waiting for endpoints to shut down +49:40.108 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Checks (0.73s) +=== CONT TestACL_ResolveIdentity +2022-12-04T05:49:39.285Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:39.294Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:39.295 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fb926106-6912-c138-f83b-615653f38e7d Address:127.0.0.1:23554}]" +49:39.296 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23554 [Follower]" leader= +49:39.296 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fb926106-6912-c138-f83b-615653f38e7d.dc1 127.0.0.1 +49:39.296 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fb926106-6912-c138-f83b-615653f38e7d 127.0.0.1 +2022-12-04T05:49:39.296Z [INFO] agent.router: Initializing LAN area manager +49:39.296 [INFO] TestAgent.server: Adding LAN server: server="Node-fb926106-6912-c138-f83b-615653f38e7d (Addr: tcp/127.0.0.1:23554) (DC: dc1)" +49:39.297 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fb926106-6912-c138-f83b-615653f38e7d.dc1 area=wan +49:39.297 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23500 network=udp +49:39.297 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23500 network=tcp +49:39.297 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23501 network=tcp +49:39.297 [INFO] TestAgent: started state syncer +49:39.297 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23506 network=tcp +49:39.334 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:39.334 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23554 [Candidate]" term=2 +49:39.334 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:39.334 [DEBUG] TestAgent.server.raft: vote granted: from=fb926106-6912-c138-f83b-615653f38e7d term=2 tally=1 +49:39.334 [INFO] TestAgent.server.raft: election won: tally=1 +49:39.334 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23554 [Leader]" +49:39.334 [INFO] TestAgent.server: cluster leadership acquired +49:39.334 [INFO] TestAgent.server: New leader elected: payload=Node-fb926106-6912-c138-f83b-615653f38e7d +49:39.335 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23554 +49:39.335 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:39.336 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:39.336 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:39.336 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:39.336 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:39.336 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:39.337 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fb926106-6912-c138-f83b-615653f38e7d +49:39.337 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fb926106-6912-c138-f83b-615653f38e7d +49:39.337 [INFO] TestAgent.server: federation state anti-entropy synced +49:39.614 [INFO] TestAgent: Requesting shutdown +49:39.615 [INFO] TestAgent.server: shutting down server +49:39.615 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:39.615 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:39.615 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:39.615 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:39.615 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:39.615 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:39.615 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:39.615 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:39.615 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:39.615 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:39.615 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:39.615 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:39.615Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:39.615Z [INFO] agent.router.manager: shutting down +49:39.615 [INFO] TestAgent: consul server down +49:39.615 [INFO] TestAgent: shutdown complete +49:39.615 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23500 network=tcp +49:39.615 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23500 network=udp +49:39.615 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23501 network=tcp +49:40.116 [INFO] TestAgent: Waiting for endpoints to shut down +49:40.116 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_HealthServiceByID (0.84s) --- PASS: TestAgent_HealthServiceByID/passing_checks (0.00s) --- PASS: TestAgent_HealthServiceByID/passing_checks/format=text (0.00s) --- PASS: TestAgent_HealthServiceByID/passing_checks/format=json (0.00s) @@ -45405,502 +51386,271 @@ --- PASS: TestAgent_HealthServiceByID/maintenance_check_on_node (0.00s) --- PASS: TestAgent_HealthServiceByID/maintenance_check_on_node/format=text (0.00s) --- PASS: TestAgent_HealthServiceByID/maintenance_check_on_node/format=json (0.00s) -=== CONT TestAgent_Services -2021-10-31T23:13:31.359Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:31.408Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:31.422 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7fb42ee3-3175-990d-60d2-97d9c820cb0b Address:127.0.0.1:16755}]" -13:31.423 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7fb42ee3-3175-990d-60d2-97d9c820cb0b.dc1 127.0.0.1 -13:31.424 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7fb42ee3-3175-990d-60d2-97d9c820cb0b 127.0.0.1 -2021-10-31T23:13:31.424Z [INFO] agent.router: Initializing LAN area manager -13:31.424 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16708 network=udp -13:31.424 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16755 [Follower]" leader= -13:31.424 [INFO] TestAgent.server: Adding LAN server: server="Node-7fb42ee3-3175-990d-60d2-97d9c820cb0b (Addr: tcp/127.0.0.1:16755) (DC: dc1)" -13:31.424 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7fb42ee3-3175-990d-60d2-97d9c820cb0b.dc1 area=wan -13:31.424 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16708 network=tcp -13:31.425 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16751 network=tcp -13:31.425 [INFO] TestAgent: started state syncer -13:31.425 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16763 network=tcp -13:31.477 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:31.478 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16755 [Candidate]" term=2 -13:31.478 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:31.478 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:31.478 [DEBUG] TestAgent.server.raft: vote granted: from=7fb42ee3-3175-990d-60d2-97d9c820cb0b term=2 tally=1 -13:31.478 [INFO] TestAgent.server.raft: election won: tally=1 -13:31.478 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16755 [Leader]" -13:31.478 [INFO] TestAgent.server: cluster leadership acquired -13:31.478 [INFO] TestAgent.server: initializing acls -13:31.479 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:31.479 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:31.480 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:31.480 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:31.480 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:31.480 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7fb42ee3-3175-990d-60d2-97d9c820cb0b -13:31.480 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7fb42ee3-3175-990d-60d2-97d9c820cb0b.dc1 -13:31.482 [INFO] TestAgent.server: New leader elected: payload=Node-7fb42ee3-3175-990d-60d2-97d9c820cb0b -13:31.482 [INFO] TestAgent.server: Updating LAN server: server="Node-7fb42ee3-3175-990d-60d2-97d9c820cb0b (Addr: tcp/127.0.0.1:16755) (DC: dc1)" -13:31.482 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7fb42ee3-3175-990d-60d2-97d9c820cb0b.dc1 area=wan -13:31.486 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:31.488 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:31.488 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:31.488 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:31.488 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:31.488 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:31.488 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7fb42ee3-3175-990d-60d2-97d9c820cb0b -13:31.488 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7fb42ee3-3175-990d-60d2-97d9c820cb0b -13:31.489 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7fb42ee3-3175-990d-60d2-97d9c820cb0b -13:31.506 [INFO] TestAgent.server: federation state anti-entropy synced -13:31.659 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:31.680 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7fb42ee3-3175-990d-60d2-97d9c820cb0b -13:31.687 [INFO] TestAgent: Synced node info -13:31.687 [DEBUG] TestAgent: Node info in sync -13:31.688 [INFO] TestAgent: Requesting shutdown -13:31.688 [INFO] TestAgent.server: shutting down server -13:31.688 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:31.688 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:31.688 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:31.688 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:31.688 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:31.688 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:31.688 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:31.688 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:31.688 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:31.688 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:31.688 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:31.688 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:31.688 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:31.688 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:31.688 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:31.688Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:31.688Z [INFO] agent.router.manager: shutting down -13:31.688 [INFO] TestAgent: consul server down -13:31.688 [INFO] TestAgent: shutdown complete -13:31.688 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16708 network=tcp -13:31.689 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16708 network=udp -13:31.689 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16751 network=tcp -13:32.193 [INFO] TestAgent: Waiting for endpoints to shut down -13:32.193 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_HealthServicesACLEnforcement (0.94s) +=== CONT TestACL_filterServices +2022-12-04T05:49:39.212Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:39.224Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:39.226 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:517dd28d-8c54-9f2e-2878-ca21fdb47c9e Address:127.0.0.1:23369}]" +49:39.227 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-517dd28d-8c54-9f2e-2878-ca21fdb47c9e.dc1 127.0.0.1 +49:39.227 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-517dd28d-8c54-9f2e-2878-ca21fdb47c9e 127.0.0.1 +2022-12-04T05:49:39.227Z [INFO] agent.router: Initializing LAN area manager +49:39.227 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23448 network=udp +49:39.227 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23369 [Follower]" leader= +49:39.228 [INFO] TestAgent.server: Adding LAN server: server="Node-517dd28d-8c54-9f2e-2878-ca21fdb47c9e (Addr: tcp/127.0.0.1:23369) (DC: dc1)" +49:39.228 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-517dd28d-8c54-9f2e-2878-ca21fdb47c9e.dc1 area=wan +49:39.228 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23448 network=tcp +49:39.228 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23309 network=tcp +49:39.228 [INFO] TestAgent: started state syncer +49:39.229 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23398 network=tcp +49:39.280 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:39.280 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:39.280 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23369 [Candidate]" term=2 +49:39.280 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:39.280 [DEBUG] TestAgent.server.raft: vote granted: from=517dd28d-8c54-9f2e-2878-ca21fdb47c9e term=2 tally=1 +49:39.280 [INFO] TestAgent.server.raft: election won: tally=1 +49:39.280 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23369 [Leader]" +49:39.280 [INFO] TestAgent.server: cluster leadership acquired +49:39.280 [INFO] TestAgent.server: initializing acls +49:39.280 [INFO] TestAgent.server: New leader elected: payload=Node-517dd28d-8c54-9f2e-2878-ca21fdb47c9e +49:39.281 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:39.281 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:39.281 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:39.281 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:39.281 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:39.281 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-517dd28d-8c54-9f2e-2878-ca21fdb47c9e +49:39.281 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-517dd28d-8c54-9f2e-2878-ca21fdb47c9e.dc1 +49:39.282 [INFO] TestAgent.server: Updating LAN server: server="Node-517dd28d-8c54-9f2e-2878-ca21fdb47c9e (Addr: tcp/127.0.0.1:23369) (DC: dc1)" +49:39.282 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-517dd28d-8c54-9f2e-2878-ca21fdb47c9e.dc1 area=wan +49:39.282 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:39.283 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:39.283 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:39.283 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:39.283 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:39.283 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:39.283 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-517dd28d-8c54-9f2e-2878-ca21fdb47c9e +49:39.283 [INFO] TestAgent.server: member joined, marking health alive: member=Node-517dd28d-8c54-9f2e-2878-ca21fdb47c9e +49:39.284 [INFO] TestAgent.server: federation state anti-entropy synced +49:39.284 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-517dd28d-8c54-9f2e-2878-ca21fdb47c9e +49:39.439 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:39.440 [INFO] TestAgent: Synced node info +49:39.440 [DEBUG] TestAgent: Node info in sync +49:39.620 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-517dd28d-8c54-9f2e-2878-ca21fdb47c9e +49:39.620 [INFO] TestAgent: Requesting shutdown +49:39.620 [INFO] TestAgent.server: shutting down server +49:39.620 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:39.621 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:39.621 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:39.621 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:39.621 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:39.621 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:39.621 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:39.621 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:39.621 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:39.621 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:39.621 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:39.621 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:39.621 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:39.621 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:39.621 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:39.621Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:39.621Z [INFO] agent.router.manager: shutting down +49:39.621 [INFO] TestAgent: consul server down +49:39.621 [INFO] TestAgent: shutdown complete +49:39.621 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23448 network=tcp +49:39.621 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23448 network=udp +49:39.621 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23309 network=tcp +49:40.122 [INFO] TestAgent: Waiting for endpoints to shut down +49:40.122 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_HealthServicesACLEnforcement (0.93s) --- PASS: TestAgent_HealthServicesACLEnforcement/no-token-health-by-id (0.00s) --- PASS: TestAgent_HealthServicesACLEnforcement/no-token-health-by-name (0.00s) --- PASS: TestAgent_HealthServicesACLEnforcement/root-token-health-by-id (0.00s) --- PASS: TestAgent_HealthServicesACLEnforcement/root-token-health-by-name (0.00s) -=== CONT TestACL_ResolveIdentity -2021-10-31T23:13:32.232Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:32.232Z [WARN] agent: bootstrap = true: do not enable unless necessary ---- PASS: TestACL_ResolveIdentity (0.03s) -=== CONT TestACLEndpoint_LoginLogout_jwt -2021-10-31T23:13:31.436Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:31.519Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:31.523 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:099cd23f-7e42-eb63-3d67-a5ee85a56a8f Address:127.0.0.1:16741}]" -13:31.524 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-099cd23f-7e42-eb63-3d67-a5ee85a56a8f.dc1 127.0.0.1 -13:31.524 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-099cd23f-7e42-eb63-3d67-a5ee85a56a8f 127.0.0.1 -2021-10-31T23:13:31.524Z [INFO] agent.router: Initializing LAN area manager -13:31.525 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16741 [Follower]" leader= -13:31.525 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16778 network=tcp -13:31.525 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-099cd23f-7e42-eb63-3d67-a5ee85a56a8f.dc1 area=wan -13:31.525 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16778 network=udp -13:31.525 [INFO] TestAgent.server: Adding LAN server: server="Node-099cd23f-7e42-eb63-3d67-a5ee85a56a8f (Addr: tcp/127.0.0.1:16741) (DC: dc1)" -13:31.531 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16737 network=tcp -13:31.531 [INFO] TestAgent: started state syncer -13:31.531 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16721 network=tcp -13:31.577 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:31.577 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16741 [Candidate]" term=2 -13:31.578 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:31.578 [DEBUG] TestAgent.server.raft: vote granted: from=099cd23f-7e42-eb63-3d67-a5ee85a56a8f term=2 tally=1 -13:31.578 [INFO] TestAgent.server.raft: election won: tally=1 -13:31.578 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16741 [Leader]" -13:31.578 [INFO] TestAgent.server: cluster leadership acquired -13:31.578 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16741 -13:31.579 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:31.579 [INFO] TestAgent.server: New leader elected: payload=Node-099cd23f-7e42-eb63-3d67-a5ee85a56a8f -13:31.581 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:31.581 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:31.581 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:31.581 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:31.581 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:31.581 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-099cd23f-7e42-eb63-3d67-a5ee85a56a8f -13:31.581 [INFO] TestAgent.server: member joined, marking health alive: member=Node-099cd23f-7e42-eb63-3d67-a5ee85a56a8f -13:31.582 [INFO] TestAgent.server: federation state anti-entropy synced -13:31.598 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:31.598 [INFO] TestAgent: Synced node info -13:31.598 [DEBUG] TestAgent: Node info in sync -13:31.810 [INFO] TestAgent: Requesting shutdown -13:31.810 [INFO] TestAgent.server: shutting down server -13:31.810 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:31.810 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:31.810 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:31.810 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:31.810 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:31.810 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:31.810 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:31.810 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:31.810 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:31.810 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:31.811 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:31.811Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:31.811Z [INFO] agent.router.manager: shutting down -13:31.811 [INFO] TestAgent: consul server down -13:31.811 [INFO] TestAgent: shutdown complete -13:31.811 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16778 network=tcp -13:31.811 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16778 network=udp -13:31.811 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16737 network=tcp -13:32.314 [INFO] TestAgent: Waiting for endpoints to shut down -13:32.314 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ChecksWithFilter (0.91s) -=== CONT TestACL_filterServices -2021-10-31T23:13:32.336Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:32.336Z [WARN] agent: bootstrap = true: do not enable unless necessary -13:32.336 [DEBUG] TestACL_filterServices: dropping service from result due to ACLs: service=my-other ---- PASS: TestACL_filterServices (0.02s) === CONT TestACL_filterMembers -2021-10-31T23:13:31.600Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:31.638Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:31.640 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:08e9a396-70b7-27d3-7640-edd9c37bc9bb Address:127.0.0.1:16727}]" -13:31.640 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-08e9a396-70b7-27d3-7640-edd9c37bc9bb.dc1 127.0.0.1 -13:31.641 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-08e9a396-70b7-27d3-7640-edd9c37bc9bb 127.0.0.1 -2021-10-31T23:13:31.641Z [INFO] agent.router: Initializing LAN area manager -13:31.641 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16722 network=udp -13:31.641 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16727 [Follower]" leader= -13:31.641 [INFO] TestAgent.server: Adding LAN server: server="Node-08e9a396-70b7-27d3-7640-edd9c37bc9bb (Addr: tcp/127.0.0.1:16727) (DC: dc1)" -13:31.641 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-08e9a396-70b7-27d3-7640-edd9c37bc9bb.dc1 area=wan -13:31.642 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16722 network=tcp -13:31.642 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16723 network=tcp -13:31.643 [INFO] TestAgent: started state syncer -13:31.643 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16707 network=tcp -13:31.706 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:31.706 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16727 [Candidate]" term=2 -13:31.706 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:31.706 [DEBUG] TestAgent.server.raft: vote granted: from=08e9a396-70b7-27d3-7640-edd9c37bc9bb term=2 tally=1 -13:31.706 [INFO] TestAgent.server.raft: election won: tally=1 -13:31.706 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16727 [Leader]" -13:31.706 [INFO] TestAgent.server: cluster leadership acquired -13:31.706 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16727 -13:31.708 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:31.709 [INFO] TestAgent.server: New leader elected: payload=Node-08e9a396-70b7-27d3-7640-edd9c37bc9bb -13:31.719 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:31.719 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:31.719 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:31.719 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:31.719 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:31.719 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-08e9a396-70b7-27d3-7640-edd9c37bc9bb -13:31.719 [INFO] TestAgent.server: member joined, marking health alive: member=Node-08e9a396-70b7-27d3-7640-edd9c37bc9bb -13:31.720 [INFO] TestAgent.server: federation state anti-entropy synced -13:31.845 [INFO] TestAgent: Requesting shutdown -13:31.845 [INFO] TestAgent.server: shutting down server -13:31.845 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:31.845 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:31.845 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:31.845 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:31.845 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:31.845 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:31.845 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:31.845 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:31.845 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:31.845 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:31.846 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:31.846 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:31.846Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:31.846Z [INFO] agent.router.manager: shutting down -13:31.846 [INFO] TestAgent: consul server down -13:31.846 [INFO] TestAgent: shutdown complete -13:31.846 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16722 network=tcp -13:31.846 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16722 network=udp -13:31.846 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16723 network=tcp -13:32.349 [INFO] TestAgent: Waiting for endpoints to shut down -13:32.349 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Checks (0.78s) +2022-12-04T05:49:40.131Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.131Z [WARN] agent: bootstrap = true: do not enable unless necessary +--- PASS: TestACL_ResolveIdentity (0.02s) === CONT TestACL_vetCheckUpdate -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=33)_compressed=true -2021-10-31T23:13:32.379Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:32.379Z [WARN] agent: bootstrap = true: do not enable unless necessary -13:32.379 [DEBUG] TestACL_filterMembers: dropping node from result due to ACLs: node=Nope accessorID=9df2d1a4-2d07-414e-8ead-6053f56ed2eb ---- PASS: TestACL_filterMembers (0.04s) +2022-12-04T05:49:40.137Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.137Z [WARN] agent: bootstrap = true: do not enable unless necessary +49:40.137 [DEBUG] TestACL_filterServices: dropping service from result due to ACLs: service=my-other +--- PASS: TestACL_filterServices (0.02s) === CONT TestACL_vetCheckRegister -2021-10-31T23:13:32.390Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:32.390Z [WARN] agent: bootstrap = true: do not enable unless necessary ---- PASS: TestACL_vetCheckUpdate (0.04s) -=== CONT TestACL_vetServiceUpdate -2021-10-31T23:13:31.718Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:31.803Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:31.805 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aa670502-fb94-c8ed-4108-4c3d158673a0 Address:127.0.0.1:16692}]" -13:31.805 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aa670502-fb94-c8ed-4108-4c3d158673a0.dc1 127.0.0.1 -13:31.805 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16692 [Follower]" leader= -13:31.810 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aa670502-fb94-c8ed-4108-4c3d158673a0 127.0.0.1 -2021-10-31T23:13:31.810Z [INFO] agent.router: Initializing LAN area manager -13:31.811 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16687 network=udp -13:31.811 [INFO] TestAgent.server: Adding LAN server: server="Node-aa670502-fb94-c8ed-4108-4c3d158673a0 (Addr: tcp/127.0.0.1:16692) (DC: dc1)" -13:31.811 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aa670502-fb94-c8ed-4108-4c3d158673a0.dc1 area=wan -13:31.811 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16687 network=tcp -13:31.812 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16688 network=tcp -13:31.812 [INFO] TestAgent: started state syncer -13:31.812 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16805 network=tcp -13:31.856 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:31.856 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16692 [Candidate]" term=2 -13:31.857 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:31.857 [DEBUG] TestAgent.server.raft: vote granted: from=aa670502-fb94-c8ed-4108-4c3d158673a0 term=2 tally=1 -13:31.857 [INFO] TestAgent.server.raft: election won: tally=1 -13:31.857 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16692 [Leader]" -13:31.857 [INFO] TestAgent.server: cluster leadership acquired -13:31.857 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16692 -13:31.858 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:31.859 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:31.859 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:31.859 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:31.859 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:31.859 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:31.859 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aa670502-fb94-c8ed-4108-4c3d158673a0 -13:31.859 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aa670502-fb94-c8ed-4108-4c3d158673a0 -13:31.860 [INFO] TestAgent.server: New leader elected: payload=Node-aa670502-fb94-c8ed-4108-4c3d158673a0 -13:31.860 [INFO] TestAgent.server: federation state anti-entropy synced -13:31.892 [INFO] TestAgent: Requesting shutdown -13:31.892 [INFO] TestAgent.server: shutting down server -13:31.892 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:31.892 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:31.892 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:31.892 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:31.892 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:31.892 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:31.892 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:31.892 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:31.892 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:31.892 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:31.892 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:31.892 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:31.892Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:31.892Z [INFO] agent.router.manager: shutting down -13:31.892 [INFO] TestAgent: consul server down -13:31.892 [INFO] TestAgent: shutdown complete -13:31.892 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16687 network=tcp -13:31.892 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16687 network=udp -13:31.892 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16688 network=tcp -13:32.393 [INFO] TestAgent: Waiting for endpoints to shut down -13:32.393 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Services_TerminatingGateway (0.76s) +2022-12-04T05:49:40.142Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.142Z [WARN] agent: bootstrap = true: do not enable unless necessary +--- PASS: TestACL_vetCheckUpdate (0.01s) +=== CONT TestRemoteExecWrites +2022-12-04T05:49:40.144Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.144Z [WARN] agent: bootstrap = true: do not enable unless necessary +49:40.144 [DEBUG] TestACL_filterMembers: dropping node from result due to ACLs: node=Nope accessorID=9df2d1a4-2d07-414e-8ead-6053f56ed2eb +--- PASS: TestACL_filterMembers (0.02s) +=== CONT TestACL_Bootstrap +2022-12-04T05:49:40.148Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.148Z [WARN] agent: bootstrap = true: do not enable unless necessary +--- PASS: TestACL_vetCheckRegister (0.01s) === CONT TestACL_vetServiceRegister -2021-10-31T23:13:31.627Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:31.627Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:31.710Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:31.710Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:31.718 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4e7486b9-08bd-47f4-f7af-efd2d2894788 Address:127.0.0.1:16581}]" -13:31.718 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16581 [Follower]" leader= -13:31.719 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4e7486b9-08bd-47f4-f7af-efd2d2894788.dc1 127.0.0.1 -13:31.721 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4e7486b9-08bd-47f4-f7af-efd2d2894788 127.0.0.1 -2021-10-31T23:13:31.721Z [INFO] agent.router: Initializing LAN area manager -13:31.721 [INFO] TestAgent.server: Adding LAN server: server="Node-4e7486b9-08bd-47f4-f7af-efd2d2894788 (Addr: tcp/127.0.0.1:16581) (DC: dc1)" -13:31.721 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16729 network=udp -13:31.721 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4e7486b9-08bd-47f4-f7af-efd2d2894788.dc1 area=wan -13:31.721 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16729 network=tcp -13:31.736 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16577 network=tcp -13:31.736 [INFO] TestAgent: started state syncer -13:31.736 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16679 network=tcp -13:31.773 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:31.794 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:31.794 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16581 [Candidate]" term=2 -13:31.794 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:31.794 [DEBUG] TestAgent.server.raft: vote granted: from=4e7486b9-08bd-47f4-f7af-efd2d2894788 term=2 tally=1 -13:31.794 [INFO] TestAgent.server.raft: election won: tally=1 -13:31.794 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16581 [Leader]" -13:31.794 [INFO] TestAgent.server: cluster leadership acquired -13:31.794 [INFO] TestAgent.server: initializing acls -13:31.795 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:31.795 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:31.795 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:31.796 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:31.796 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:31.796 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:31.796 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4e7486b9-08bd-47f4-f7af-efd2d2894788 -13:31.796 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4e7486b9-08bd-47f4-f7af-efd2d2894788.dc1 -13:31.797 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:31.797 [INFO] TestAgent.server: New leader elected: payload=Node-4e7486b9-08bd-47f4-f7af-efd2d2894788 -13:31.798 [INFO] TestAgent.server: Updating LAN server: server="Node-4e7486b9-08bd-47f4-f7af-efd2d2894788 (Addr: tcp/127.0.0.1:16581) (DC: dc1)" -13:31.798 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4e7486b9-08bd-47f4-f7af-efd2d2894788.dc1 area=wan -13:31.799 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:31.799 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:31.799 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:31.799 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:31.799 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:31.799 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4e7486b9-08bd-47f4-f7af-efd2d2894788 -13:31.799 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4e7486b9-08bd-47f4-f7af-efd2d2894788 -13:31.799 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4e7486b9-08bd-47f4-f7af-efd2d2894788 -13:31.800 [INFO] TestAgent.server: federation state anti-entropy synced -13:31.832 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:31.833 [INFO] TestAgent: Synced node info -13:31.833 [DEBUG] TestAgent: Node info in sync -13:31.914 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4e7486b9-08bd-47f4-f7af-efd2d2894788 -13:31.914 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4e7486b9-08bd-47f4-f7af-efd2d2894788 -13:31.914 [DEBUG] TestAgent: dropping service from result due to ACLs: service=mysql -13:31.914 [INFO] TestAgent: Requesting shutdown -13:31.914 [INFO] TestAgent.server: shutting down server -13:31.914 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:31.914 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:31.914 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:31.914 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:31.914 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:31.914 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:31.914 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:31.914 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:31.914 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:31.914 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:31.914 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:31.914 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:31.914 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:31.914 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:31.914 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:31.915Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:31.915Z [INFO] agent.router.manager: shutting down -13:31.915 [INFO] TestAgent: consul server down -13:31.915 [INFO] TestAgent: shutdown complete -13:31.915 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16729 network=tcp -13:31.915 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16729 network=udp -13:31.915 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16577 network=tcp -13:32.417 [INFO] TestAgent: Waiting for endpoints to shut down -13:32.417 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Services_ACLFilter (0.84s) - --- PASS: TestAgent_Services_ACLFilter/no_token (0.00s) - --- PASS: TestAgent_Services_ACLFilter/root_token (0.00s) +2022-12-04T05:49:40.159Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.159Z [WARN] agent: bootstrap = true: do not enable unless necessary +--- PASS: TestACL_vetServiceRegister (0.01s) === CONT TestACL_RootAuthorizersDenied -2021-10-31T23:13:32.428Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:32.428Z [WARN] agent: bootstrap = true: do not enable unless necessary ---- PASS: TestACL_vetCheckRegister (0.05s) +2022-12-04T05:49:40.171Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.171Z [WARN] agent: bootstrap = true: do not enable unless necessary +--- PASS: TestACL_RootAuthorizersDenied (0.01s) === CONT TestACL_AgentMasterToken -2021-10-31T23:13:32.430Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:32.430Z [WARN] agent: bootstrap = true: do not enable unless necessary ---- PASS: TestACL_vetServiceRegister (0.04s) +2022-12-04T05:49:40.179Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.179Z [WARN] agent: bootstrap = true: do not enable unless necessary +--- PASS: TestACL_AgentMasterToken (0.01s) === CONT TestACL_Version8EnabledByDefault -2021-10-31T23:13:32.432Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:32.432Z [WARN] agent: bootstrap = true: do not enable unless necessary ---- PASS: TestACL_vetServiceUpdate (0.04s) +2022-12-04T05:49:40.187Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.188Z [WARN] agent: bootstrap = true: do not enable unless necessary +--- PASS: TestACL_Version8EnabledByDefault (0.01s) === CONT TestACL_Authorize -2021-10-31T23:13:32.443Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:32.443Z [WARN] agent: bootstrap = true: do not enable unless necessary ---- PASS: TestACL_RootAuthorizersDenied (0.03s) -=== CONT TestKVSEndpoint_DELETE_ConflictingFlags -2021-10-31T23:13:32.467Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:32.467Z [WARN] agent: bootstrap = true: do not enable unless necessary ---- PASS: TestACL_AgentMasterToken (0.04s) -=== CONT TestOperator_KeyringListLocalOnly -2021-10-31T23:13:32.471Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:32.471Z [WARN] agent: bootstrap = true: do not enable unless necessary ---- PASS: TestACL_Version8EnabledByDefault (0.04s) -=== CONT TestOperator_KeyringList -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:31.760Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:31.829Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:31.832 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a1f15b88-9925-0c68-d334-18ad93272046 Address:127.0.0.1:16776}]" -13:31.833 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a1f15b88-9925-0c68-d334-18ad93272046.dc1 127.0.0.1 -13:31.833 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16776 [Follower]" leader= -13:31.833 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a1f15b88-9925-0c68-d334-18ad93272046 127.0.0.1 -2021-10-31T23:13:31.833Z [INFO] agent.router: Initializing LAN area manager -13:31.833 [INFO] TestAgent.server: Adding LAN server: server="Node-a1f15b88-9925-0c68-d334-18ad93272046 (Addr: tcp/127.0.0.1:16776) (DC: dc1)" -13:31.834 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a1f15b88-9925-0c68-d334-18ad93272046.dc1 area=wan -13:31.834 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16806 network=tcp -13:31.846 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16806 network=udp -13:31.846 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16772 network=tcp -13:31.846 [INFO] TestAgent: started state syncer -13:31.846 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16770 network=tcp -13:31.883 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:31.883 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16776 [Candidate]" term=2 -13:31.883 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:31.883 [DEBUG] TestAgent.server.raft: vote granted: from=a1f15b88-9925-0c68-d334-18ad93272046 term=2 tally=1 -13:31.883 [INFO] TestAgent.server.raft: election won: tally=1 -13:31.883 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16776 [Leader]" -13:31.883 [INFO] TestAgent.server: cluster leadership acquired -13:31.883 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16776 -13:31.884 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:31.885 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:31.885 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:31.885 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:31.885 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:31.885 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:31.885 [INFO] TestAgent.server: New leader elected: payload=Node-a1f15b88-9925-0c68-d334-18ad93272046 -13:31.885 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1f15b88-9925-0c68-d334-18ad93272046 -13:31.885 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a1f15b88-9925-0c68-d334-18ad93272046 -13:31.886 [INFO] TestAgent.server: federation state anti-entropy synced -13:31.970 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:31.970 [INFO] TestAgent: Synced node info -13:32.211 [INFO] TestAgent: Requesting shutdown -13:32.211 [INFO] TestAgent.server: shutting down server -13:32.212 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:32.212 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:32.212 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:32.212 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:32.212 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:32.212 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:32.212 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:32.212 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:32.212 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:32.212 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:32.212 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:32.212Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:32.212Z [INFO] agent.router.manager: shutting down -13:32.212 [INFO] TestAgent: consul server down -13:32.212 [INFO] TestAgent: shutdown complete -13:32.212 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16806 network=tcp -13:32.212 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16806 network=udp -13:32.212 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16772 network=tcp -13:32.713 [INFO] TestAgent: Waiting for endpoints to shut down -13:32.713 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:31.844Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:31.878Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:31.880 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7213eac7-8bd4-d512-772a-e15883f03c7e Address:127.0.0.1:16783}]" -13:31.881 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7213eac7-8bd4-d512-772a-e15883f03c7e.dc1 127.0.0.1 -13:31.881 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16783 [Follower]" leader= -13:31.890 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7213eac7-8bd4-d512-772a-e15883f03c7e 127.0.0.1 -2021-10-31T23:13:31.890Z [INFO] agent.router: Initializing LAN area manager -13:31.890 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16785 network=udp -13:31.890 [INFO] TestAgent.server: Adding LAN server: server="Node-7213eac7-8bd4-d512-772a-e15883f03c7e (Addr: tcp/127.0.0.1:16783) (DC: dc1)" -13:31.890 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7213eac7-8bd4-d512-772a-e15883f03c7e.dc1 area=wan -13:31.891 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16785 network=tcp -13:31.891 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16779 network=tcp -13:31.891 [INFO] TestAgent: started state syncer -13:31.891 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16742 network=tcp -13:31.949 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:31.949 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16783 [Candidate]" term=2 -13:31.950 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:31.950 [DEBUG] TestAgent.server.raft: vote granted: from=7213eac7-8bd4-d512-772a-e15883f03c7e term=2 tally=1 -13:31.950 [INFO] TestAgent.server.raft: election won: tally=1 -13:31.950 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16783 [Leader]" -13:31.950 [INFO] TestAgent.server: cluster leadership acquired -13:31.950 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16783 -13:31.951 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:31.952 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:31.952 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:31.952 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:31.952 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:31.953 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:31.953 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7213eac7-8bd4-d512-772a-e15883f03c7e -13:31.953 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7213eac7-8bd4-d512-772a-e15883f03c7e -13:31.953 [INFO] TestAgent.server: New leader elected: payload=Node-7213eac7-8bd4-d512-772a-e15883f03c7e -13:31.962 [INFO] TestAgent.server: federation state anti-entropy synced -13:32.210 [INFO] TestAgent: Requesting shutdown -13:32.210 [INFO] TestAgent.server: shutting down server -13:32.210 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:32.210 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:32.210 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:32.210 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:32.210 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:32.210 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:32.210 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:32.210 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:32.210 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:32.210 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:32.210 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:32.211 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:32.211Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:32.211Z [INFO] agent.router.manager: shutting down -13:32.211 [INFO] TestAgent: consul server down -13:32.211 [INFO] TestAgent: shutdown complete -13:32.211 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16785 network=tcp -13:32.211 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16785 network=udp -13:32.211 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16779 network=tcp -13:32.714 [INFO] TestAgent: Waiting for endpoints to shut down -13:32.714 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Services_MeshGateway (1.00s) -=== CONT TestOperator_KeyringInstall ---- PASS: TestAgent_Services_Sidecar (0.91s) -=== CONT TestOperator_RaftPeer -=== RUN TestOperator_RaftPeer/#00 +2022-12-04T05:49:39.547Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:39.557Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:39.558 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0f907d88-46ba-a0fb-3ddd-bea331771ba3 Address:127.0.0.1:23540}]" +49:39.558 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0f907d88-46ba-a0fb-3ddd-bea331771ba3.dc1 127.0.0.1 +49:39.559 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0f907d88-46ba-a0fb-3ddd-bea331771ba3 127.0.0.1 +2022-12-04T05:49:39.559Z [INFO] agent.router: Initializing LAN area manager +49:39.559 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23556 network=udp +49:39.559 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23540 [Follower]" leader= +49:39.559 [INFO] TestAgent.server: Adding LAN server: server="Node-0f907d88-46ba-a0fb-3ddd-bea331771ba3 (Addr: tcp/127.0.0.1:23540) (DC: dc1)" +49:39.559 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0f907d88-46ba-a0fb-3ddd-bea331771ba3.dc1 area=wan +49:39.559 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23556 network=tcp +49:39.560 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23557 network=tcp +49:39.560 [INFO] TestAgent: started state syncer +49:39.560 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23548 network=tcp +49:39.607 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:39.607 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23540 [Candidate]" term=2 +49:39.607 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:39.607 [DEBUG] TestAgent.server.raft: vote granted: from=0f907d88-46ba-a0fb-3ddd-bea331771ba3 term=2 tally=1 +49:39.607 [INFO] TestAgent.server.raft: election won: tally=1 +49:39.607 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23540 [Leader]" +49:39.607 [INFO] TestAgent.server: cluster leadership acquired +49:39.607 [INFO] TestAgent.server: New leader elected: payload=Node-0f907d88-46ba-a0fb-3ddd-bea331771ba3 +49:39.607 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23540 +49:39.608 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:39.609 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:39.609 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:39.609 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:39.609 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:39.609 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:39.609 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0f907d88-46ba-a0fb-3ddd-bea331771ba3 +49:39.609 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0f907d88-46ba-a0fb-3ddd-bea331771ba3 +49:39.609 [INFO] TestAgent.server: federation state anti-entropy synced +49:39.754 [INFO] TestAgent: Requesting shutdown +49:39.754 [INFO] TestAgent.server: shutting down server +49:39.754 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:39.754 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:39.754 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:39.754 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:39.754 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:39.754 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:39.754 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:39.754 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:39.754 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:39.754 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:39.754 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:39.755 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:39.755Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:39.755Z [INFO] agent.router.manager: shutting down +49:39.755 [INFO] TestAgent: consul server down +49:39.755 [INFO] TestAgent: shutdown complete +49:39.755 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23556 network=tcp +49:39.755 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23556 network=udp +49:39.755 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23557 network=tcp +49:40.255 [INFO] TestAgent: Waiting for endpoints to shut down +49:40.255 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Services_TerminatingGateway (0.72s) +=== CONT TestACL_Legacy_Clone +2022-12-04T05:49:39.477Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:39.477Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:39.487Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:39.487Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:39.488 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:20965350-57fb-6143-63dc-77ad50d435e9 Address:127.0.0.1:23526}]" +49:39.488 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23526 [Follower]" leader= +49:39.488 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-20965350-57fb-6143-63dc-77ad50d435e9.dc1 127.0.0.1 +49:39.489 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-20965350-57fb-6143-63dc-77ad50d435e9 127.0.0.1 +2022-12-04T05:49:39.489Z [INFO] agent.router: Initializing LAN area manager +49:39.489 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23507 network=udp +49:39.489 [INFO] TestAgent.server: Adding LAN server: server="Node-20965350-57fb-6143-63dc-77ad50d435e9 (Addr: tcp/127.0.0.1:23526) (DC: dc1)" +49:39.489 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-20965350-57fb-6143-63dc-77ad50d435e9.dc1 area=wan +49:39.489 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23507 network=tcp +49:39.490 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23508 network=tcp +49:39.490 [INFO] TestAgent: started state syncer +49:39.490 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23534 network=tcp +49:39.539 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:39.539 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23526 [Candidate]" term=2 +49:39.539 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:39.539 [DEBUG] TestAgent.server.raft: vote granted: from=20965350-57fb-6143-63dc-77ad50d435e9 term=2 tally=1 +49:39.539 [INFO] TestAgent.server.raft: election won: tally=1 +49:39.539 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23526 [Leader]" +49:39.539 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:39.539 [INFO] TestAgent.server: cluster leadership acquired +49:39.539 [INFO] TestAgent.server: New leader elected: payload=Node-20965350-57fb-6143-63dc-77ad50d435e9 +49:39.540 [INFO] TestAgent.server: initializing acls +49:39.540 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:39.540 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:39.540 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:39.541 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:39.541 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:39.541 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:39.541 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-20965350-57fb-6143-63dc-77ad50d435e9 +49:39.541 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-20965350-57fb-6143-63dc-77ad50d435e9.dc1 +49:39.541 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:39.542 [INFO] TestAgent.server: Updating LAN server: server="Node-20965350-57fb-6143-63dc-77ad50d435e9 (Addr: tcp/127.0.0.1:23526) (DC: dc1)" +49:39.542 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-20965350-57fb-6143-63dc-77ad50d435e9.dc1 area=wan +49:39.543 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:39.543 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:39.543 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:39.543 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:39.543 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:39.543 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-20965350-57fb-6143-63dc-77ad50d435e9 +49:39.543 [INFO] TestAgent.server: member joined, marking health alive: member=Node-20965350-57fb-6143-63dc-77ad50d435e9 +49:39.543 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-20965350-57fb-6143-63dc-77ad50d435e9 +49:39.543 [INFO] TestAgent.server: federation state anti-entropy synced +49:39.614 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:39.614 [INFO] TestAgent: Synced node info +49:39.714 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:39.714 [DEBUG] TestAgent: Node info in sync +49:39.714 [DEBUG] TestAgent: Node info in sync +49:39.756 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-20965350-57fb-6143-63dc-77ad50d435e9 +49:39.756 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-20965350-57fb-6143-63dc-77ad50d435e9 +49:39.756 [DEBUG] TestAgent: dropping service from result due to ACLs: service=mysql +49:39.757 [INFO] TestAgent: Requesting shutdown +49:39.757 [INFO] TestAgent.server: shutting down server +49:39.757 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:39.757 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:39.757 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:39.757 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:39.757 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:39.757 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:39.757 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:39.757 [DEBUG] TestAgent: Node info in sync +49:39.757 [WARN] TestAgent: Service registration blocked by ACLs: service=mysql accessorID=00000000-0000-0000-0000-000000000002 +49:39.757 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:39.757 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:39.757 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:39.757 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:39.757 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:39.757 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:39.757 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:39.757 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +2022-12-04T05:49:39.757Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:39.757Z [INFO] agent.router.manager: shutting down +49:39.757 [INFO] TestAgent: consul server down +49:39.757 [INFO] TestAgent: shutdown complete +49:39.757 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23507 network=tcp +49:39.757 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23507 network=udp +49:39.757 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23508 network=tcp +49:40.257 [INFO] TestAgent: Waiting for endpoints to shut down +49:40.257 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Services_ACLFilter (0.79s) + --- PASS: TestAgent_Services_ACLFilter/no_token (0.00s) + --- PASS: TestAgent_Services_ACLFilter/root_token (0.00s) +=== CONT TestACLEndpoint_LoginLogout_jwt +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! +=== RUN TestACL_Bootstrap/bootstrap +=== RUN TestACL_Bootstrap/not_again +[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys === RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/invalid_bearer_token === RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/valid_bearer_token_no_bindings @@ -45911,644 +51661,240 @@ === RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/valid_bearer_token_1_service_binding === RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery === RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/invalid_bearer_token -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/valid_bearer_token_no_bindings === RUN TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/valid_bearer_token_1_service_binding -2021-10-31T23:13:32.046Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:32.094Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:32.097 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6bbc323e-f378-7270-519b-e7e0cbd94851 Address:127.0.0.1:16818}]" -13:32.098 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6bbc323e-f378-7270-519b-e7e0cbd94851.dc1 127.0.0.1 -13:32.098 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6bbc323e-f378-7270-519b-e7e0cbd94851 127.0.0.1 -2021-10-31T23:13:32.098Z [INFO] agent.router: Initializing LAN area manager -13:32.099 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16792 network=udp -13:32.099 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16818 [Follower]" leader= -13:32.105 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6bbc323e-f378-7270-519b-e7e0cbd94851.dc1 area=wan -13:32.105 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16792 network=tcp -13:32.106 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16814 network=tcp -13:32.106 [INFO] TestAgent: started state syncer -13:32.106 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16840 network=tcp -13:32.113 [INFO] TestAgent.server: Adding LAN server: server="Node-6bbc323e-f378-7270-519b-e7e0cbd94851 (Addr: tcp/127.0.0.1:16818) (DC: dc1)" -13:32.140 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:32.140 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16818 [Candidate]" term=2 -13:32.140 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:32.140 [DEBUG] TestAgent.server.raft: vote granted: from=6bbc323e-f378-7270-519b-e7e0cbd94851 term=2 tally=1 -13:32.140 [INFO] TestAgent.server.raft: election won: tally=1 -13:32.140 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16818 [Leader]" -13:32.140 [INFO] TestAgent.server: cluster leadership acquired -13:32.140 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16818 -13:32.142 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:32.142 [INFO] TestAgent.server: New leader elected: payload=Node-6bbc323e-f378-7270-519b-e7e0cbd94851 -13:32.144 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:32.144 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:32.144 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:32.144 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:32.144 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:32.144 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6bbc323e-f378-7270-519b-e7e0cbd94851 -13:32.144 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6bbc323e-f378-7270-519b-e7e0cbd94851 -13:32.145 [INFO] TestAgent.server: federation state anti-entropy synced -13:32.465 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:32.466 [INFO] TestAgent: Synced node info -13:32.482 [INFO] TestAgent: Requesting shutdown -13:32.483 [INFO] TestAgent.server: shutting down server -13:32.483 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:32.483 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:32.483 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:32.483 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:32.483 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:32.483 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:32.483 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:32.483 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:32.483 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:32.483 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:32.483 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:32.483Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:32.483Z [INFO] agent.router.manager: shutting down -13:32.483 [INFO] TestAgent: consul server down -13:32.483 [INFO] TestAgent: shutdown complete -13:32.483 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16792 network=tcp -13:32.483 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16792 network=udp -13:32.483 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16814 network=tcp -13:32.985 [INFO] TestAgent: Waiting for endpoints to shut down -13:32.985 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ServicesFiltered (0.96s) -=== CONT TestOperator_RaftConfiguration -2021-10-31T23:13:32.036Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:32.094Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:32.097 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:76f27584-a049-949d-8f5e-1764e7a61e21 Address:127.0.0.1:16804}]" -13:32.104 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16804 [Follower]" leader= -13:32.110 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-76f27584-a049-949d-8f5e-1764e7a61e21.dc1 127.0.0.1 -13:32.111 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-76f27584-a049-949d-8f5e-1764e7a61e21 127.0.0.1 -2021-10-31T23:13:32.111Z [INFO] agent.router: Initializing LAN area manager -13:32.111 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16764 network=udp -13:32.112 [INFO] TestAgent.server: Adding LAN server: server="Node-76f27584-a049-949d-8f5e-1764e7a61e21 (Addr: tcp/127.0.0.1:16804) (DC: dc1)" -13:32.112 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-76f27584-a049-949d-8f5e-1764e7a61e21.dc1 area=wan -13:32.112 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16764 network=tcp -13:32.113 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16800 network=tcp -13:32.113 [INFO] TestAgent: started state syncer -13:32.153 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:32.153 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16804 [Candidate]" term=2 -13:32.154 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:32.154 [DEBUG] TestAgent.server.raft: vote granted: from=76f27584-a049-949d-8f5e-1764e7a61e21 term=2 tally=1 -13:32.154 [INFO] TestAgent.server.raft: election won: tally=1 -13:32.154 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16804 [Leader]" -13:32.154 [INFO] TestAgent.server: cluster leadership acquired -13:32.154 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16804 -13:32.155 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:32.157 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:32.157 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:32.157 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:32.157 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:32.157 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:32.157 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-76f27584-a049-949d-8f5e-1764e7a61e21 -13:32.157 [INFO] TestAgent.server: member joined, marking health alive: member=Node-76f27584-a049-949d-8f5e-1764e7a61e21 -13:32.158 [INFO] TestAgent.server: New leader elected: payload=Node-76f27584-a049-949d-8f5e-1764e7a61e21 -13:32.159 [INFO] TestAgent.server: federation state anti-entropy synced -13:32.334 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:32.334 [INFO] TestAgent: Synced node info -13:32.486 [INFO] TestAgent: Requesting shutdown -13:32.486 [INFO] TestAgent.server: shutting down server -13:32.486 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:32.486 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:32.486 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:32.486 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:32.486 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:32.487 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:32.487 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:32.487 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:32.487 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:32.487 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:32.487 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:32.487Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:32.487Z [INFO] agent.router.manager: shutting down -13:32.487 [INFO] TestAgent: consul server down -13:32.487 [INFO] TestAgent: shutdown complete -13:32.487 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16764 network=tcp -13:32.487 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16764 network=udp -13:32.487 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16800 network=tcp -13:32.988 [INFO] TestAgent: Waiting for endpoints to shut down -13:32.988 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Self (1.99s) - --- PASS: TestAgent_Self/normal (1.01s) - --- PASS: TestAgent_Self/no_grpc (0.98s) -=== CONT TestUiServices -2021-10-31T23:13:32.009Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:32.079Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:32.082 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ccffa51d-3337-0322-b487-47566730cd6f Address:127.0.0.1:16769}]" -13:32.083 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ccffa51d-3337-0322-b487-47566730cd6f.dc1 127.0.0.1 -13:32.083 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ccffa51d-3337-0322-b487-47566730cd6f 127.0.0.1 -2021-10-31T23:13:32.083Z [INFO] agent.router: Initializing LAN area manager -13:32.084 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16757 network=udp -13:32.084 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16769 [Follower]" leader= -13:32.084 [INFO] TestAgent.server: Adding LAN server: server="Node-ccffa51d-3337-0322-b487-47566730cd6f (Addr: tcp/127.0.0.1:16769) (DC: dc1)" -13:32.084 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ccffa51d-3337-0322-b487-47566730cd6f.dc1 area=wan -13:32.084 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16757 network=tcp -13:32.104 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16765 network=tcp -13:32.104 [INFO] TestAgent: started state syncer -13:32.104 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16784 network=tcp -13:32.122 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:32.122 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16769 [Candidate]" term=2 -13:32.122 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:32.122 [DEBUG] TestAgent.server.raft: vote granted: from=ccffa51d-3337-0322-b487-47566730cd6f term=2 tally=1 -13:32.122 [INFO] TestAgent.server.raft: election won: tally=1 -13:32.122 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16769 [Leader]" -13:32.122 [INFO] TestAgent.server: cluster leadership acquired -13:32.122 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16769 -13:32.124 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:32.124 [INFO] TestAgent.server: New leader elected: payload=Node-ccffa51d-3337-0322-b487-47566730cd6f -13:32.131 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:32.131 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:32.131 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:32.131 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:32.131 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:32.131 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ccffa51d-3337-0322-b487-47566730cd6f -13:32.131 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ccffa51d-3337-0322-b487-47566730cd6f -13:32.132 [INFO] TestAgent.server: federation state anti-entropy synced -13:32.277 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:32.278 [INFO] TestAgent: Synced node info -13:32.278 [DEBUG] TestAgent: Node info in sync -13:32.518 [INFO] TestAgent: Requesting shutdown -13:32.518 [INFO] TestAgent.server: shutting down server -13:32.518 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:32.518 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:32.518 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:32.518 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:32.518 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:32.519 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:32.519 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:32.519 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:32.519 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:32.519 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:32.519 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:32.519Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:32.519Z [INFO] agent.router.manager: shutting down -13:32.526 [INFO] TestAgent: consul server down -13:32.526 [INFO] TestAgent: shutdown complete -13:32.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16757 network=tcp -13:32.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16757 network=udp -13:32.526 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16765 network=tcp -13:33.028 [INFO] TestAgent: Waiting for endpoints to shut down -13:33.028 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Services_ExternalConnectProxy (1.06s) -=== CONT TestFireReceiveEvent -2021-10-31T23:13:32.238Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:32.272Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:32.278 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cb823aac-961f-bee2-1ca3-d0c68bc0ed79 Address:127.0.0.1:16811}]" -13:32.279 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cb823aac-961f-bee2-1ca3-d0c68bc0ed79.dc1 127.0.0.1 -13:32.279 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cb823aac-961f-bee2-1ca3-d0c68bc0ed79 127.0.0.1 -2021-10-31T23:13:32.279Z [INFO] agent.router: Initializing LAN area manager -13:32.280 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16841 network=udp -13:32.280 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16811 [Follower]" leader= -13:32.280 [INFO] TestAgent.server: Adding LAN server: server="Node-cb823aac-961f-bee2-1ca3-d0c68bc0ed79 (Addr: tcp/127.0.0.1:16811) (DC: dc1)" -13:32.280 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cb823aac-961f-bee2-1ca3-d0c68bc0ed79.dc1 area=wan -13:32.280 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16841 network=tcp -13:32.281 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16807 network=tcp -13:32.281 [INFO] TestAgent: started state syncer -13:32.281 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16812 network=tcp -13:32.317 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:32.318 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16811 [Candidate]" term=2 -13:32.318 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:32.318 [DEBUG] TestAgent.server.raft: vote granted: from=cb823aac-961f-bee2-1ca3-d0c68bc0ed79 term=2 tally=1 -13:32.318 [INFO] TestAgent.server.raft: election won: tally=1 -13:32.318 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16811 [Leader]" -13:32.318 [INFO] TestAgent.server: cluster leadership acquired -13:32.318 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16811 -13:32.319 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:32.320 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:32.320 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:32.320 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:32.320 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:32.320 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:32.320 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cb823aac-961f-bee2-1ca3-d0c68bc0ed79 -13:32.320 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cb823aac-961f-bee2-1ca3-d0c68bc0ed79 -13:32.320 [INFO] TestAgent.server: New leader elected: payload=Node-cb823aac-961f-bee2-1ca3-d0c68bc0ed79 -13:32.321 [INFO] TestAgent.server: federation state anti-entropy synced -13:32.550 [INFO] TestAgent: Requesting shutdown -13:32.550 [INFO] TestAgent.server: shutting down server -13:32.550 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:32.550 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:32.550 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:32.550 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:32.550 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:32.550 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:32.550 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:32.550 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:32.550 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:32.550 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:32.550 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:32.550 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:32.551Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:32.551Z [INFO] agent.router.manager: shutting down -13:32.551 [INFO] TestAgent: consul server down -13:32.551 [INFO] TestAgent: shutdown complete -13:32.551 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16841 network=tcp -13:32.551 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16841 network=udp -13:32.551 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16807 network=tcp -13:33.053 [INFO] TestAgent: Waiting for endpoints to shut down -13:33.053 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Services (0.88s) -=== CONT TestIngestUserEvent +2022-12-04T05:49:39.923Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:39.932Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:39.933 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b10d0848-0446-6d50-ecfb-6d1de0fd6478 Address:127.0.0.1:23470}]" +49:39.933 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23470 [Follower]" leader= +49:39.934 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b10d0848-0446-6d50-ecfb-6d1de0fd6478.dc1 127.0.0.1 +49:39.934 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b10d0848-0446-6d50-ecfb-6d1de0fd6478 127.0.0.1 +2022-12-04T05:49:39.934Z [INFO] agent.router: Initializing LAN area manager +49:39.934 [INFO] TestAgent.server: Adding LAN server: server="Node-b10d0848-0446-6d50-ecfb-6d1de0fd6478 (Addr: tcp/127.0.0.1:23470) (DC: dc1)" +49:39.934 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b10d0848-0446-6d50-ecfb-6d1de0fd6478.dc1 area=wan +49:39.934 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23465 network=udp +49:39.935 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23465 network=tcp +49:39.935 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23466 network=tcp +49:39.935 [INFO] TestAgent: started state syncer +49:39.935 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23541 network=tcp +49:39.995 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:39.995 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23470 [Candidate]" term=2 +49:39.995 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:39.995 [DEBUG] TestAgent.server.raft: vote granted: from=b10d0848-0446-6d50-ecfb-6d1de0fd6478 term=2 tally=1 +49:39.995 [INFO] TestAgent.server.raft: election won: tally=1 +49:39.995 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23470 [Leader]" +49:39.995 [INFO] TestAgent.server: cluster leadership acquired +49:39.996 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23470 +49:39.996 [INFO] TestAgent.server: New leader elected: payload=Node-b10d0848-0446-6d50-ecfb-6d1de0fd6478 +49:39.996 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:39.997 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:39.997 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:39.997 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:39.997 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:39.997 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:39.997 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b10d0848-0446-6d50-ecfb-6d1de0fd6478 +49:39.997 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b10d0848-0446-6d50-ecfb-6d1de0fd6478 +49:39.998 [INFO] TestAgent.server: federation state anti-entropy synced +49:40.120 [INFO] TestAgent: Requesting shutdown +49:40.120 [INFO] TestAgent.server: shutting down server +49:40.120 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:40.120 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:40.120 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:40.120 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:40.120 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:40.121 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:40.121 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:40.121 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:40.121 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:40.121 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:40.121 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:40.121 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:40.121Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:40.121Z [INFO] agent.router.manager: shutting down +49:40.121 [INFO] TestAgent: consul server down +49:40.121 [INFO] TestAgent: shutdown complete +49:40.121 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23465 network=tcp +49:40.122 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23465 network=udp +49:40.122 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23466 network=tcp +49:40.622 [INFO] TestAgent: Waiting for endpoints to shut down +49:40.622 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Services_ExternalConnectProxy (0.71s) +=== CONT TestACL_Disabled_Response +2022-12-04T05:49:39.826Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:39.837Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:39.839 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1fcce1d6-ec8a-babb-3288-9fc2d860884f Address:127.0.0.1:23610}]" +49:39.839 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23610 [Follower]" leader= +49:39.839 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1fcce1d6-ec8a-babb-3288-9fc2d860884f.dc1 127.0.0.1 +49:39.840 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1fcce1d6-ec8a-babb-3288-9fc2d860884f 127.0.0.1 +2022-12-04T05:49:39.840Z [INFO] agent.router: Initializing LAN area manager +49:39.840 [INFO] TestAgent.server: Adding LAN server: server="Node-1fcce1d6-ec8a-babb-3288-9fc2d860884f (Addr: tcp/127.0.0.1:23610) (DC: dc1)" +49:39.840 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1fcce1d6-ec8a-babb-3288-9fc2d860884f.dc1 area=wan +49:39.840 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23577 network=udp +49:39.840 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23577 network=tcp +49:39.841 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23578 network=tcp +49:39.841 [INFO] TestAgent: started state syncer +49:39.841 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23527 network=tcp +49:39.877 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:39.877 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23610 [Candidate]" term=2 +49:39.877 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:39.877 [DEBUG] TestAgent.server.raft: vote granted: from=1fcce1d6-ec8a-babb-3288-9fc2d860884f term=2 tally=1 +49:39.877 [INFO] TestAgent.server.raft: election won: tally=1 +49:39.877 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23610 [Leader]" +49:39.878 [INFO] TestAgent.server: cluster leadership acquired +49:39.878 [INFO] TestAgent.server: New leader elected: payload=Node-1fcce1d6-ec8a-babb-3288-9fc2d860884f +49:39.878 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23610 +49:39.878 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:39.880 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:39.880 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:39.880 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:39.880 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:39.880 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:39.880 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1fcce1d6-ec8a-babb-3288-9fc2d860884f +49:39.880 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1fcce1d6-ec8a-babb-3288-9fc2d860884f +49:39.880 [INFO] TestAgent.server: federation state anti-entropy synced +49:40.112 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:40.113 [INFO] TestAgent: Synced node info +49:40.113 [DEBUG] TestAgent: Node info in sync +49:40.169 [INFO] TestAgent: Requesting shutdown +49:40.169 [INFO] TestAgent.server: shutting down server +49:40.169 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:40.169 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:40.169 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:40.169 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:40.169 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:40.169 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:40.169 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:40.169 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:40.169 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:40.169 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:40.169 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:40.169Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:40.169Z [INFO] agent.router.manager: shutting down +49:40.169 [INFO] TestAgent: consul server down +49:40.169 [INFO] TestAgent: shutdown complete +49:40.169 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23577 network=tcp +49:40.169 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23577 network=udp +49:40.169 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23578 network=tcp +49:40.669 [INFO] TestAgent: Waiting for endpoints to shut down +49:40.670 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Services_MeshGateway (0.86s) +=== CONT TestACL_LoginProcedure_HTTP +2022-12-04T05:49:40.015Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:40.027Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:40.028 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:336ae04f-714d-7949-9d57-2a374b026a9f Address:127.0.0.1:23547}]" +49:40.029 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23547 [Follower]" leader= +49:40.029 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-336ae04f-714d-7949-9d57-2a374b026a9f.dc1 127.0.0.1 +49:40.029 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-336ae04f-714d-7949-9d57-2a374b026a9f 127.0.0.1 +2022-12-04T05:49:40.029Z [INFO] agent.router: Initializing LAN area manager +49:40.029 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23542 network=udp +49:40.029 [INFO] TestAgent.server: Adding LAN server: server="Node-336ae04f-714d-7949-9d57-2a374b026a9f (Addr: tcp/127.0.0.1:23547) (DC: dc1)" +49:40.030 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23542 network=tcp +49:40.030 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-336ae04f-714d-7949-9d57-2a374b026a9f.dc1 area=wan +49:40.030 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23543 network=tcp +49:40.030 [INFO] TestAgent: started state syncer +49:40.030 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23639 network=tcp +49:40.075 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:40.075 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23547 [Candidate]" term=2 +49:40.076 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:40.076 [DEBUG] TestAgent.server.raft: vote granted: from=336ae04f-714d-7949-9d57-2a374b026a9f term=2 tally=1 +49:40.076 [INFO] TestAgent.server.raft: election won: tally=1 +49:40.076 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23547 [Leader]" +49:40.076 [INFO] TestAgent.server: cluster leadership acquired +49:40.076 [INFO] TestAgent.server: New leader elected: payload=Node-336ae04f-714d-7949-9d57-2a374b026a9f +49:40.077 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23547 +49:40.078 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:40.079 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:40.079 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:40.079 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:40.079 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:40.079 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:40.079 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-336ae04f-714d-7949-9d57-2a374b026a9f +49:40.079 [INFO] TestAgent.server: member joined, marking health alive: member=Node-336ae04f-714d-7949-9d57-2a374b026a9f +49:40.080 [INFO] TestAgent.server: federation state anti-entropy synced +49:40.265 [INFO] TestAgent: Requesting shutdown +49:40.265 [INFO] TestAgent.server: shutting down server +49:40.265 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:40.265 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:40.265 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:40.265 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:40.265 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:40.265 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:40.265 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:40.265 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:40.265 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:40.265 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:40.265 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:40.265 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:40.265Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:40.265Z [INFO] agent.router.manager: shutting down +49:40.266 [INFO] TestAgent: consul server down +49:40.266 [INFO] TestAgent: shutdown complete +49:40.266 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23542 network=tcp +49:40.266 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23542 network=udp +49:40.266 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23543 network=tcp +49:40.766 [INFO] TestAgent: Waiting for endpoints to shut down +49:40.766 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ServicesFiltered (0.76s) +=== CONT TestACLReplicationStatus +2022-12-04T05:49:39.907Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:39.916Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:39.917 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:004c9370-29b5-5d41-c405-43818e6bf66b Address:127.0.0.1:23533}]" +49:39.917 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23533 [Follower]" leader= +49:39.918 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-004c9370-29b5-5d41-c405-43818e6bf66b.dc1 127.0.0.1 +49:39.918 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-004c9370-29b5-5d41-c405-43818e6bf66b 127.0.0.1 +2022-12-04T05:49:39.918Z [INFO] agent.router: Initializing LAN area manager +49:39.918 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-004c9370-29b5-5d41-c405-43818e6bf66b.dc1 area=wan +49:39.918 [INFO] TestAgent.server: Adding LAN server: server="Node-004c9370-29b5-5d41-c405-43818e6bf66b (Addr: tcp/127.0.0.1:23533) (DC: dc1)" +49:39.918 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23528 network=udp +49:39.918 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23528 network=tcp +49:39.919 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23529 network=tcp +49:39.919 [INFO] TestAgent: started state syncer +49:39.919 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23457 network=tcp +49:39.962 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:39.962 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23533 [Candidate]" term=2 +49:39.962 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:39.962 [DEBUG] TestAgent.server.raft: vote granted: from=004c9370-29b5-5d41-c405-43818e6bf66b term=2 tally=1 +49:39.962 [INFO] TestAgent.server.raft: election won: tally=1 +49:39.962 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23533 [Leader]" +49:39.962 [INFO] TestAgent.server: cluster leadership acquired +49:39.962 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23533 +49:39.963 [INFO] TestAgent.server: New leader elected: payload=Node-004c9370-29b5-5d41-c405-43818e6bf66b +49:39.963 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:39.964 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:39.964 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:39.964 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:39.964 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:39.964 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:39.964 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-004c9370-29b5-5d41-c405-43818e6bf66b +49:39.964 [INFO] TestAgent.server: member joined, marking health alive: member=Node-004c9370-29b5-5d41-c405-43818e6bf66b +49:39.965 [INFO] TestAgent.server: federation state anti-entropy synced +49:39.986 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:39.987 [INFO] TestAgent: Synced node info +49:40.279 [INFO] TestAgent: Requesting shutdown +49:40.280 [INFO] TestAgent.server: shutting down server +49:40.280 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:40.280 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:40.280 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:40.280 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:40.280 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:40.280 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:40.280 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:40.280 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:40.280 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:40.280 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:40.280 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:40.280Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:40.280Z [INFO] agent.router.manager: shutting down +49:40.281 [INFO] TestAgent: consul server down +49:40.281 [INFO] TestAgent: shutdown complete +49:40.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23528 network=tcp +49:40.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23528 network=udp +49:40.281 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23529 network=tcp +49:40.782 [INFO] TestAgent: Waiting for endpoints to shut down +49:40.782 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Services_Sidecar (0.88s) +=== CONT TestACL_HTTP [WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:32.475Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:32.542Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:32.544 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e825db64-55bc-a79b-fc5a-d83c20841d23 Address:127.0.0.1:16874}]" -13:32.545 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16874 [Follower]" leader= -13:32.554 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e825db64-55bc-a79b-fc5a-d83c20841d23.dc1 127.0.0.1 -13:32.555 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e825db64-55bc-a79b-fc5a-d83c20841d23 127.0.0.1 -2021-10-31T23:13:32.555Z [INFO] agent.router: Initializing LAN area manager -13:32.555 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16848 network=udp -13:32.555 [INFO] TestAgent.server: Adding LAN server: server="Node-e825db64-55bc-a79b-fc5a-d83c20841d23 (Addr: tcp/127.0.0.1:16874) (DC: dc1)" -13:32.555 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e825db64-55bc-a79b-fc5a-d83c20841d23.dc1 area=wan -13:32.555 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16848 network=tcp -13:32.571 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16870 network=tcp -13:32.571 [INFO] TestAgent: started state syncer -13:32.571 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16791 network=tcp -13:32.605 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:32.606 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16874 [Candidate]" term=2 -13:32.606 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:32.606 [DEBUG] TestAgent.server.raft: vote granted: from=e825db64-55bc-a79b-fc5a-d83c20841d23 term=2 tally=1 -13:32.606 [INFO] TestAgent.server.raft: election won: tally=1 -13:32.606 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16874 [Leader]" -13:32.606 [INFO] TestAgent.server: cluster leadership acquired -13:32.606 [INFO] TestAgent.server: New leader elected: payload=Node-e825db64-55bc-a79b-fc5a-d83c20841d23 -13:32.606 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16874 -13:32.607 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:32.610 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:32.610 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:32.610 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:32.610 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:32.610 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:32.610 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e825db64-55bc-a79b-fc5a-d83c20841d23 -13:32.610 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e825db64-55bc-a79b-fc5a-d83c20841d23 -13:32.611 [INFO] TestAgent.server: federation state anti-entropy synced -13:32.638 [INFO] TestAgent: Requesting shutdown -13:32.638 [INFO] TestAgent.server: shutting down server -13:32.638 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:32.638 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:32.638 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:32.638 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:32.638 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:32.638 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:32.638 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:32.638 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:32.638 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:32.638 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:32.638 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:32.638 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:32.639Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:32.639Z [INFO] agent.router.manager: shutting down -13:32.639 [INFO] TestAgent: consul server down -13:32.639 [INFO] TestAgent: shutdown complete -13:32.639 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16848 network=tcp -13:32.639 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16848 network=udp -13:32.639 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16870 network=tcp -13:33.145 [INFO] TestAgent: Waiting for endpoints to shut down -13:33.145 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_DELETE_ConflictingFlags (0.70s) -=== CONT TestShouldProcessUserEvent -=== RUN TestUiServices/No_Filter -=== PAUSE TestUiServices/No_Filter -=== RUN TestUiServices/Filtered -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:32.740Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:32.766Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:32.768 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:02494510-89de-0c4c-1287-5b761debb007 Address:127.0.0.1:16860}]" -13:32.768 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16860 [Follower]" leader= -13:32.769 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-02494510-89de-0c4c-1287-5b761debb007.dc1 127.0.0.1 -13:32.770 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-02494510-89de-0c4c-1287-5b761debb007 127.0.0.1 -2021-10-31T23:13:32.770Z [INFO] agent.router: Initializing LAN area manager -13:32.770 [INFO] TestAgent.server: Adding LAN server: server="Node-02494510-89de-0c4c-1287-5b761debb007 (Addr: tcp/127.0.0.1:16860) (DC: dc1)" -13:32.770 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-02494510-89de-0c4c-1287-5b761debb007.dc1 area=wan -13:32.770 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16799 network=udp -13:32.770 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16799 network=tcp -13:32.771 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16856 network=tcp -13:32.771 [INFO] TestAgent: started state syncer -13:32.771 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16854 network=tcp -13:32.809 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:32.810 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16860 [Candidate]" term=2 -13:32.810 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:32.810 [DEBUG] TestAgent.server.raft: vote granted: from=02494510-89de-0c4c-1287-5b761debb007 term=2 tally=1 -13:32.810 [INFO] TestAgent.server.raft: election won: tally=1 -13:32.810 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16860 [Leader]" -13:32.810 [INFO] TestAgent.server: cluster leadership acquired -13:32.810 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16860 -13:32.811 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:32.812 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:32.812 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:32.812 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:32.812 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:32.812 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:32.812 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-02494510-89de-0c4c-1287-5b761debb007 -13:32.812 [INFO] TestAgent.server: member joined, marking health alive: member=Node-02494510-89de-0c4c-1287-5b761debb007 -13:32.813 [INFO] TestAgent.server: New leader elected: payload=Node-02494510-89de-0c4c-1287-5b761debb007 -13:32.813 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:32.813 [INFO] TestAgent.server: federation state anti-entropy synced -13:32.813 [INFO] TestAgent: Synced node info -13:32.813 [DEBUG] TestAgent: Node info in sync -13:32.854 [INFO] TestAgent: Requesting shutdown -13:32.854 [INFO] TestAgent.server: shutting down server -13:32.854 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:32.854 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:32.854 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:32.854 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:32.854 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:32.854 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:32.854 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:32.854 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:32.854 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:32.854 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:32.854 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2021-10-31T23:13:32.854Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:32.854Z [INFO] agent.router.manager: shutting down -13:32.855 [INFO] TestAgent: consul server down -13:32.855 [INFO] TestAgent: shutdown complete -13:32.855 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16799 network=tcp -13:32.855 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16799 network=udp -13:32.855 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16856 network=tcp -13:33.358 [INFO] TestAgent: Waiting for endpoints to shut down -13:33.358 [INFO] TestAgent: Endpoints down -=== RUN TestOperator_RaftPeer/#01 -2021-10-31T23:13:32.277Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:32.306Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:32.308 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:41ea28c3-43c1-f052-e1e1-8d7f8bcb5902 Address:127.0.0.1:16790}]" -13:32.308 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-41ea28c3-43c1-f052-e1e1-8d7f8bcb5902.dc1 127.0.0.1 -13:32.309 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-41ea28c3-43c1-f052-e1e1-8d7f8bcb5902 127.0.0.1 -2021-10-31T23:13:32.309Z [INFO] agent.router: Initializing LAN area manager -13:32.309 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16813 network=udp -13:32.309 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16790 [Follower]" leader= -13:32.309 [INFO] TestAgent.server: Adding LAN server: server="Node-41ea28c3-43c1-f052-e1e1-8d7f8bcb5902 (Addr: tcp/127.0.0.1:16790) (DC: dc1)" -13:32.309 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-41ea28c3-43c1-f052-e1e1-8d7f8bcb5902.dc1 area=wan -13:32.309 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16813 network=tcp -13:32.310 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16786 network=tcp -13:32.310 [INFO] TestAgent: started state syncer -13:32.310 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16819 network=tcp -13:32.361 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:32.369 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:32.369 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16790 [Candidate]" term=2 -13:32.370 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:32.370 [DEBUG] TestAgent.server.raft: vote granted: from=41ea28c3-43c1-f052-e1e1-8d7f8bcb5902 term=2 tally=1 -13:32.370 [INFO] TestAgent.server.raft: election won: tally=1 -13:32.370 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16790 [Leader]" -13:32.370 [INFO] TestAgent.server: cluster leadership acquired -13:32.370 [INFO] TestAgent.server: initializing acls -13:32.370 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:32.371 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:32.371 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:32.371 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:32.371 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:32.371 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-41ea28c3-43c1-f052-e1e1-8d7f8bcb5902 -13:32.371 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-41ea28c3-43c1-f052-e1e1-8d7f8bcb5902.dc1 -13:32.372 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:32.372 [INFO] TestAgent.server: New leader elected: payload=Node-41ea28c3-43c1-f052-e1e1-8d7f8bcb5902 -13:32.372 [INFO] TestAgent.server: Updating LAN server: server="Node-41ea28c3-43c1-f052-e1e1-8d7f8bcb5902 (Addr: tcp/127.0.0.1:16790) (DC: dc1)" -13:32.372 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-41ea28c3-43c1-f052-e1e1-8d7f8bcb5902.dc1 area=wan -13:32.378 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:32.378 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:32.378 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:32.378 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:32.378 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:32.378 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-41ea28c3-43c1-f052-e1e1-8d7f8bcb5902 -13:32.378 [INFO] TestAgent.server: member joined, marking health alive: member=Node-41ea28c3-43c1-f052-e1e1-8d7f8bcb5902 -13:32.379 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-41ea28c3-43c1-f052-e1e1-8d7f8bcb5902 -13:32.379 [INFO] TestAgent.server: federation state anti-entropy synced -13:32.594 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:32.595 [INFO] TestAgent: Synced node info -13:32.730 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-41ea28c3-43c1-f052-e1e1-8d7f8bcb5902 -13:32.730 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-41ea28c3-43c1-f052-e1e1-8d7f8bcb5902 -13:32.907 [INFO] TestAgent: Requesting shutdown -13:32.907 [INFO] TestAgent.server: shutting down server -13:32.907 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:32.907 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:32.907 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:32.907 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:32.907 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:32.907 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:32.907 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:32.907 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:32.907 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:32.907 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:32.907 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:32.907 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:32.907 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:32.907 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:32.907 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:32.907Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:32.907Z [INFO] agent.router.manager: shutting down -13:32.908 [INFO] TestAgent: consul server down -13:32.908 [INFO] TestAgent: shutdown complete -13:32.908 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16813 network=tcp -13:32.908 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16813 network=udp -13:32.908 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16786 network=tcp -13:33.414 [INFO] TestAgent: Waiting for endpoints to shut down -13:33.414 [INFO] TestAgent: Endpoints down ---- PASS: TestACLEndpoint_LoginLogout_jwt (1.18s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys (0.01s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/invalid_bearer_token (0.00s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/valid_bearer_token_no_bindings (0.00s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/valid_bearer_token_1_service_binding (0.00s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks (0.05s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/invalid_bearer_token (0.01s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/valid_bearer_token_no_bindings (0.00s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/valid_bearer_token_1_service_binding (0.01s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery (0.12s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/invalid_bearer_token (0.05s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/valid_bearer_token_no_bindings (0.02s) - --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/valid_bearer_token_1_service_binding (0.02s) -=== CONT TestValidateUserEventParams ---- PASS: TestValidateUserEventParams (0.00s) -=== CONT TestUIGatewayServiceNodes_Ingress -2021-10-31T23:13:32.591Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:32.643Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:32.645 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3a07a149-ef1b-3f5f-5486-11cd6ce6c7d4 Address:127.0.0.1:16713}]" -13:32.645 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16713 [Follower]" leader= -13:32.654 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3a07a149-ef1b-3f5f-5486-11cd6ce6c7d4.dc1 127.0.0.1 -13:32.654 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3a07a149-ef1b-3f5f-5486-11cd6ce6c7d4 127.0.0.1 -2021-10-31T23:13:32.654Z [INFO] agent.router: Initializing LAN area manager -13:32.655 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16680 network=udp -13:32.655 [INFO] TestAgent.server: Adding LAN server: server="Node-3a07a149-ef1b-3f5f-5486-11cd6ce6c7d4 (Addr: tcp/127.0.0.1:16713) (DC: dc1)" -13:32.655 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3a07a149-ef1b-3f5f-5486-11cd6ce6c7d4.dc1 area=wan -13:32.655 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16680 network=tcp -13:32.656 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16709 network=tcp -13:32.656 [INFO] TestAgent: started state syncer -13:32.656 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16875 network=tcp -13:32.709 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:32.709 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16713 [Candidate]" term=2 -13:32.710 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:32.710 [DEBUG] TestAgent.server.raft: vote granted: from=3a07a149-ef1b-3f5f-5486-11cd6ce6c7d4 term=2 tally=1 -13:32.710 [INFO] TestAgent.server.raft: election won: tally=1 -13:32.710 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16713 [Leader]" -13:32.710 [INFO] TestAgent.server: cluster leadership acquired -13:32.710 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16713 -13:32.711 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:32.712 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:32.712 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:32.712 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:32.712 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:32.712 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:32.712 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3a07a149-ef1b-3f5f-5486-11cd6ce6c7d4 -13:32.712 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3a07a149-ef1b-3f5f-5486-11cd6ce6c7d4 -13:32.713 [INFO] TestAgent.server: New leader elected: payload=Node-3a07a149-ef1b-3f5f-5486-11cd6ce6c7d4 -13:32.713 [INFO] TestAgent.server: federation state anti-entropy synced -13:32.734 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:32.734 [INFO] TestAgent: Synced node info -13:32.734 [DEBUG] TestAgent: Node info in sync -13:32.982 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -13:32.990 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-3a07a149-ef1b-3f5f-5486-11cd6ce6c7d4.dc1 -13:32.990 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -13:32.991 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-3a07a149-ef1b-3f5f-5486-11cd6ce6c7d4 -13:32.991 [INFO] TestAgent: Requesting shutdown -13:32.991 [INFO] TestAgent.server: shutting down server -13:32.991 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:32.991 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:32.991 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:32.991 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:32.991 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:32.991 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:32.991 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:32.991 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:32.991 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:32.991 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:32.991 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:32.991Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:32.991Z [INFO] agent.router.manager: shutting down -13:32.991 [INFO] TestAgent: consul server down -13:32.991 [INFO] TestAgent: shutdown complete -13:32.991 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16680 network=tcp -13:32.991 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16680 network=udp -13:32.991 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16709 network=tcp -13:33.493 [INFO] TestAgent: Waiting for endpoints to shut down -13:33.493 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_KeyringList (1.02s) -=== CONT TestUIGatewayServiceNodes_Terminating -2021-10-31T23:13:32.581Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:32.655Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:32.657 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:29bf93ab-91f6-66aa-42e7-6a8fb5ebad37 Address:127.0.0.1:16797}]" -13:32.657 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16797 [Follower]" leader= -13:32.674 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-29bf93ab-91f6-66aa-42e7-6a8fb5ebad37.dc1 127.0.0.1 -13:32.675 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-29bf93ab-91f6-66aa-42e7-6a8fb5ebad37 127.0.0.1 -2021-10-31T23:13:32.675Z [INFO] agent.router: Initializing LAN area manager -13:32.675 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16771 network=udp -13:32.675 [INFO] TestAgent.server: Adding LAN server: server="Node-29bf93ab-91f6-66aa-42e7-6a8fb5ebad37 (Addr: tcp/127.0.0.1:16797) (DC: dc1)" -13:32.675 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-29bf93ab-91f6-66aa-42e7-6a8fb5ebad37.dc1 area=wan -13:32.675 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16771 network=tcp -13:32.676 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16793 network=tcp -13:32.676 [INFO] TestAgent: started state syncer -13:32.676 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16693 network=tcp -13:32.714 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:32.714 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16797 [Candidate]" term=2 -13:32.714 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:32.714 [DEBUG] TestAgent.server.raft: vote granted: from=29bf93ab-91f6-66aa-42e7-6a8fb5ebad37 term=2 tally=1 -13:32.714 [INFO] TestAgent.server.raft: election won: tally=1 -13:32.714 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16797 [Leader]" -13:32.715 [INFO] TestAgent.server: cluster leadership acquired -13:32.715 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16797 -13:32.715 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:32.717 [INFO] TestAgent.server: New leader elected: payload=Node-29bf93ab-91f6-66aa-42e7-6a8fb5ebad37 -13:32.717 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:32.717 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:32.717 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:32.717 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:32.717 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:32.717 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-29bf93ab-91f6-66aa-42e7-6a8fb5ebad37 -13:32.717 [INFO] TestAgent.server: member joined, marking health alive: member=Node-29bf93ab-91f6-66aa-42e7-6a8fb5ebad37 -13:32.727 [INFO] TestAgent.server: federation state anti-entropy synced -13:32.802 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:32.802 [INFO] TestAgent: Synced node info -13:32.802 [DEBUG] TestAgent: Node info in sync -13:32.998 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -13:33.020 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-29bf93ab-91f6-66aa-42e7-6a8fb5ebad37 -13:33.020 [INFO] TestAgent: Requesting shutdown -13:33.020 [INFO] TestAgent.server: shutting down server -13:33.020 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:33.020 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:33.020 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:33.020 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:33.020 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:33.020 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:33.020 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:33.020 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:33.020 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:33.021 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:33.021 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:33.021Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:33.021Z [INFO] agent.router.manager: shutting down -13:33.022 [INFO] TestAgent: consul server down -13:33.022 [INFO] TestAgent: shutdown complete -13:33.022 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16771 network=tcp -13:33.022 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16771 network=udp -13:33.022 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16793 network=tcp -13:33.525 [INFO] TestAgent: Waiting for endpoints to shut down -13:33.525 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_KeyringListLocalOnly (1.06s) -=== CONT TestKVSEndpoint_AcquireRelease -[WARN] freeport: 10 out of 39 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:32.764Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:32.864Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:32.871 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ed54a933-2d96-4a68-5468-8eb048b87ef0 Address:127.0.0.1:16853}]" -13:32.872 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ed54a933-2d96-4a68-5468-8eb048b87ef0.dc1 127.0.0.1 -13:32.873 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ed54a933-2d96-4a68-5468-8eb048b87ef0 127.0.0.1 -2021-10-31T23:13:32.873Z [INFO] agent.router: Initializing LAN area manager -13:32.873 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16876 network=udp -13:32.873 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16853 [Follower]" leader= -13:32.877 [INFO] TestAgent.server: Adding LAN server: server="Node-ed54a933-2d96-4a68-5468-8eb048b87ef0 (Addr: tcp/127.0.0.1:16853) (DC: dc1)" -13:32.882 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16876 network=tcp -13:32.883 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ed54a933-2d96-4a68-5468-8eb048b87ef0.dc1 area=wan -13:32.883 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16849 network=tcp -13:32.883 [INFO] TestAgent: started state syncer -13:32.883 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16798 network=tcp -13:32.941 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:32.942 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16853 [Candidate]" term=2 -13:32.942 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:32.942 [DEBUG] TestAgent.server.raft: vote granted: from=ed54a933-2d96-4a68-5468-8eb048b87ef0 term=2 tally=1 -13:32.942 [INFO] TestAgent.server.raft: election won: tally=1 -13:32.942 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16853 [Leader]" -13:32.942 [INFO] TestAgent.server: cluster leadership acquired -13:32.942 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16853 -13:32.942 [INFO] TestAgent.server: New leader elected: payload=Node-ed54a933-2d96-4a68-5468-8eb048b87ef0 -13:32.943 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:32.944 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:32.944 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:32.944 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:32.944 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:32.944 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:32.944 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ed54a933-2d96-4a68-5468-8eb048b87ef0 -13:32.944 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ed54a933-2d96-4a68-5468-8eb048b87ef0 -13:32.945 [INFO] TestAgent.server: federation state anti-entropy synced -13:33.157 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:33.158 [INFO] TestAgent: Synced node info -13:33.158 [DEBUG] TestAgent: Node info in sync -13:33.202 [INFO] TestAgent.server.serf.wan: serf: Received install-key query -13:33.204 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-ed54a933-2d96-4a68-5468-8eb048b87ef0.dc1 -13:33.204 [INFO] TestAgent.server.serf.lan: serf: Received install-key query -13:33.205 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-ed54a933-2d96-4a68-5468-8eb048b87ef0 -13:33.205 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -13:33.205 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-ed54a933-2d96-4a68-5468-8eb048b87ef0.dc1 -13:33.205 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -13:33.205 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-ed54a933-2d96-4a68-5468-8eb048b87ef0 -13:33.205 [INFO] TestAgent: Requesting shutdown -13:33.205 [INFO] TestAgent.server: shutting down server -13:33.205 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:33.205 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:33.205 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:33.205 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:33.205 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:33.205 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:33.205 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:33.205 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:33.205 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:33.205 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:33.205 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2021-10-31T23:13:33.206Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:33.206Z [INFO] agent.router.manager: shutting down -13:33.206 [INFO] TestAgent: consul server down -13:33.206 [INFO] TestAgent: shutdown complete -13:33.206 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16876 network=tcp -13:33.206 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16876 network=udp -13:33.206 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16849 network=tcp -13:33.706 [INFO] TestAgent: Waiting for endpoints to shut down -13:33.706 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_KeyringInstall (0.99s) -=== CONT TestKVSEndpoint_PUT_ConflictingFlags -=== CONT TestKVSEndpoint_GET_Raw === RUN TestACL_Authorize/master-token === RUN TestACL_Authorize/master-token/dc1 === RUN TestACL_Authorize/master-token/dc2 @@ -46560,1051 +51906,226 @@ === RUN TestACL_Authorize/acl-not-found === RUN TestACL_Authorize/local-token-in-secondary-dc === RUN TestACL_Authorize/local-token-wrong-dc -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:33.058Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:33.153Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:33.171 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cb042135-acb3-a16a-2ccd-c7dfd5d8e175 Address:127.0.0.1:16868}]" -13:33.171 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16868 [Follower]" leader= -13:33.172 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cb042135-acb3-a16a-2ccd-c7dfd5d8e175.dc1 127.0.0.1 -13:33.172 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cb042135-acb3-a16a-2ccd-c7dfd5d8e175 127.0.0.1 -2021-10-31T23:13:33.172Z [INFO] agent.router: Initializing LAN area manager -13:33.173 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16877 network=udp -13:33.173 [INFO] TestAgent.server: Adding LAN server: server="Node-cb042135-acb3-a16a-2ccd-c7dfd5d8e175 (Addr: tcp/127.0.0.1:16868) (DC: dc1)" -13:33.173 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cb042135-acb3-a16a-2ccd-c7dfd5d8e175.dc1 area=wan -13:33.173 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16877 network=tcp -13:33.182 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16878 network=tcp -13:33.182 [INFO] TestAgent: started state syncer -13:33.182 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16869 network=tcp -13:33.209 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:33.209 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16868 [Candidate]" term=2 -13:33.210 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:33.210 [DEBUG] TestAgent.server.raft: vote granted: from=cb042135-acb3-a16a-2ccd-c7dfd5d8e175 term=2 tally=1 -13:33.210 [INFO] TestAgent.server.raft: election won: tally=1 -13:33.210 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16868 [Leader]" -13:33.210 [INFO] TestAgent.server: cluster leadership acquired -13:33.210 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16868 -13:33.211 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:33.212 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:33.212 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:33.212 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:33.212 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:33.212 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:33.212 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cb042135-acb3-a16a-2ccd-c7dfd5d8e175 -13:33.212 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cb042135-acb3-a16a-2ccd-c7dfd5d8e175 -13:33.213 [INFO] TestAgent.server: New leader elected: payload=Node-cb042135-acb3-a16a-2ccd-c7dfd5d8e175 -13:33.234 [INFO] TestAgent.server: federation state anti-entropy synced -13:33.330 [INFO] TestAgent: Requesting shutdown -13:33.341 [INFO] TestAgent.server: shutting down server -13:33.341 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:33.341 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:33.341 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:33.341 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:33.341 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:33.342 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:33.342 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:33.342 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:33.342 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:33.342 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:33.342 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:33.342 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:33.342Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:33.342Z [INFO] agent.router.manager: shutting down -13:33.342 [INFO] TestAgent: consul server down -13:33.342 [INFO] TestAgent: shutdown complete -13:33.342 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16877 network=tcp -13:33.342 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16877 network=udp -13:33.342 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16878 network=tcp -13:33.849 [INFO] TestAgent: Waiting for endpoints to shut down -13:33.883 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_RaftConfiguration (0.90s) -=== CONT TestOperator_AutopilotCASConfiguration -2021-10-31T23:13:33.188Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:33.261Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:33.283 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:683f3606-ea83-9ec6-77b0-20ee0a428f46 Address:127.0.0.1:16949}]" -13:33.284 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-683f3606-ea83-9ec6-77b0-20ee0a428f46.dc1 127.0.0.1 -13:33.284 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-683f3606-ea83-9ec6-77b0-20ee0a428f46 127.0.0.1 -2021-10-31T23:13:33.284Z [INFO] agent.router: Initializing LAN area manager -13:33.285 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16842 network=udp -13:33.285 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16949 [Follower]" leader= -13:33.288 [INFO] TestAgent.server: Adding LAN server: server="Node-683f3606-ea83-9ec6-77b0-20ee0a428f46 (Addr: tcp/127.0.0.1:16949) (DC: dc1)" -13:33.288 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-683f3606-ea83-9ec6-77b0-20ee0a428f46.dc1 area=wan -13:33.289 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16842 network=tcp -13:33.290 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16843 network=tcp -13:33.290 [INFO] TestAgent: started state syncer -13:33.290 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16950 network=tcp -13:33.326 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:33.326 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16949 [Candidate]" term=2 -13:33.326 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:33.326 [DEBUG] TestAgent.server.raft: vote granted: from=683f3606-ea83-9ec6-77b0-20ee0a428f46 term=2 tally=1 -13:33.326 [INFO] TestAgent.server.raft: election won: tally=1 -13:33.326 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16949 [Leader]" -13:33.327 [INFO] TestAgent.server: cluster leadership acquired -13:33.342 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16949 -13:33.342 [INFO] TestAgent.server: New leader elected: payload=Node-683f3606-ea83-9ec6-77b0-20ee0a428f46 -13:33.343 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:33.344 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:33.344 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:33.344 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:33.344 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:33.344 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:33.344 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-683f3606-ea83-9ec6-77b0-20ee0a428f46 -13:33.344 [INFO] TestAgent.server: member joined, marking health alive: member=Node-683f3606-ea83-9ec6-77b0-20ee0a428f46 -13:33.358 [INFO] TestAgent.server: federation state anti-entropy synced -13:33.502 [INFO] TestAgent: Requesting shutdown -13:33.502 [INFO] TestAgent.server: shutting down server -13:33.502 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:33.502 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:33.502 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:33.502 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:33.502 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:33.502 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:33.502 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:33.502 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:33.502 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:33.502 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:33.502 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:33.502 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:33.502Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:33.502Z [INFO] agent.router.manager: shutting down -13:33.514 [INFO] TestAgent: consul server down -13:33.514 [INFO] TestAgent: shutdown complete -13:33.514 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16842 network=tcp -13:33.514 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16842 network=udp -13:33.514 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16843 network=tcp -13:34.022 [INFO] TestAgent: Waiting for endpoints to shut down -13:34.022 [INFO] TestAgent: Endpoints down ---- PASS: TestShouldProcessUserEvent (0.88s) -=== CONT TestPreparedQuery_Execute -=== RUN TestPreparedQuery_Execute/#00 -2021-10-31T23:13:33.149Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:33.215Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:33.246 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f97e26ea-a3ab-2fe8-7a52-104a933cb0d1 Address:127.0.0.1:16882}]" -13:33.247 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f97e26ea-a3ab-2fe8-7a52-104a933cb0d1.dc1 127.0.0.1 -13:33.248 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f97e26ea-a3ab-2fe8-7a52-104a933cb0d1 127.0.0.1 -2021-10-31T23:13:33.248Z [INFO] agent.router: Initializing LAN area manager -13:33.248 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16882 [Follower]" leader= -13:33.248 [INFO] TestAgent.server: Adding LAN server: server="Node-f97e26ea-a3ab-2fe8-7a52-104a933cb0d1 (Addr: tcp/127.0.0.1:16882) (DC: dc1)" -13:33.248 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f97e26ea-a3ab-2fe8-7a52-104a933cb0d1.dc1 area=wan -13:33.249 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16898 network=tcp -13:33.249 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16898 network=udp -13:33.250 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16899 network=tcp -13:33.250 [INFO] TestAgent: started state syncer -13:33.250 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16883 network=tcp -13:33.289 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:33.289 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16882 [Candidate]" term=2 -13:33.290 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:33.290 [DEBUG] TestAgent.server.raft: vote granted: from=f97e26ea-a3ab-2fe8-7a52-104a933cb0d1 term=2 tally=1 -13:33.290 [INFO] TestAgent.server.raft: election won: tally=1 -13:33.290 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16882 [Leader]" -13:33.290 [INFO] TestAgent.server: cluster leadership acquired -13:33.290 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16882 -13:33.290 [INFO] TestAgent.server: New leader elected: payload=Node-f97e26ea-a3ab-2fe8-7a52-104a933cb0d1 -13:33.291 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:33.294 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:33.294 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:33.294 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:33.294 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:33.294 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:33.294 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f97e26ea-a3ab-2fe8-7a52-104a933cb0d1 -13:33.294 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f97e26ea-a3ab-2fe8-7a52-104a933cb0d1 -13:33.295 [INFO] TestAgent.server: federation state anti-entropy synced -13:33.482 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:33.482 [INFO] TestAgent: Synced node info -13:33.482 [DEBUG] TestAgent: Node info in sync -13:33.534 [DEBUG] TestAgent.server: User event: event=deploy -13:33.534 [DEBUG] TestAgent.server: User event: event=deploy -13:33.534 [DEBUG] TestAgent: new event: event_name=deploy event_id=1b0ca046-3c76-bcd7-2e50-af3bd3b08d08 -13:33.561 [INFO] TestAgent: Requesting shutdown -13:33.562 [INFO] TestAgent.server: shutting down server -13:33.562 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:33.562 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:33.562 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:33.562 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:33.562 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:33.562 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:33.562 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:33.562 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:33.562 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:33.562 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:33.562 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2021-10-31T23:13:33.562Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:33.562Z [INFO] agent.router.manager: shutting down -13:33.562 [INFO] TestAgent: consul server down -13:33.562 [INFO] TestAgent: shutdown complete -13:33.562 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16898 network=tcp -13:33.562 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16898 network=udp -13:33.562 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16899 network=tcp -13:34.065 [INFO] TestAgent: Waiting for endpoints to shut down -13:34.065 [INFO] TestAgent: Endpoints down ---- PASS: TestFireReceiveEvent (1.04s) -=== CONT TestPreparedQuery_List -=== RUN TestPreparedQuery_List/#00 -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=33)_compressed=false -2021-10-31T23:13:33.413Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:33.471Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:33.489 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4503c33c-fb71-affe-5b28-ba73b1c20520 Address:127.0.0.1:16931}]" -13:33.489 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16931 [Follower]" leader= -13:33.490 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4503c33c-fb71-affe-5b28-ba73b1c20520.dc1 127.0.0.1 -13:33.491 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4503c33c-fb71-affe-5b28-ba73b1c20520 127.0.0.1 -2021-10-31T23:13:33.491Z [INFO] agent.router: Initializing LAN area manager -13:33.492 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4503c33c-fb71-affe-5b28-ba73b1c20520.dc1 area=wan -13:33.492 [INFO] TestAgent.server: Adding LAN server: server="Node-4503c33c-fb71-affe-5b28-ba73b1c20520 (Addr: tcp/127.0.0.1:16931) (DC: dc1)" -13:33.497 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16937 network=udp -13:33.497 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16937 network=tcp -13:33.499 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16938 network=tcp -13:33.499 [INFO] TestAgent: started state syncer -13:33.499 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16918 network=tcp -13:33.553 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:33.553 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16931 [Candidate]" term=2 -13:33.556 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:33.556 [DEBUG] TestAgent.server.raft: vote granted: from=4503c33c-fb71-affe-5b28-ba73b1c20520 term=2 tally=1 -13:33.556 [INFO] TestAgent.server.raft: election won: tally=1 -13:33.556 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16931 [Leader]" -13:33.556 [INFO] TestAgent.server: cluster leadership acquired -13:33.556 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16931 -13:33.557 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:33.557 [INFO] TestAgent.server: New leader elected: payload=Node-4503c33c-fb71-affe-5b28-ba73b1c20520 -13:33.560 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:33.560 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:33.560 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:33.560 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:33.560 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:33.560 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4503c33c-fb71-affe-5b28-ba73b1c20520 -13:33.560 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4503c33c-fb71-affe-5b28-ba73b1c20520 -13:33.562 [INFO] TestAgent.server: federation state anti-entropy synced -13:33.654 [INFO] TestAgent: Requesting shutdown -13:33.654 [INFO] TestAgent.server: shutting down server -13:33.654 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:33.654 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:33.654 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:33.654 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:33.654 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:33.654 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:33.654 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:33.654 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:33.654 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:33.654 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:33.654 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:33.654 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:33.654Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:33.654Z [INFO] agent.router.manager: shutting down -13:33.654 [INFO] TestAgent: consul server down -13:33.654 [INFO] TestAgent: shutdown complete -13:33.654 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16937 network=tcp -13:33.654 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16937 network=udp -13:33.654 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16938 network=tcp -13:34.155 [INFO] TestAgent: Waiting for endpoints to shut down -13:34.155 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_RaftPeer (1.44s) - --- PASS: TestOperator_RaftPeer/#00 (0.64s) - --- PASS: TestOperator_RaftPeer/#01 (0.80s) -=== CONT TestPreparedQuery_Create -2021-10-31T23:13:33.444Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:33.542Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:33.545 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5b69b0f4-f44f-3f71-c904-6f0c5673cb0c Address:127.0.0.1:16956}]" -13:33.546 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16956 [Follower]" leader= -13:33.567 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5b69b0f4-f44f-3f71-c904-6f0c5673cb0c.dc1 127.0.0.1 -13:33.568 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5b69b0f4-f44f-3f71-c904-6f0c5673cb0c 127.0.0.1 -2021-10-31T23:13:33.568Z [INFO] agent.router: Initializing LAN area manager -13:33.569 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16905 network=udp -13:33.569 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16905 network=tcp -13:33.569 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5b69b0f4-f44f-3f71-c904-6f0c5673cb0c.dc1 area=wan -13:33.570 [INFO] TestAgent.server: Adding LAN server: server="Node-5b69b0f4-f44f-3f71-c904-6f0c5673cb0c (Addr: tcp/127.0.0.1:16956) (DC: dc1)" -13:33.571 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16906 network=tcp -13:33.571 [INFO] TestAgent: started state syncer -13:33.571 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16957 network=tcp -13:33.609 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:33.609 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16956 [Candidate]" term=2 -13:33.610 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:33.610 [DEBUG] TestAgent.server.raft: vote granted: from=5b69b0f4-f44f-3f71-c904-6f0c5673cb0c term=2 tally=1 -13:33.610 [INFO] TestAgent.server.raft: election won: tally=1 -13:33.610 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16956 [Leader]" -13:33.610 [INFO] TestAgent.server: cluster leadership acquired -13:33.610 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16956 -13:33.611 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:33.613 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:33.613 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:33.613 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:33.613 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:33.613 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:33.613 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5b69b0f4-f44f-3f71-c904-6f0c5673cb0c -13:33.613 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5b69b0f4-f44f-3f71-c904-6f0c5673cb0c -13:33.615 [INFO] TestAgent.server: New leader elected: payload=Node-5b69b0f4-f44f-3f71-c904-6f0c5673cb0c -13:33.618 [INFO] TestAgent.server: federation state anti-entropy synced -13:33.692 [INFO] TestAgent: Requesting shutdown -13:33.692 [INFO] TestAgent.server: shutting down server -13:33.692 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:33.692 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:33.692 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:33.692 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:33.692 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:33.692 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:33.692 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:33.692 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:33.693 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:33.693 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:33.693 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:33.693 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:33.693Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:33.693Z [INFO] agent.router.manager: shutting down -13:33.693 [INFO] TestAgent: consul server down -13:33.693 [INFO] TestAgent: shutdown complete -13:33.693 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16905 network=tcp -13:33.693 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16905 network=udp -13:33.693 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16906 network=tcp -13:34.193 [INFO] TestAgent: Waiting for endpoints to shut down -13:34.193 [INFO] TestAgent: Endpoints down ---- PASS: TestUIGatewayServiceNodes_Ingress (0.78s) -=== CONT TestOperator_ServerHealth_Unhealthy -2021-10-31T23:13:33.575Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:33.642Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:33.644 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:754679b5-1d48-28e2-abf3-25093dd1a7cd Address:127.0.0.1:16847}]" -13:33.645 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-754679b5-1d48-28e2-abf3-25093dd1a7cd.dc1 127.0.0.1 -13:33.645 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16847 [Follower]" leader= -13:33.658 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-754679b5-1d48-28e2-abf3-25093dd1a7cd 127.0.0.1 -2021-10-31T23:13:33.658Z [INFO] agent.router: Initializing LAN area manager -13:33.658 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16986 network=udp -13:33.658 [INFO] TestAgent.server: Adding LAN server: server="Node-754679b5-1d48-28e2-abf3-25093dd1a7cd (Addr: tcp/127.0.0.1:16847) (DC: dc1)" -13:33.659 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-754679b5-1d48-28e2-abf3-25093dd1a7cd.dc1 area=wan -13:33.659 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16986 network=tcp -13:33.659 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16987 network=tcp -13:33.659 [INFO] TestAgent: started state syncer -13:33.659 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16855 network=tcp -13:33.693 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:33.694 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16847 [Candidate]" term=2 -13:33.694 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:33.694 [DEBUG] TestAgent.server.raft: vote granted: from=754679b5-1d48-28e2-abf3-25093dd1a7cd term=2 tally=1 -13:33.694 [INFO] TestAgent.server.raft: election won: tally=1 -13:33.694 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16847 [Leader]" -13:33.694 [INFO] TestAgent.server: cluster leadership acquired -13:33.694 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16847 -13:33.695 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:33.696 [INFO] TestAgent.server: New leader elected: payload=Node-754679b5-1d48-28e2-abf3-25093dd1a7cd -13:33.697 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:33.697 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:33.697 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:33.697 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:33.697 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:33.697 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-754679b5-1d48-28e2-abf3-25093dd1a7cd -13:33.697 [INFO] TestAgent.server: member joined, marking health alive: member=Node-754679b5-1d48-28e2-abf3-25093dd1a7cd -13:33.703 [INFO] TestAgent: Requesting shutdown -13:33.703 [INFO] TestAgent.server: shutting down server -13:33.703 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:33.703 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:33.703 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:33.703 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:33.703 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:33.703 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:33.703 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:33.703 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:33.703 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:33.703 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:33.703Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:33.703Z [INFO] agent.router.manager: shutting down -13:33.703 [INFO] TestAgent: consul server down -13:33.703 [INFO] TestAgent: shutdown complete -13:33.703 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16986 network=tcp -13:33.703 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16986 network=udp -13:33.703 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16987 network=tcp -13:33.709 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="error performing federation state anti-entropy sync: raft is already shutdown" -13:33.709 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:34.205 [INFO] TestAgent: Waiting for endpoints to shut down -13:34.205 [INFO] TestAgent: Endpoints down ---- PASS: TestUIGatewayServiceNodes_Terminating (0.71s) -=== CONT TestOperator_ServerHealth -2021-10-31T23:13:33.169Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:33.283Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:33.294 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:25d373a2-7f38-58af-da86-7534d401ec8d Address:127.0.0.1:16889}]" -13:33.295 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-25d373a2-7f38-58af-da86-7534d401ec8d.dc1 127.0.0.1 -13:33.296 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16889 [Follower]" leader= -13:33.296 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-25d373a2-7f38-58af-da86-7534d401ec8d 127.0.0.1 -2021-10-31T23:13:33.296Z [INFO] agent.router: Initializing LAN area manager -13:33.296 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-25d373a2-7f38-58af-da86-7534d401ec8d.dc1 area=wan -13:33.296 [INFO] TestAgent.server: Adding LAN server: server="Node-25d373a2-7f38-58af-da86-7534d401ec8d (Addr: tcp/127.0.0.1:16889) (DC: dc1)" -13:33.296 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16884 network=udp -13:33.297 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16884 network=tcp -13:33.306 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16885 network=tcp -13:33.307 [INFO] TestAgent: started state syncer -13:33.314 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16890 network=tcp -13:33.354 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:33.354 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16889 [Candidate]" term=2 -13:33.354 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:33.354 [DEBUG] TestAgent.server.raft: vote granted: from=25d373a2-7f38-58af-da86-7534d401ec8d term=2 tally=1 -13:33.354 [INFO] TestAgent.server.raft: election won: tally=1 -13:33.354 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16889 [Leader]" -13:33.354 [INFO] TestAgent.server: cluster leadership acquired -13:33.354 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16889 -13:33.355 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:33.357 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:33.357 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:33.357 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:33.357 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:33.357 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:33.357 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-25d373a2-7f38-58af-da86-7534d401ec8d -13:33.357 [INFO] TestAgent.server: member joined, marking health alive: member=Node-25d373a2-7f38-58af-da86-7534d401ec8d -13:33.358 [INFO] TestAgent.server: New leader elected: payload=Node-25d373a2-7f38-58af-da86-7534d401ec8d -13:33.358 [INFO] TestAgent.server: federation state anti-entropy synced -13:33.626 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:33.626 [INFO] TestAgent: Synced node info -13:33.626 [DEBUG] TestAgent: Node info in sync -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.698 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.699 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.700 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [DEBUG] TestAgent: new event: event_name=test event_id= -13:33.701 [INFO] TestAgent: Requesting shutdown -13:33.701 [INFO] TestAgent.server: shutting down server -13:33.701 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:33.701 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:33.701 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:33.701 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:33.701 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:33.701 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:33.701 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:33.701 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:33.701 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:33.701 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:33.701 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:33.706Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:33.706Z [INFO] agent.router.manager: shutting down -13:33.706 [INFO] TestAgent: consul server down -13:33.706 [INFO] TestAgent: shutdown complete -13:33.706 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16884 network=tcp -13:33.706 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16884 network=udp -13:33.706 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16885 network=tcp -13:34.209 [INFO] TestAgent: Waiting for endpoints to shut down -13:34.209 [INFO] TestAgent: Endpoints down ---- PASS: TestIngestUserEvent (1.14s) +=== RUN TestACL_Disabled_Response/ACLBootstrap +=== RUN TestACL_Disabled_Response/ACLReplicationStatus +=== RUN TestACL_Disabled_Response/AgentToken +=== RUN TestACL_Disabled_Response/ACLRulesTranslate +=== RUN TestACL_Disabled_Response/ACLRulesTranslateLegacyToken +=== RUN TestACL_Disabled_Response/ACLPolicyList +=== RUN TestACL_Disabled_Response/ACLPolicyCRUD +=== RUN TestACL_Disabled_Response/ACLPolicyCreate +=== RUN TestACL_Disabled_Response/ACLTokenList +=== RUN TestACL_Disabled_Response/ACLTokenCreate +=== RUN TestACL_Disabled_Response/ACLTokenSelf +=== RUN TestACL_Disabled_Response/ACLTokenCRUD +=== RUN TestACL_Disabled_Response/ACLRoleList +=== RUN TestACL_Disabled_Response/ACLRoleCreate +=== RUN TestACL_Disabled_Response/ACLRoleCRUD +=== RUN TestACL_Disabled_Response/ACLBindingRuleList +=== RUN TestACL_Disabled_Response/ACLBindingRuleCreate +=== RUN TestACL_Disabled_Response/ACLBindingRuleCRUD +=== RUN TestACL_Disabled_Response/ACLAuthMethodList +=== RUN TestACL_Disabled_Response/ACLAuthMethodCreate +=== RUN TestACL_Disabled_Response/ACLAuthMethodCRUD +=== RUN TestACL_Disabled_Response/ACLLogin +=== RUN TestACL_Disabled_Response/ACLLogout +=== RUN TestACL_Disabled_Response/ACLAuthorize +2022-12-04T05:49:40.120Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:40.144Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:40.146 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f4ee6fef-1599-abfd-36ec-d904dc2a010d Address:127.0.0.1:23631}]" +49:40.146 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23631 [Follower]" leader= +49:40.146 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f4ee6fef-1599-abfd-36ec-d904dc2a010d.dc1 127.0.0.1 +49:40.147 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f4ee6fef-1599-abfd-36ec-d904dc2a010d 127.0.0.1 +2022-12-04T05:49:40.147Z [INFO] agent.router: Initializing LAN area manager +49:40.147 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f4ee6fef-1599-abfd-36ec-d904dc2a010d.dc1 area=wan +49:40.147 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23654 network=udp +49:40.147 [INFO] TestAgent.server: Adding LAN server: server="Node-f4ee6fef-1599-abfd-36ec-d904dc2a010d (Addr: tcp/127.0.0.1:23631) (DC: dc1)" +49:40.147 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23654 network=tcp +49:40.148 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23655 network=tcp +49:40.148 [INFO] TestAgent: started state syncer +49:40.148 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23660 network=tcp +49:40.204 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:40.204 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23631 [Candidate]" term=2 +49:40.204 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:40.204 [DEBUG] TestAgent.server.raft: vote granted: from=f4ee6fef-1599-abfd-36ec-d904dc2a010d term=2 tally=1 +49:40.204 [INFO] TestAgent.server.raft: election won: tally=1 +49:40.204 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23631 [Leader]" +49:40.204 [INFO] TestAgent.server: cluster leadership acquired +49:40.204 [INFO] TestAgent.server: New leader elected: payload=Node-f4ee6fef-1599-abfd-36ec-d904dc2a010d +49:40.204 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23631 +49:40.205 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:40.206 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:40.206 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:40.206 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:40.206 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:40.206 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:40.206 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f4ee6fef-1599-abfd-36ec-d904dc2a010d +49:40.206 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f4ee6fef-1599-abfd-36ec-d904dc2a010d +49:40.207 [INFO] TestAgent.server: federation state anti-entropy synced +49:40.335 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:40.335 [INFO] TestAgent: Synced node info +49:40.335 [DEBUG] TestAgent: Node info in sync +49:40.377 [INFO] TestAgent: Requesting shutdown +49:40.377 [INFO] TestAgent.server: shutting down server +49:40.377 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:40.377 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:40.377 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:40.377 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:40.377 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:40.377 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:40.377 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:40.377 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:40.377 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:40.377 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:40.377 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:40.378Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:40.378Z [INFO] agent.router.manager: shutting down +49:40.378 [INFO] TestAgent: consul server down +49:40.378 [INFO] TestAgent: shutdown complete +49:40.378 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23654 network=tcp +49:40.378 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23654 network=udp +49:40.378 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23655 network=tcp +49:40.878 [INFO] TestAgent: Waiting for endpoints to shut down +49:40.878 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Services (0.78s) === CONT TestACL_Legacy_List -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:33.581Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:33.663Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:33.673 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2d3a3462-096d-dbfc-3a9c-fbf047d13cbe Address:127.0.0.1:16991}]" -13:33.674 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2d3a3462-096d-dbfc-3a9c-fbf047d13cbe.dc1 127.0.0.1 -13:33.675 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2d3a3462-096d-dbfc-3a9c-fbf047d13cbe 127.0.0.1 -2021-10-31T23:13:33.675Z [INFO] agent.router: Initializing LAN area manager -13:33.675 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16991 [Follower]" leader= -13:33.676 [INFO] TestAgent.server: Adding LAN server: server="Node-2d3a3462-096d-dbfc-3a9c-fbf047d13cbe (Addr: tcp/127.0.0.1:16991) (DC: dc1)" -13:33.676 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2d3a3462-096d-dbfc-3a9c-fbf047d13cbe.dc1 area=wan -13:33.676 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16926 network=tcp -13:33.676 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16926 network=udp -13:33.677 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16927 network=tcp -13:33.677 [INFO] TestAgent: started state syncer -13:33.677 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16992 network=tcp -13:33.734 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:33.734 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16991 [Candidate]" term=2 -13:33.734 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:33.734 [DEBUG] TestAgent.server.raft: vote granted: from=2d3a3462-096d-dbfc-3a9c-fbf047d13cbe term=2 tally=1 -13:33.734 [INFO] TestAgent.server.raft: election won: tally=1 -13:33.734 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16991 [Leader]" -13:33.734 [INFO] TestAgent.server: cluster leadership acquired -13:33.734 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16991 -13:33.736 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:33.737 [INFO] TestAgent.server: New leader elected: payload=Node-2d3a3462-096d-dbfc-3a9c-fbf047d13cbe -13:33.750 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:33.750 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:33.750 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:33.750 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:33.750 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:33.750 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2d3a3462-096d-dbfc-3a9c-fbf047d13cbe -13:33.750 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2d3a3462-096d-dbfc-3a9c-fbf047d13cbe -13:33.752 [INFO] TestAgent.server: federation state anti-entropy synced -13:33.882 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:33.882 [INFO] TestAgent: Synced node info -13:33.984 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:33.985 [DEBUG] TestAgent: Node info in sync -13:33.985 [DEBUG] TestAgent: Node info in sync -13:34.003 [INFO] TestAgent: Requesting shutdown -13:34.003 [INFO] TestAgent.server: shutting down server -13:34.003 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:34.003 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:34.003 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:34.003 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:34.003 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:34.003 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:34.003 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:34.003 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:34.003 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:34.003 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:34.003 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:34.004Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:34.004Z [INFO] agent.router.manager: shutting down -13:34.004 [INFO] TestAgent: consul server down -13:34.004 [INFO] TestAgent: shutdown complete -13:34.004 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16926 network=tcp -13:34.004 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16926 network=udp -13:34.004 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16927 network=tcp -13:34.505 [INFO] TestAgent: Waiting for endpoints to shut down -13:34.505 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_AcquireRelease (0.98s) -=== CONT TestACL_LoginProcedure_HTTP -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:33.791Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:33.881Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:33.883 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:85e3f638-06b1-e208-dd66-53137f17dfd8 Address:127.0.0.1:16935}]" -13:33.884 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-85e3f638-06b1-e208-dd66-53137f17dfd8.dc1 127.0.0.1 -13:33.885 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-85e3f638-06b1-e208-dd66-53137f17dfd8 127.0.0.1 -2021-10-31T23:13:33.885Z [INFO] agent.router: Initializing LAN area manager -13:33.885 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16935 [Follower]" leader= -13:33.886 [INFO] TestAgent.server: Adding LAN server: server="Node-85e3f638-06b1-e208-dd66-53137f17dfd8 (Addr: tcp/127.0.0.1:16935) (DC: dc1)" -13:33.886 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-85e3f638-06b1-e208-dd66-53137f17dfd8.dc1 area=wan -13:33.889 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16979 network=udp -13:33.889 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16979 network=tcp -13:33.890 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16980 network=tcp -13:33.890 [INFO] TestAgent: started state syncer -13:33.891 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16936 network=tcp -13:33.923 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:33.923 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16935 [Candidate]" term=2 -13:33.923 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:33.923 [DEBUG] TestAgent.server.raft: vote granted: from=85e3f638-06b1-e208-dd66-53137f17dfd8 term=2 tally=1 -13:33.923 [INFO] TestAgent.server.raft: election won: tally=1 -13:33.923 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16935 [Leader]" -13:33.923 [INFO] TestAgent.server: cluster leadership acquired -13:33.923 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16935 -13:33.924 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:33.925 [INFO] TestAgent.server: New leader elected: payload=Node-85e3f638-06b1-e208-dd66-53137f17dfd8 -13:33.926 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:33.926 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:33.926 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:33.926 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:33.926 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:33.926 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-85e3f638-06b1-e208-dd66-53137f17dfd8 -13:33.926 [INFO] TestAgent.server: member joined, marking health alive: member=Node-85e3f638-06b1-e208-dd66-53137f17dfd8 -13:33.927 [INFO] TestAgent.server: federation state anti-entropy synced -13:33.994 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:33.995 [INFO] TestAgent: Synced node info -13:34.166 [INFO] TestAgent: Requesting shutdown -13:34.166 [INFO] TestAgent.server: shutting down server -13:34.166 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:34.166 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:34.166 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:34.166 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:34.166 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:34.166 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:34.166 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:34.166 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:34.166 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:34.166 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:34.166 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:34.166Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:34.166Z [INFO] agent.router.manager: shutting down -13:34.166 [INFO] TestAgent: consul server down -13:34.166 [INFO] TestAgent: shutdown complete -13:34.166 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16979 network=tcp -13:34.166 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16979 network=udp -13:34.166 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16980 network=tcp -13:34.670 [INFO] TestAgent: Waiting for endpoints to shut down -13:34.670 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_PUT_ConflictingFlags (0.96s) -=== CONT TestACL_HTTP +2022-12-04T05:49:40.155Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.155Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:40.165Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.165Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:40.167 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e6d3eaa7-1caf-40f7-e20b-6c84e1552494 Address:127.0.0.1:23575}]" +49:40.167 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23575 [Follower]" leader= +49:40.167 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e6d3eaa7-1caf-40f7-e20b-6c84e1552494.dc1 127.0.0.1 +49:40.168 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e6d3eaa7-1caf-40f7-e20b-6c84e1552494 127.0.0.1 +2022-12-04T05:49:40.168Z [INFO] agent.router: Initializing LAN area manager +49:40.168 [INFO] TestAgent.server: Adding LAN server: server="Node-e6d3eaa7-1caf-40f7-e20b-6c84e1552494 (Addr: tcp/127.0.0.1:23575) (DC: dc1)" +49:40.168 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e6d3eaa7-1caf-40f7-e20b-6c84e1552494.dc1 area=wan +49:40.168 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23584 network=tcp +49:40.168 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23584 network=udp +49:40.169 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23585 network=tcp +49:40.169 [INFO] TestAgent: started state syncer +49:40.169 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23590 network=tcp +49:40.218 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:40.232 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:40.232 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23575 [Candidate]" term=2 +49:40.232 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:40.232 [DEBUG] TestAgent.server.raft: vote granted: from=e6d3eaa7-1caf-40f7-e20b-6c84e1552494 term=2 tally=1 +49:40.232 [INFO] TestAgent.server.raft: election won: tally=1 +49:40.232 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23575 [Leader]" +49:40.232 [INFO] TestAgent.server: cluster leadership acquired +49:40.232 [INFO] TestAgent.server: New leader elected: payload=Node-e6d3eaa7-1caf-40f7-e20b-6c84e1552494 +49:40.232 [INFO] TestAgent.server: initializing acls +49:40.233 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:40.233 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:40.233 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:40.233 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:40.233 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e6d3eaa7-1caf-40f7-e20b-6c84e1552494 +49:40.233 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e6d3eaa7-1caf-40f7-e20b-6c84e1552494.dc1 +49:40.233 [INFO] TestAgent.server: Updating LAN server: server="Node-e6d3eaa7-1caf-40f7-e20b-6c84e1552494 (Addr: tcp/127.0.0.1:23575) (DC: dc1)" +49:40.233 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e6d3eaa7-1caf-40f7-e20b-6c84e1552494.dc1 area=wan +49:40.234 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:40.235 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:40.235 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:40.235 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:40.235 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:40.235 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:40.235 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e6d3eaa7-1caf-40f7-e20b-6c84e1552494 +49:40.235 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e6d3eaa7-1caf-40f7-e20b-6c84e1552494 +49:40.236 [INFO] TestAgent.server: federation state anti-entropy synced +49:40.236 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e6d3eaa7-1caf-40f7-e20b-6c84e1552494 +49:40.277 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" +49:40.460 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e6d3eaa7-1caf-40f7-e20b-6c84e1552494 +49:40.460 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e6d3eaa7-1caf-40f7-e20b-6c84e1552494 +49:40.460 [WARN] TestAgent.server.acl: failed to remove bootstrap file: error="remove /tmp/consul-test/TestACL_Bootstrap-TestAgent-agent524014433/acl-bootstrap-reset: no such file or directory" +49:40.461 [INFO] TestAgent.server.acl: ACL bootstrap completed +49:40.461 [INFO] TestAgent: Requesting shutdown +49:40.461 [INFO] TestAgent.server: shutting down server +49:40.461 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:40.461 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:40.461 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:40.461 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:40.461 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:40.461 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:40.461 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:40.461 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:40.461 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:40.461 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:40.461 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:40.461 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:40.461 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:40.461 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:40.461 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +2022-12-04T05:49:40.461Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:40.461Z [INFO] agent.router.manager: shutting down +49:40.461 [INFO] TestAgent: consul server down +49:40.461 [INFO] TestAgent: shutdown complete +49:40.461 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23584 network=tcp +49:40.461 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23584 network=udp +49:40.461 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23585 network=tcp +49:40.961 [INFO] TestAgent: Waiting for endpoints to shut down +49:40.961 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Bootstrap (0.82s) + --- PASS: TestACL_Bootstrap/bootstrap (0.00s) + --- PASS: TestACL_Bootstrap/not_again (0.00s) +=== CONT TestACL_Legacy_Get +=== RUN TestACL_Legacy_Get/wrong_id +2022-12-04T05:49:40.155Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:40.164Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:40.165 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6a83b688-6dca-e356-e0fb-c5ef018c13b9 Address:127.0.0.1:23638}]" +49:40.165 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23638 [Follower]" leader= +49:40.166 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6a83b688-6dca-e356-e0fb-c5ef018c13b9.dc1 127.0.0.1 +49:40.166 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6a83b688-6dca-e356-e0fb-c5ef018c13b9 127.0.0.1 +2022-12-04T05:49:40.166Z [INFO] agent.router: Initializing LAN area manager +49:40.166 [INFO] TestAgent.server: Adding LAN server: server="Node-6a83b688-6dca-e356-e0fb-c5ef018c13b9 (Addr: tcp/127.0.0.1:23638) (DC: dc1)" +49:40.166 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6a83b688-6dca-e356-e0fb-c5ef018c13b9.dc1 area=wan +49:40.167 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23619 network=udp +49:40.167 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23619 network=tcp +49:40.167 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23620 network=tcp +49:40.167 [INFO] TestAgent: started state syncer +49:40.167 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23611 network=tcp +49:40.202 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:40.202 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23638 [Candidate]" term=2 +49:40.202 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:40.202 [DEBUG] TestAgent.server.raft: vote granted: from=6a83b688-6dca-e356-e0fb-c5ef018c13b9 term=2 tally=1 +49:40.202 [INFO] TestAgent.server.raft: election won: tally=1 +49:40.202 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23638 [Leader]" +49:40.202 [INFO] TestAgent.server: cluster leadership acquired +49:40.203 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23638 +49:40.203 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:40.203 [INFO] TestAgent.server: New leader elected: payload=Node-6a83b688-6dca-e356-e0fb-c5ef018c13b9 +49:40.204 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:40.204 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:40.204 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:40.204 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:40.204 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:40.204 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6a83b688-6dca-e356-e0fb-c5ef018c13b9 +49:40.204 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6a83b688-6dca-e356-e0fb-c5ef018c13b9 +49:40.205 [INFO] TestAgent.server: federation state anti-entropy synced +49:40.467 [INFO] TestAgent: Requesting shutdown +49:40.467 [INFO] TestAgent.server: shutting down server +49:40.467 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:40.467 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:40.467 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:40.467 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:40.467 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:40.467 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:40.467 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:40.467 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:40.467 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:40.467 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:40.467 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:40.467 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:40.467Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:40.467Z [INFO] agent.router.manager: shutting down +49:40.467 [INFO] TestAgent: consul server down +49:40.467 [INFO] TestAgent: shutdown complete +49:40.467 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23619 network=tcp +49:40.467 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23619 network=udp +49:40.467 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23620 network=tcp +49:40.968 [INFO] TestAgent: Waiting for endpoints to shut down +49:40.968 [INFO] TestAgent: Endpoints down +--- PASS: TestRemoteExecWrites (0.83s) +=== CONT TestACL_Legacy_UpdateUpsert === RUN TestACL_LoginProcedure_HTTP/AuthMethod === RUN TestACL_LoginProcedure_HTTP/AuthMethod/Create === RUN TestACL_LoginProcedure_HTTP/AuthMethod/Create_other @@ -47622,638 +52143,95 @@ === RUN TestACL_LoginProcedure_HTTP/BindingRule/ID_Supplied === RUN TestACL_LoginProcedure_HTTP/BindingRule/Invalid_payload === RUN TestACL_LoginProcedure_HTTP/BindingRule/List +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! === RUN TestACL_LoginProcedure_HTTP/BindingRule/Delete === RUN TestACL_LoginProcedure_HTTP/BindingRule/Read === RUN TestACL_LoginProcedure_HTTP/Login === RUN TestACL_LoginProcedure_HTTP/Login/Create_Token_1 -2021-10-31T23:13:33.891Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:33.933Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:33.947 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef4a6a05-90ca-03e5-3d6a-2839ae20b0ae Address:127.0.0.1:16984}]" -13:33.948 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef4a6a05-90ca-03e5-3d6a-2839ae20b0ae.dc1 127.0.0.1 -13:33.948 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef4a6a05-90ca-03e5-3d6a-2839ae20b0ae 127.0.0.1 -2021-10-31T23:13:33.948Z [INFO] agent.router: Initializing LAN area manager -13:33.948 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16951 network=udp -13:33.948 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16984 [Follower]" leader= -13:33.949 [INFO] TestAgent.server: Adding LAN server: server="Node-ef4a6a05-90ca-03e5-3d6a-2839ae20b0ae (Addr: tcp/127.0.0.1:16984) (DC: dc1)" -13:33.949 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef4a6a05-90ca-03e5-3d6a-2839ae20b0ae.dc1 area=wan -13:33.949 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16951 network=tcp -13:33.958 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16952 network=tcp -13:33.958 [INFO] TestAgent: started state syncer -13:33.958 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16985 network=tcp -13:34.011 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:34.011 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16984 [Candidate]" term=2 -13:34.011 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:34.011 [DEBUG] TestAgent.server.raft: vote granted: from=ef4a6a05-90ca-03e5-3d6a-2839ae20b0ae term=2 tally=1 -13:34.011 [INFO] TestAgent.server.raft: election won: tally=1 -13:34.011 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16984 [Leader]" -13:34.011 [INFO] TestAgent.server: cluster leadership acquired -13:34.012 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16984 -13:34.013 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:34.013 [INFO] TestAgent.server: New leader elected: payload=Node-ef4a6a05-90ca-03e5-3d6a-2839ae20b0ae -13:34.015 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:34.015 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:34.015 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:34.015 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:34.015 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:34.015 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef4a6a05-90ca-03e5-3d6a-2839ae20b0ae -13:34.015 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef4a6a05-90ca-03e5-3d6a-2839ae20b0ae -13:34.017 [INFO] TestAgent.server: federation state anti-entropy synced -13:34.080 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:34.080 [INFO] TestAgent: Synced node info -13:34.330 [INFO] TestAgent: Requesting shutdown -13:34.330 [INFO] TestAgent.server: shutting down server -13:34.330 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:34.330 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:34.330 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:34.330 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:34.330 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:34.330 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:34.330 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:34.330 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:34.330 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:34.330 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:34.330 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:34.331Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:34.331Z [INFO] agent.router.manager: shutting down -13:34.331 [INFO] TestAgent: consul server down -13:34.331 [INFO] TestAgent: shutdown complete -13:34.331 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16951 network=tcp -13:34.331 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16951 network=udp -13:34.331 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16952 network=tcp -13:34.861 [INFO] TestAgent: Waiting for endpoints to shut down -13:34.861 [INFO] TestAgent: Endpoints down -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! ---- PASS: TestKVSEndpoint_GET_Raw (1.08s) -=== CONT TestACL_Bootstrap === RUN TestACL_LoginProcedure_HTTP/Login/Create_Token_2 === RUN TestACL_LoginProcedure_HTTP/Login/List_Tokens_by_(incorrect)_Method === RUN TestACL_LoginProcedure_HTTP/Login/List_Tokens_by_(correct)_Method === RUN TestACL_LoginProcedure_HTTP/Login/Logout === RUN TestACL_LoginProcedure_HTTP/Login/Token_is_gone_after_Logout -2021-10-31T23:13:32.939Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:32.991Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:32.993 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 Address:127.0.0.1:16903}]" -13:32.993 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16903 [Follower]" leader= -13:32.998 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0.dc2 127.0.0.1 -13:32.999 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 127.0.0.1 -2021-10-31T23:13:32.999Z [INFO] agent.router: Initializing LAN area manager -13:32.999 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16828 network=udp -13:32.999 [INFO] TestAgent.server: Adding LAN server: server="Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 (Addr: tcp/127.0.0.1:16903) (DC: dc2)" -13:32.999 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0.dc2 area=wan -13:33.000 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16828 network=tcp -13:33.025 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16829 network=tcp -13:33.025 [INFO] TestAgent: started state syncer -13:33.025 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16911 network=tcp -13:33.054 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false -13:33.054 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:33.054 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16903 [Candidate]" term=2 -13:33.054 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:33.054 [DEBUG] TestAgent.server.raft: vote granted: from=33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 term=2 tally=1 -13:33.054 [INFO] TestAgent.server.raft: election won: tally=1 -13:33.054 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16903 [Leader]" -13:33.054 [INFO] TestAgent.server: cluster leadership acquired -13:33.055 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false -13:33.055 [INFO] TestAgent.leader: started routine: routine="legacy ACL replication" -13:33.055 [INFO] TestAgent.server: started legacy ACL replication -13:33.055 [WARN] TestAgent.server.connect: primary datacenter is configured but unreachable - deferring initialization of the secondary datacenter CA -13:33.055 [INFO] TestAgent.leader: started routine: routine="config entry replication" -13:33.055 [INFO] TestAgent.leader: started routine: routine="federation state replication" -13:33.055 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:33.055 [INFO] TestAgent.leader: started routine: routine="secondary CA roots watch" -13:33.055 [INFO] TestAgent.leader: started routine: routine="intention replication" -13:33.055 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:33.055 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:33.055 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 -13:33.055 [INFO] TestAgent.server: member joined, marking health alive: member=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 -13:33.056 [INFO] TestAgent.server: New leader elected: payload=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 -13:33.057 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List -13:33.057 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" -13:33.057 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List -13:33.057 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" -13:33.057 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List -13:33.057 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" -13:33.057 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List -13:33.057 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" -13:33.057 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List -13:33.057 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" -13:33.057 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConfigEntry.ListAll -13:33.057 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc -13:33.057 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc -13:33.057 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc -13:33.057 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc -13:33.057 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc -13:33.057 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc -13:33.057 [DEBUG] TestAgent.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 -13:33.057 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConnectCA.Roots -13:33.057 [ERROR] TestAgent.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: No path to datacenter" -13:33.057 [DEBUG] TestAgent.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 -13:33.057 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=Intention.List -13:33.057 [ERROR] TestAgent.server.connect: error replicating intentions: routine="intention replication" error="No path to datacenter" -13:33.098 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.GetPolicy -13:33.098 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -13:33.098 [WARN] TestAgent: Node info update blocked by ACLs: node=33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 accessorID=legacy-token -13:33.098 [DEBUG] TestAgent: Node info in sync -13:33.157 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false -13:33.358 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false -13:33.469 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.GetPolicy -13:33.469 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 -13:33.469 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:16838] -13:33.472 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:16838 -13:33.474 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e003fbca-5731-0c37-0a4f-ee0c158eb6f7.dc1 127.0.0.1 -13:33.474 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -13:33.474 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 -13:33.474 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e003fbca-5731-0c37-0a4f-ee0c158eb6f7.dc1 area=wan -13:33.500 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 -13:33.526 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 -13:33.558 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 -13:33.563 [DEBUG] TestAgent.server.serf.wan: serf: messageJoinType: Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0.dc2 -13:33.615 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 -13:33.642 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 -13:33.668 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 -13:33.693 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 -13:33.734 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 -13:33.762 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 -13:33.762 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL replication" -13:33.762 [INFO] TestAgent.leader: started routine: routine="ACL policy replication" -13:33.762 [INFO] TestAgent.leader: started routine: routine="ACL role replication" -13:33.762 [INFO] TestAgent.leader: started routine: routine="ACL token replication" -13:33.762 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:33.762 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -13:33.762 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 -13:33.762 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0.dc2 -13:33.763 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0.dc2 area=wan -13:33.763 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL replication" -13:33.763 [INFO] TestAgent.server.replication.acl.policy: started ACL Policy replication -13:33.763 [INFO] TestAgent.server.replication.acl.role: started ACL Role replication -13:33.763 [INFO] TestAgent.server.replication.acl.token: started ACL Token replication -13:33.763 [INFO] TestAgent.server: Updating LAN server: server="Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 (Addr: tcp/127.0.0.1:16903) (DC: dc2)" -13:33.763 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0 -13:33.765 [DEBUG] TestAgent.server.replication.acl.token: finished fetching acls: amount=3 -13:33.765 [DEBUG] TestAgent.server.replication.acl.token: acl replication: local=0 remote=3 -13:33.765 [DEBUG] TestAgent.server.replication.acl.token: acl replication: deletions=0 updates=3 -13:33.765 [DEBUG] TestAgent.server.replication.acl.policy: finished fetching acls: amount=2 -13:33.765 [DEBUG] TestAgent.server.replication.acl.policy: acl replication: local=0 remote=2 -13:33.766 [DEBUG] TestAgent.server.replication.acl.token: acl replication - downloaded updates: amount=3 -13:33.766 [DEBUG] TestAgent.server.replication.acl.token: acl replication - performing updates -13:33.767 [DEBUG] TestAgent.server.replication.acl.token: acl replication - upserted batch: number_upserted=3 batch_size=442 -13:33.767 [DEBUG] TestAgent.server.replication.acl.token: acl replication - finished updates -13:33.767 [DEBUG] TestAgent.server.replication.acl.token: ACL replication completed through remote index: index=17 -13:33.770 [DEBUG] TestAgent.server.replication.acl.policy: acl replication: deletions=0 updates=2 -13:33.773 [DEBUG] TestAgent.server.replication.acl.policy: acl replication - downloaded updates: amount=2 -13:33.773 [DEBUG] TestAgent.server.replication.acl.policy: acl replication - performing updates -13:33.773 [DEBUG] TestAgent.server.replication.acl.policy: acl replication - upserted batch: number_upserted=2 batch_size=675 -13:33.773 [DEBUG] TestAgent.server.replication.acl.policy: acl replication - finished updates -13:33.773 [DEBUG] TestAgent.server.replication.acl.policy: ACL replication completed through remote index: index=16 -13:33.802 [DEBUG] TestAgent.server.replication.acl.role: finished fetching acls: amount=0 -13:33.802 [DEBUG] TestAgent.server.replication.acl.role: acl replication: local=0 remote=0 -13:33.802 [DEBUG] TestAgent.server.replication.acl.role: acl replication: deletions=0 updates=0 -13:33.802 [DEBUG] TestAgent.server.replication.acl.role: ACL replication completed through remote index: index=1 -13:33.882 [INFO] TestAgent: Requesting shutdown -13:33.882 [INFO] TestAgent.server: shutting down server -13:33.882 [DEBUG] TestAgent.leader: stopping routine: routine="ACL policy replication" -13:33.882 [DEBUG] TestAgent.leader: stopping routine: routine="config entry replication" -13:33.882 [DEBUG] TestAgent.leader: stopping routine: routine="federation state replication" -13:33.882 [DEBUG] TestAgent.leader: stopping routine: routine="secondary CA roots watch" -13:33.882 [DEBUG] TestAgent.leader: stopping routine: routine="intention replication" -13:33.882 [DEBUG] TestAgent.leader: stopping routine: routine="ACL token replication" -13:33.882 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:33.882 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:33.882 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:33.882 [DEBUG] TestAgent.leader: stopping routine: routine="ACL role replication" -13:33.882 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:33.882 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:33.882 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:33.882 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:33.882 [INFO] TestAgent.server.replication.config_entry: stopped replication -13:33.882 [DEBUG] TestAgent.leader: stopped routine: routine="config entry replication" -13:33.882 [INFO] TestAgent.replication.federation_state: stopped replication -13:33.882 [DEBUG] TestAgent.leader: stopped routine: routine="federation state replication" -13:33.882 [DEBUG] TestAgent.leader: stopped routine: routine="secondary CA roots watch" -13:33.882 [DEBUG] TestAgent.leader: stopped routine: routine="intention replication" -13:33.882 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:33.882 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:33.882 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:33.883 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:33.883Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:33.883Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:33.883Z [INFO] agent.router.manager: shutting down -13:33.883 [INFO] TestAgent: consul server down -13:33.883 [INFO] TestAgent: shutdown complete -13:33.883 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16828 network=tcp -13:33.883 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16828 network=udp -13:33.883 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16829 network=tcp -13:33.883 [INFO] TestAgent: Synced node info -13:33.883 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:16839 datacenter=dc1 method=ACL.PolicyList error="rpc error making call: EOF" -13:33.883 [WARN] TestAgent.server.replication.acl.policy: ACL replication error (will retry if still leader): error="failed to retrieve remote ACL policies: rpc error making call: EOF" -13:33.883 [DEBUG] TestAgent.leader: stopped routine: routine="ACL policy replication" -13:33.883 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:16839 datacenter=dc1 method=ACL.RoleList error="rpc error making call: EOF" -13:33.883 [WARN] TestAgent.server.replication.acl.role: ACL replication error (will retry if still leader): error="failed to retrieve remote ACL roles: rpc error making call: EOF" -13:33.883 [DEBUG] TestAgent.leader: stopped routine: routine="ACL role replication" -13:33.883 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:16839 datacenter=dc1 method=ACL.TokenList error="rpc error making call: EOF" -13:33.883 [WARN] TestAgent.server.replication.acl.token: ACL replication error (will retry if still leader): error="failed to retrieve remote ACL tokens: rpc error making call: EOF" -13:33.883 [DEBUG] TestAgent.leader: stopped routine: routine="ACL token replication" -13:34.385 [INFO] TestAgent: Waiting for endpoints to shut down -13:34.385 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:32.480Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:32.554Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:32.557 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e003fbca-5731-0c37-0a4f-ee0c158eb6f7 Address:127.0.0.1:16839}]" -13:32.557 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16839 [Follower]" leader= -13:32.562 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e003fbca-5731-0c37-0a4f-ee0c158eb6f7.dc1 127.0.0.1 -13:32.563 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e003fbca-5731-0c37-0a4f-ee0c158eb6f7 127.0.0.1 -2021-10-31T23:13:32.563Z [INFO] agent.router: Initializing LAN area manager -13:32.563 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16820 network=udp -13:32.563 [INFO] TestAgent.server: Adding LAN server: server="Node-e003fbca-5731-0c37-0a4f-ee0c158eb6f7 (Addr: tcp/127.0.0.1:16839) (DC: dc1)" -13:32.563 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e003fbca-5731-0c37-0a4f-ee0c158eb6f7.dc1 area=wan -13:32.564 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16820 network=tcp -13:32.565 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16835 network=tcp -13:32.565 [INFO] TestAgent: started state syncer -13:32.565 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16861 network=tcp -13:32.614 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:32.634 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:32.634 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16839 [Candidate]" term=2 -13:32.634 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:32.634 [DEBUG] TestAgent.server.raft: vote granted: from=e003fbca-5731-0c37-0a4f-ee0c158eb6f7 term=2 tally=1 -13:32.634 [INFO] TestAgent.server.raft: election won: tally=1 -13:32.634 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16839 [Leader]" -13:32.634 [INFO] TestAgent.server: cluster leadership acquired -13:32.635 [INFO] TestAgent.server: initializing acls -13:32.635 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:32.635 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:32.636 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:32.636 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:32.636 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:32.636 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e003fbca-5731-0c37-0a4f-ee0c158eb6f7 -13:32.636 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e003fbca-5731-0c37-0a4f-ee0c158eb6f7.dc1 -13:32.637 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:32.638 [INFO] TestAgent.server: New leader elected: payload=Node-e003fbca-5731-0c37-0a4f-ee0c158eb6f7 -13:32.638 [INFO] TestAgent.server: Updating LAN server: server="Node-e003fbca-5731-0c37-0a4f-ee0c158eb6f7 (Addr: tcp/127.0.0.1:16839) (DC: dc1)" -13:32.638 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e003fbca-5731-0c37-0a4f-ee0c158eb6f7.dc1 area=wan -13:32.638 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:32.638 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:32.638 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:32.638 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:32.638 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:32.638 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e003fbca-5731-0c37-0a4f-ee0c158eb6f7 -13:32.638 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e003fbca-5731-0c37-0a4f-ee0c158eb6f7 -13:32.639 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e003fbca-5731-0c37-0a4f-ee0c158eb6f7 -13:32.639 [INFO] TestAgent.server: federation state anti-entropy synced -13:32.769 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:32.770 [INFO] TestAgent: Synced node info -13:32.838 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-e003fbca-5731-0c37-0a4f-ee0c158eb6f7 -13:33.472 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:59010 -13:33.483 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0.dc2 127.0.0.1 -13:33.483 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0.dc2 area=wan -13:33.500 [DEBUG] TestAgent.server.serf.wan: serf: messageJoinType: Node-33daa85d-9e08-c3fe-84b2-f6ef4e779fe0.dc2 -13:34.385 [INFO] TestAgent: Requesting shutdown -13:34.386 [INFO] TestAgent.server: shutting down server -13:34.386 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:34.386 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:34.386 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:34.386 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:34.386 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:34.386 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:34.386 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:34.386 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:34.386 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:34.386 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:34.386 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:34.386 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:34.386 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:34.386 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:34.386 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:34.386Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:34.386Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:34.386Z [INFO] agent.router.manager: shutting down -13:34.386 [INFO] TestAgent: consul server down -13:34.386 [INFO] TestAgent: shutdown complete -13:34.386 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16820 network=tcp -13:34.386 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16820 network=udp -13:34.386 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16835 network=tcp -13:34.889 [INFO] TestAgent: Waiting for endpoints to shut down -13:34.889 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Authorize (2.46s) - --- PASS: TestACL_Authorize/master-token (0.01s) - --- PASS: TestACL_Authorize/master-token/dc1 (0.00s) - --- PASS: TestACL_Authorize/master-token/dc2 (0.01s) - --- PASS: TestACL_Authorize/custom-token (0.02s) - --- PASS: TestACL_Authorize/custom-token/dc1 (0.00s) - --- PASS: TestACL_Authorize/custom-token/dc2 (0.02s) - --- PASS: TestACL_Authorize/too-many-requests (0.00s) - --- PASS: TestACL_Authorize/decode-failure (0.00s) - --- PASS: TestACL_Authorize/acl-not-found (0.00s) - --- PASS: TestACL_Authorize/local-token-in-secondary-dc (0.01s) - --- PASS: TestACL_Authorize/local-token-wrong-dc (0.00s) -=== CONT TestACL_Disabled_Response -2021-10-31T23:13:33.909Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:33.994Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:34.006 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:941042ab-ea4f-e198-c5d7-1fac2c8a7ca9 Address:127.0.0.1:16910}]" -13:34.007 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-941042ab-ea4f-e198-c5d7-1fac2c8a7ca9.dc1 127.0.0.1 -13:34.008 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-941042ab-ea4f-e198-c5d7-1fac2c8a7ca9 127.0.0.1 -2021-10-31T23:13:34.008Z [INFO] agent.router: Initializing LAN area manager -13:34.008 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16943 network=udp -13:34.008 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16910 [Follower]" leader= -13:34.010 [INFO] TestAgent.server: Adding LAN server: server="Node-941042ab-ea4f-e198-c5d7-1fac2c8a7ca9 (Addr: tcp/127.0.0.1:16910) (DC: dc1)" -13:34.010 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-941042ab-ea4f-e198-c5d7-1fac2c8a7ca9.dc1 area=wan -13:34.010 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16943 network=tcp -13:34.011 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16944 network=tcp -13:34.011 [INFO] TestAgent: started state syncer -13:34.011 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16925 network=tcp -13:34.053 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:34.054 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16910 [Candidate]" term=2 -13:34.054 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:34.054 [DEBUG] TestAgent.server.raft: vote granted: from=941042ab-ea4f-e198-c5d7-1fac2c8a7ca9 term=2 tally=1 -13:34.054 [INFO] TestAgent.server.raft: election won: tally=1 -13:34.054 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16910 [Leader]" -13:34.054 [INFO] TestAgent.server: cluster leadership acquired -13:34.054 [INFO] TestAgent.server: New leader elected: payload=Node-941042ab-ea4f-e198-c5d7-1fac2c8a7ca9 -13:34.054 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16910 -13:34.056 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:34.068 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:34.068 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:34.068 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:34.068 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:34.068 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:34.068 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-941042ab-ea4f-e198-c5d7-1fac2c8a7ca9 -13:34.068 [INFO] TestAgent.server: member joined, marking health alive: member=Node-941042ab-ea4f-e198-c5d7-1fac2c8a7ca9 -13:34.070 [INFO] TestAgent.server: federation state anti-entropy synced -13:34.402 [INFO] TestAgent: Requesting shutdown -13:34.403 [INFO] TestAgent.server: shutting down server -13:34.403 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:34.403 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:34.403 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:34.403 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:34.403 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:34.403 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:34.403 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:34.403 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:34.403 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:34.403 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:34.403 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:34.403 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:34.403Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:34.403Z [INFO] agent.router.manager: shutting down -13:34.403 [INFO] TestAgent: consul server down -13:34.403 [INFO] TestAgent: shutdown complete -13:34.403 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16943 network=tcp -13:34.403 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16943 network=udp -13:34.403 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16944 network=tcp -13:34.904 [INFO] TestAgent: Waiting for endpoints to shut down -13:34.904 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_AutopilotCASConfiguration (1.02s) -=== CONT TestACLReplicationStatus -2021-10-31T23:13:34.170Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:34.190Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:34.193 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a16fdf3d-b959-ea38-0b61-b4a2b9712b6c Address:127.0.0.1:16977}]" -13:34.193 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a16fdf3d-b959-ea38-0b61-b4a2b9712b6c.dc1 127.0.0.1 -13:34.194 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16977 [Follower]" leader= -13:34.194 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a16fdf3d-b959-ea38-0b61-b4a2b9712b6c 127.0.0.1 -2021-10-31T23:13:34.194Z [INFO] agent.router: Initializing LAN area manager -13:34.194 [INFO] TestAgent.server: Adding LAN server: server="Node-a16fdf3d-b959-ea38-0b61-b4a2b9712b6c (Addr: tcp/127.0.0.1:16977) (DC: dc1)" -13:34.194 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a16fdf3d-b959-ea38-0b61-b4a2b9712b6c.dc1 area=wan -13:34.195 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17007 network=tcp -13:34.195 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17007 network=udp -13:34.195 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17008 network=tcp -13:34.195 [INFO] TestAgent: started state syncer -13:34.196 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16978 network=tcp -13:34.241 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:34.242 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16977 [Candidate]" term=2 -13:34.242 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:34.242 [DEBUG] TestAgent.server.raft: vote granted: from=a16fdf3d-b959-ea38-0b61-b4a2b9712b6c term=2 tally=1 -13:34.242 [INFO] TestAgent.server.raft: election won: tally=1 -13:34.242 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16977 [Leader]" -13:34.242 [INFO] TestAgent.server: cluster leadership acquired -13:34.242 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16977 -13:34.242 [INFO] TestAgent.server: New leader elected: payload=Node-a16fdf3d-b959-ea38-0b61-b4a2b9712b6c -13:34.243 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:34.244 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:34.244 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:34.244 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:34.244 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:34.244 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:34.245 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a16fdf3d-b959-ea38-0b61-b4a2b9712b6c -13:34.245 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a16fdf3d-b959-ea38-0b61-b4a2b9712b6c -13:34.256 [INFO] TestAgent.server: federation state anti-entropy synced -13:34.316 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:34.316 [INFO] TestAgent: Synced node info -13:34.462 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:34.462 [INFO] TestAgent: Requesting shutdown -13:34.462 [INFO] TestAgent.server: shutting down server -13:34.462 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:34.462 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:34.462 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:34.462 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:34.462 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:34.462 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:34.462 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:34.462 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:34.462 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:34.462 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:34.462 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:34.463Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:34.463Z [INFO] agent.router.manager: shutting down -13:34.463 [INFO] TestAgent: consul server down -13:34.463 [INFO] TestAgent: shutdown complete -13:34.463 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17007 network=tcp -13:34.463 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17007 network=udp -13:34.463 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17008 network=tcp -13:34.985 [INFO] TestAgent: Waiting for endpoints to shut down -13:34.985 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_Create (0.83s) -=== CONT TestKVSEndpoint_CAS -2021-10-31T23:13:34.157Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:34.192Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:34.210 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ede618d0-449c-6709-abb1-ba0314c9e220 Address:127.0.0.1:16826}]" -13:34.211 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ede618d0-449c-6709-abb1-ba0314c9e220.dc1 127.0.0.1 -13:34.212 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ede618d0-449c-6709-abb1-ba0314c9e220 127.0.0.1 -2021-10-31T23:13:34.212Z [INFO] agent.router: Initializing LAN area manager -13:34.212 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16863 network=udp -13:34.212 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16826 [Follower]" leader= -13:34.212 [INFO] TestAgent.server: Adding LAN server: server="Node-ede618d0-449c-6709-abb1-ba0314c9e220 (Addr: tcp/127.0.0.1:16826) (DC: dc1)" -13:34.212 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ede618d0-449c-6709-abb1-ba0314c9e220.dc1 area=wan -13:34.212 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16863 network=tcp -13:34.213 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16864 network=tcp -13:34.213 [INFO] TestAgent: started state syncer -13:34.213 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16827 network=tcp -13:34.278 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:34.278 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16826 [Candidate]" term=2 -13:34.278 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:34.278 [DEBUG] TestAgent.server.raft: vote granted: from=ede618d0-449c-6709-abb1-ba0314c9e220 term=2 tally=1 -13:34.278 [INFO] TestAgent.server.raft: election won: tally=1 -13:34.278 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16826 [Leader]" -13:34.278 [INFO] TestAgent.server: cluster leadership acquired -13:34.278 [INFO] TestAgent.server: New leader elected: payload=Node-ede618d0-449c-6709-abb1-ba0314c9e220 -13:34.278 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16826 -13:34.279 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:34.280 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:34.280 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:34.280 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:34.280 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:34.280 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:34.280 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ede618d0-449c-6709-abb1-ba0314c9e220 -13:34.280 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ede618d0-449c-6709-abb1-ba0314c9e220 -13:34.281 [INFO] TestAgent.server: federation state anti-entropy synced -13:34.495 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:34.496 [INFO] TestAgent: Requesting shutdown -13:34.496 [INFO] TestAgent.server: shutting down server -13:34.496 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:34.496 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:34.496 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:34.496 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:34.496 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:34.496 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:34.496 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:34.496 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:34.496 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:34.496 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:34.496 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:34.496 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:34.496Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:34.496Z [INFO] agent.router.manager: shutting down -13:34.496 [INFO] TestAgent: consul server down -13:34.496 [INFO] TestAgent: shutdown complete -13:34.496 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16863 network=tcp -13:34.496 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16863 network=udp -13:34.496 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16864 network=tcp -13:34.997 [INFO] TestAgent: Waiting for endpoints to shut down -13:34.997 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_List/#01 -2021-10-31T23:13:34.227Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:34.227Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:34.241Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:34.241Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:34.243 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1bfb96e0-07d7-1728-a5ff-b00a32c03b11 Address:127.0.0.1:17012}]" -13:34.244 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1bfb96e0-07d7-1728-a5ff-b00a32c03b11.dc1 127.0.0.1 -13:34.244 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1bfb96e0-07d7-1728-a5ff-b00a32c03b11 127.0.0.1 -2021-10-31T23:13:34.244Z [INFO] agent.router: Initializing LAN area manager -13:34.245 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17003 network=udp -13:34.245 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17012 [Follower]" leader= -13:34.246 [INFO] TestAgent.server: Adding LAN server: server="Node-1bfb96e0-07d7-1728-a5ff-b00a32c03b11 (Addr: tcp/127.0.0.1:17012) (DC: dc1)" -13:34.246 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17003 network=tcp -13:34.246 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1bfb96e0-07d7-1728-a5ff-b00a32c03b11.dc1 area=wan -13:34.246 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17004 network=tcp -13:34.246 [INFO] TestAgent: started state syncer -13:34.246 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17013 network=tcp -13:34.307 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:34.309 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:34.309 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17012 [Candidate]" term=2 -13:34.310 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:34.310 [DEBUG] TestAgent.server.raft: vote granted: from=1bfb96e0-07d7-1728-a5ff-b00a32c03b11 term=2 tally=1 -13:34.310 [INFO] TestAgent.server.raft: election won: tally=1 -13:34.310 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17012 [Leader]" -13:34.310 [INFO] TestAgent.server: cluster leadership acquired -13:34.310 [INFO] TestAgent.server: initializing acls -13:34.310 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:34.310 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:34.311 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:34.311 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:34.311 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:34.311 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:34.311 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1bfb96e0-07d7-1728-a5ff-b00a32c03b11 -13:34.311 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1bfb96e0-07d7-1728-a5ff-b00a32c03b11.dc1 -13:34.312 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:34.313 [INFO] TestAgent.server: New leader elected: payload=Node-1bfb96e0-07d7-1728-a5ff-b00a32c03b11 -13:34.314 [INFO] TestAgent.server: Updating LAN server: server="Node-1bfb96e0-07d7-1728-a5ff-b00a32c03b11 (Addr: tcp/127.0.0.1:17012) (DC: dc1)" -13:34.314 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1bfb96e0-07d7-1728-a5ff-b00a32c03b11.dc1 area=wan -13:34.315 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:34.316 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:34.316 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:34.316 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:34.316 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:34.316 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1bfb96e0-07d7-1728-a5ff-b00a32c03b11 -13:34.316 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1bfb96e0-07d7-1728-a5ff-b00a32c03b11 -13:34.316 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1bfb96e0-07d7-1728-a5ff-b00a32c03b11 -13:34.317 [INFO] TestAgent.server: federation state anti-entropy synced -13:34.410 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:34.411 [INFO] TestAgent: Synced node info -13:34.514 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-1bfb96e0-07d7-1728-a5ff-b00a32c03b11 -13:34.514 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-1bfb96e0-07d7-1728-a5ff-b00a32c03b11 -13:34.539 [INFO] TestAgent: Requesting shutdown -13:34.539 [INFO] TestAgent.server: shutting down server -13:34.539 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:34.539 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:34.539 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:34.539 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:34.539 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:34.539 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:34.539 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:34.539 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:34.539 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:34.539 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:34.540 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:34.540 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:34.540 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:34.540 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:34.540 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:34.540Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:34.540Z [INFO] agent.router.manager: shutting down -13:34.540 [INFO] TestAgent: consul server down -13:34.540 [INFO] TestAgent: shutdown complete -13:34.540 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17003 network=tcp -13:34.540 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17003 network=udp -13:34.540 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17004 network=tcp -13:35.046 [INFO] TestAgent: Waiting for endpoints to shut down -13:35.046 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Legacy_List (0.84s) -=== CONT TestKVSEndpoint_ListKeys -2021-10-31T23:13:34.095Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:34.158Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:34.160 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5fccee9c-6015-7302-0b9c-b7869ea726f6 Address:127.0.0.1:16833}]" -13:34.161 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5fccee9c-6015-7302-0b9c-b7869ea726f6.dc1 127.0.0.1 -13:34.161 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16833 [Follower]" leader= -13:34.166 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5fccee9c-6015-7302-0b9c-b7869ea726f6 127.0.0.1 -2021-10-31T23:13:34.166Z [INFO] agent.router: Initializing LAN area manager -13:34.166 [INFO] TestAgent.server: Adding LAN server: server="Node-5fccee9c-6015-7302-0b9c-b7869ea726f6 (Addr: tcp/127.0.0.1:16833) (DC: dc1)" -13:34.166 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5fccee9c-6015-7302-0b9c-b7869ea726f6.dc1 area=wan -13:34.166 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16958 network=udp -13:34.167 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16958 network=tcp -13:34.167 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16959 network=tcp -13:34.167 [INFO] TestAgent: started state syncer -13:34.167 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16834 network=tcp -13:34.209 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:34.209 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16833 [Candidate]" term=2 -13:34.210 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:34.210 [DEBUG] TestAgent.server.raft: vote granted: from=5fccee9c-6015-7302-0b9c-b7869ea726f6 term=2 tally=1 -13:34.210 [INFO] TestAgent.server.raft: election won: tally=1 -13:34.210 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16833 [Leader]" -13:34.210 [INFO] TestAgent.server: cluster leadership acquired -13:34.210 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16833 -13:34.211 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:34.212 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:34.212 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:34.212 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:34.212 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:34.212 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:34.212 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5fccee9c-6015-7302-0b9c-b7869ea726f6 -13:34.212 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5fccee9c-6015-7302-0b9c-b7869ea726f6 -13:34.213 [INFO] TestAgent.server: New leader elected: payload=Node-5fccee9c-6015-7302-0b9c-b7869ea726f6 -13:34.225 [INFO] TestAgent.server: federation state anti-entropy synced -13:34.278 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:34.278 [INFO] TestAgent: Synced node info -13:34.581 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:34.582 [INFO] TestAgent: Requesting shutdown -13:34.582 [INFO] TestAgent.server: shutting down server -13:34.582 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:34.582 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:34.582 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:34.582 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:34.582 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:34.582 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:34.582 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:34.582 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:34.582 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:34.582 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:34.582 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:34.582Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:34.582Z [INFO] agent.router.manager: shutting down -13:34.582 [INFO] TestAgent: consul server down -13:34.582 [INFO] TestAgent: shutdown complete -13:34.582 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16958 network=tcp -13:34.582 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16958 network=udp -13:34.582 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16959 network=tcp -13:35.083 [INFO] TestAgent: Waiting for endpoints to shut down -13:35.083 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Execute/#01 -[WARN] freeport: 10 out of 59 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:40.266Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.266Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:40.276Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.276Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:40.277 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cf819db4-3f52-d3fb-1f2a-85b0eff4b311 Address:127.0.0.1:23582}]" +49:40.277 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23582 [Follower]" leader= +49:40.278 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cf819db4-3f52-d3fb-1f2a-85b0eff4b311.dc1 127.0.0.1 +49:40.278 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cf819db4-3f52-d3fb-1f2a-85b0eff4b311 127.0.0.1 +2022-12-04T05:49:40.278Z [INFO] agent.router: Initializing LAN area manager +49:40.278 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cf819db4-3f52-d3fb-1f2a-85b0eff4b311.dc1 area=wan +49:40.278 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23626 network=udp +49:40.278 [INFO] TestAgent.server: Adding LAN server: server="Node-cf819db4-3f52-d3fb-1f2a-85b0eff4b311 (Addr: tcp/127.0.0.1:23582) (DC: dc1)" +49:40.279 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23626 network=tcp +49:40.279 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23627 network=tcp +49:40.279 [INFO] TestAgent: started state syncer +49:40.279 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23618 network=tcp +49:40.328 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:40.341 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:40.341 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23582 [Candidate]" term=2 +49:40.341 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:40.341 [DEBUG] TestAgent.server.raft: vote granted: from=cf819db4-3f52-d3fb-1f2a-85b0eff4b311 term=2 tally=1 +49:40.341 [INFO] TestAgent.server.raft: election won: tally=1 +49:40.341 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23582 [Leader]" +49:40.341 [INFO] TestAgent.server: cluster leadership acquired +49:40.341 [INFO] TestAgent.server: New leader elected: payload=Node-cf819db4-3f52-d3fb-1f2a-85b0eff4b311 +49:40.341 [INFO] TestAgent.server: initializing acls +49:40.341 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:40.341 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:40.342 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:40.342 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:40.342 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:40.342 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:40.342 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-cf819db4-3f52-d3fb-1f2a-85b0eff4b311 +49:40.342 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-cf819db4-3f52-d3fb-1f2a-85b0eff4b311.dc1 +49:40.342 [INFO] TestAgent.server: Updating LAN server: server="Node-cf819db4-3f52-d3fb-1f2a-85b0eff4b311 (Addr: tcp/127.0.0.1:23582) (DC: dc1)" +49:40.342 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-cf819db4-3f52-d3fb-1f2a-85b0eff4b311.dc1 area=wan +49:40.343 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:40.344 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:40.344 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:40.344 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:40.344 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:40.344 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:40.344 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cf819db4-3f52-d3fb-1f2a-85b0eff4b311 +49:40.344 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cf819db4-3f52-d3fb-1f2a-85b0eff4b311 +49:40.345 [INFO] TestAgent.server: federation state anti-entropy synced +49:40.345 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cf819db4-3f52-d3fb-1f2a-85b0eff4b311 +49:40.434 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:40.434 [INFO] TestAgent: Synced node info +49:40.434 [DEBUG] TestAgent: Node info in sync +49:40.572 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-cf819db4-3f52-d3fb-1f2a-85b0eff4b311 +49:40.572 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-cf819db4-3f52-d3fb-1f2a-85b0eff4b311 +49:40.573 [INFO] TestAgent: Requesting shutdown +49:40.573 [INFO] TestAgent.server: shutting down server +49:40.573 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:40.573 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:40.573 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:40.573 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:40.573 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:40.573 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:40.573 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:40.573 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:40.573 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:40.573 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:40.573 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:40.573 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:40.573 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:40.573 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:40.573 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:40.573Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:40.573Z [INFO] agent.router.manager: shutting down +49:40.573 [INFO] TestAgent: consul server down +49:40.573 [INFO] TestAgent: shutdown complete +49:40.573 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23626 network=tcp +49:40.573 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23626 network=udp +49:40.573 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23627 network=tcp +49:41.073 [INFO] TestAgent: Waiting for endpoints to shut down +49:41.073 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Legacy_Clone (0.82s) +=== CONT TestACL_Legacy_Destroy === RUN TestACL_HTTP/Policy === RUN TestACL_HTTP/Policy/Create === RUN TestACL_HTTP/Policy/Minimal @@ -48299,265 +52277,496 @@ === RUN TestACL_HTTP/Token/Create_with_Accessor_as_Secret_Dup === RUN TestACL_HTTP/Token/Create_with_Reserved_Accessor === RUN TestACL_HTTP/Token/Create_with_Reserved_Secret -=== RUN TestACL_Bootstrap/bootstrap -=== RUN TestACL_Bootstrap/not_again -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:34.590Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:34.590Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:34.640Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:34.640Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:34.650 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:73a7d6bb-4cfe-7a2e-02f9-ce036ac5e655 Address:127.0.0.1:17022}]" -13:34.651 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-73a7d6bb-4cfe-7a2e-02f9-ce036ac5e655.dc1 127.0.0.1 -13:34.652 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-73a7d6bb-4cfe-7a2e-02f9-ce036ac5e655 127.0.0.1 -2021-10-31T23:13:34.652Z [INFO] agent.router: Initializing LAN area manager -13:34.652 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16965 network=udp -13:34.652 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17022 [Follower]" leader= -13:34.652 [INFO] TestAgent.server: Adding LAN server: server="Node-73a7d6bb-4cfe-7a2e-02f9-ce036ac5e655 (Addr: tcp/127.0.0.1:17022) (DC: dc1)" -13:34.652 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-73a7d6bb-4cfe-7a2e-02f9-ce036ac5e655.dc1 area=wan -13:34.652 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16965 network=tcp -13:34.653 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16966 network=tcp -13:34.653 [INFO] TestAgent: started state syncer -13:34.653 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17023 network=tcp -13:34.710 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:34.717 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:34.717 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17022 [Candidate]" term=2 -13:34.718 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:34.718 [DEBUG] TestAgent.server.raft: vote granted: from=73a7d6bb-4cfe-7a2e-02f9-ce036ac5e655 term=2 tally=1 -13:34.718 [INFO] TestAgent.server.raft: election won: tally=1 -13:34.718 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17022 [Leader]" -13:34.718 [INFO] TestAgent.server: cluster leadership acquired -13:34.718 [INFO] TestAgent.server: initializing acls -13:34.719 [INFO] TestAgent.server: New leader elected: payload=Node-73a7d6bb-4cfe-7a2e-02f9-ce036ac5e655 -13:34.719 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:34.719 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:34.719 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:34.720 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:34.720 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:34.720 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:34.720 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-73a7d6bb-4cfe-7a2e-02f9-ce036ac5e655 -13:34.720 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-73a7d6bb-4cfe-7a2e-02f9-ce036ac5e655.dc1 -13:34.721 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:34.721 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-73a7d6bb-4cfe-7a2e-02f9-ce036ac5e655.dc1 area=wan -13:34.722 [INFO] TestAgent.server: Updating LAN server: server="Node-73a7d6bb-4cfe-7a2e-02f9-ce036ac5e655 (Addr: tcp/127.0.0.1:17022) (DC: dc1)" -13:34.744 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:34.744 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:34.744 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:34.744 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:34.744 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:34.744 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-73a7d6bb-4cfe-7a2e-02f9-ce036ac5e655 -13:34.744 [INFO] TestAgent.server: member joined, marking health alive: member=Node-73a7d6bb-4cfe-7a2e-02f9-ce036ac5e655 -13:34.745 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-73a7d6bb-4cfe-7a2e-02f9-ce036ac5e655 -13:34.755 [INFO] TestAgent.server: federation state anti-entropy synced -13:34.778 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-73a7d6bb-4cfe-7a2e-02f9-ce036ac5e655 -13:34.874 [INFO] TestAgent: Requesting shutdown -13:34.874 [INFO] TestAgent.server: shutting down server -13:34.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:34.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:34.874 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:34.874 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:34.874 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:34.874 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:34.874 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:34.874 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:34.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:34.874 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:34.874 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:34.874 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:34.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:34.874 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:34.874 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:34.874 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:34.874Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:34.874Z [INFO] agent.router.manager: shutting down -13:34.874 [INFO] TestAgent: consul server down -13:34.874 [INFO] TestAgent: shutdown complete -13:34.874 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16965 network=tcp -13:34.874 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16965 network=udp -13:34.874 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16966 network=tcp -13:35.375 [INFO] TestAgent: Waiting for endpoints to shut down -13:35.375 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_LoginProcedure_HTTP (0.87s) - --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod (0.05s) +2022-12-04T05:49:40.272Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:40.285Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:40.287 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:22bb554c-013c-b78e-1df5-549403c8c6b4 Address:127.0.0.1:23568}]" +49:40.287 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23568 [Follower]" leader= +49:40.287 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-22bb554c-013c-b78e-1df5-549403c8c6b4.dc1 127.0.0.1 +49:40.288 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-22bb554c-013c-b78e-1df5-549403c8c6b4 127.0.0.1 +2022-12-04T05:49:40.288Z [INFO] agent.router: Initializing LAN area manager +49:40.288 [INFO] TestAgent.server: Adding LAN server: server="Node-22bb554c-013c-b78e-1df5-549403c8c6b4 (Addr: tcp/127.0.0.1:23568) (DC: dc1)" +49:40.288 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-22bb554c-013c-b78e-1df5-549403c8c6b4.dc1 area=wan +49:40.288 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23640 network=udp +49:40.288 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23640 network=tcp +49:40.289 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23641 network=tcp +49:40.289 [INFO] TestAgent: started state syncer +49:40.289 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23597 network=tcp +49:40.338 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:40.347 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:40.347 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23568 [Candidate]" term=2 +49:40.347 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:40.347 [DEBUG] TestAgent.server.raft: vote granted: from=22bb554c-013c-b78e-1df5-549403c8c6b4 term=2 tally=1 +49:40.347 [INFO] TestAgent.server.raft: election won: tally=1 +49:40.347 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23568 [Leader]" +49:40.347 [INFO] TestAgent.server: cluster leadership acquired +49:40.347 [INFO] TestAgent.server: initializing acls +49:40.347 [INFO] TestAgent.server: New leader elected: payload=Node-22bb554c-013c-b78e-1df5-549403c8c6b4 +49:40.347 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:40.348 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:40.348 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:40.348 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:40.348 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:40.348 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-22bb554c-013c-b78e-1df5-549403c8c6b4 +49:40.348 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-22bb554c-013c-b78e-1df5-549403c8c6b4.dc1 +49:40.348 [INFO] TestAgent.server: Updating LAN server: server="Node-22bb554c-013c-b78e-1df5-549403c8c6b4 (Addr: tcp/127.0.0.1:23568) (DC: dc1)" +49:40.348 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-22bb554c-013c-b78e-1df5-549403c8c6b4.dc1 area=wan +49:40.349 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:40.350 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:40.350 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:40.350 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:40.350 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:40.350 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:40.350 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-22bb554c-013c-b78e-1df5-549403c8c6b4 +49:40.350 [INFO] TestAgent.server: member joined, marking health alive: member=Node-22bb554c-013c-b78e-1df5-549403c8c6b4 +49:40.350 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-22bb554c-013c-b78e-1df5-549403c8c6b4 +49:40.350 [INFO] TestAgent.server: federation state anti-entropy synced +49:40.461 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:40.461 [INFO] TestAgent: Synced node info +49:40.573 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-22bb554c-013c-b78e-1df5-549403c8c6b4 +49:40.573 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-22bb554c-013c-b78e-1df5-549403c8c6b4 +49:40.606 [INFO] TestAgent: Requesting shutdown +49:40.606 [INFO] TestAgent.server: shutting down server +49:40.606 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:40.606 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:40.606 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:40.606 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:40.606 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:40.606 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:40.606 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:40.606 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:40.606 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:40.606 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:40.606 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:40.606 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:40.606 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:40.606 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:40.606 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:40.606Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:40.606Z [INFO] agent.router.manager: shutting down +49:40.606 [INFO] TestAgent: consul server down +49:40.606 [INFO] TestAgent: shutdown complete +49:40.606 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23640 network=tcp +49:40.606 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23640 network=udp +49:40.606 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23641 network=tcp +49:41.107 [INFO] TestAgent: Waiting for endpoints to shut down +49:41.107 [INFO] TestAgent: Endpoints down +--- PASS: TestACLEndpoint_LoginLogout_jwt (0.85s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys (0.00s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/invalid_bearer_token (0.00s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/valid_bearer_token_no_bindings (0.00s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_static_keys/valid_bearer_token_1_service_binding (0.00s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks (0.01s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/invalid_bearer_token (0.00s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/valid_bearer_token_no_bindings (0.00s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_jwks/valid_bearer_token_1_service_binding (0.00s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery (0.02s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/invalid_bearer_token (0.01s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/valid_bearer_token_no_bindings (0.00s) + --- PASS: TestACLEndpoint_LoginLogout_jwt/success_-_jwt_oidc_discovery/valid_bearer_token_1_service_binding (0.00s) +=== CONT TestACL_Legacy_Update +[WARN] freeport: 10 out of 25 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:40.638Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:40.654Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:40.656 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4bfd0960-7206-489f-d780-91cece2344a5 Address:127.0.0.1:23681}]" +49:40.656 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23681 [Follower]" leader= +49:40.656 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4bfd0960-7206-489f-d780-91cece2344a5.dc1 127.0.0.1 +49:40.657 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4bfd0960-7206-489f-d780-91cece2344a5 127.0.0.1 +2022-12-04T05:49:40.657Z [INFO] agent.router: Initializing LAN area manager +49:40.657 [INFO] TestAgent.server: Adding LAN server: server="Node-4bfd0960-7206-489f-d780-91cece2344a5 (Addr: tcp/127.0.0.1:23681) (DC: dc1)" +49:40.657 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4bfd0960-7206-489f-d780-91cece2344a5.dc1 area=wan +49:40.657 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23599 network=tcp +49:40.657 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23599 network=udp +49:40.658 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23600 network=tcp +49:40.658 [INFO] TestAgent: started state syncer +49:40.658 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23682 network=tcp +49:40.720 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:40.720 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23681 [Candidate]" term=2 +49:40.720 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:40.720 [DEBUG] TestAgent.server.raft: vote granted: from=4bfd0960-7206-489f-d780-91cece2344a5 term=2 tally=1 +49:40.720 [INFO] TestAgent.server.raft: election won: tally=1 +49:40.720 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23681 [Leader]" +49:40.720 [INFO] TestAgent.server: cluster leadership acquired +49:40.720 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23681 +49:40.721 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:40.721 [INFO] TestAgent.server: New leader elected: payload=Node-4bfd0960-7206-489f-d780-91cece2344a5 +49:40.722 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:40.722 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:40.722 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:40.722 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:40.722 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:40.722 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4bfd0960-7206-489f-d780-91cece2344a5 +49:40.722 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4bfd0960-7206-489f-d780-91cece2344a5 +49:40.723 [INFO] TestAgent.server: federation state anti-entropy synced +49:40.867 [INFO] TestAgent: Requesting shutdown +49:40.868 [INFO] TestAgent.server: shutting down server +49:40.868 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:40.868 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:40.868 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:40.868 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:40.868 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:40.868 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:40.868 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:40.868 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:40.868 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:40.868 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:40.868 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:40.868 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:40.868Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:40.868Z [INFO] agent.router.manager: shutting down +49:40.868 [INFO] TestAgent: consul server down +49:40.868 [INFO] TestAgent: shutdown complete +49:40.868 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23599 network=tcp +49:40.868 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23599 network=udp +49:40.868 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23600 network=tcp +49:41.368 [INFO] TestAgent: Waiting for endpoints to shut down +49:41.368 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Disabled_Response (0.75s) + --- PASS: TestACL_Disabled_Response/ACLBootstrap (0.00s) + --- PASS: TestACL_Disabled_Response/ACLReplicationStatus (0.00s) + --- PASS: TestACL_Disabled_Response/AgentToken (0.00s) + --- PASS: TestACL_Disabled_Response/ACLRulesTranslate (0.00s) + --- PASS: TestACL_Disabled_Response/ACLRulesTranslateLegacyToken (0.00s) + --- PASS: TestACL_Disabled_Response/ACLPolicyList (0.00s) + --- PASS: TestACL_Disabled_Response/ACLPolicyCRUD (0.00s) + --- PASS: TestACL_Disabled_Response/ACLPolicyCreate (0.00s) + --- PASS: TestACL_Disabled_Response/ACLTokenList (0.00s) + --- PASS: TestACL_Disabled_Response/ACLTokenCreate (0.00s) + --- PASS: TestACL_Disabled_Response/ACLTokenSelf (0.00s) + --- PASS: TestACL_Disabled_Response/ACLTokenCRUD (0.00s) + --- PASS: TestACL_Disabled_Response/ACLRoleList (0.00s) + --- PASS: TestACL_Disabled_Response/ACLRoleCreate (0.00s) + --- PASS: TestACL_Disabled_Response/ACLRoleCRUD (0.00s) + --- PASS: TestACL_Disabled_Response/ACLBindingRuleList (0.00s) + --- PASS: TestACL_Disabled_Response/ACLBindingRuleCreate (0.00s) + --- PASS: TestACL_Disabled_Response/ACLBindingRuleCRUD (0.00s) + --- PASS: TestACL_Disabled_Response/ACLAuthMethodList (0.00s) + --- PASS: TestACL_Disabled_Response/ACLAuthMethodCreate (0.00s) + --- PASS: TestACL_Disabled_Response/ACLAuthMethodCRUD (0.00s) + --- PASS: TestACL_Disabled_Response/ACLLogin (0.00s) + --- PASS: TestACL_Disabled_Response/ACLLogout (0.00s) + --- PASS: TestACL_Disabled_Response/ACLAuthorize (0.00s) +=== CONT TestUiServices/No_Filter +2022-12-04T05:49:15.624Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:15.636Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:15.638 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:04e00d83-3286-c5e7-a595-94fd4e546f4c Address:127.0.0.1:23519}]" +49:15.638 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23519 [Follower]" leader= +49:15.638 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-04e00d83-3286-c5e7-a595-94fd4e546f4c.dc1 127.0.0.1 +49:15.639 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-04e00d83-3286-c5e7-a595-94fd4e546f4c 127.0.0.1 +2022-12-04T05:49:15.639Z [INFO] agent.router: Initializing LAN area manager +49:15.639 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23514 network=udp +49:15.639 [INFO] TestAgent.server: Adding LAN server: server="Node-04e00d83-3286-c5e7-a595-94fd4e546f4c (Addr: tcp/127.0.0.1:23519) (DC: dc1)" +49:15.639 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-04e00d83-3286-c5e7-a595-94fd4e546f4c.dc1 area=wan +49:15.639 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23514 network=tcp +49:15.640 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23515 network=tcp +49:15.640 [INFO] TestAgent: started state syncer +49:15.640 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23520 network=tcp +49:15.682 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:15.682 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23519 [Candidate]" term=2 +49:15.682 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:15.682 [DEBUG] TestAgent.server.raft: vote granted: from=04e00d83-3286-c5e7-a595-94fd4e546f4c term=2 tally=1 +49:15.682 [INFO] TestAgent.server.raft: election won: tally=1 +49:15.682 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23519 [Leader]" +49:15.682 [INFO] TestAgent.server: cluster leadership acquired +49:15.682 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23519 +49:15.683 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:15.684 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:15.684 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:15.684 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:15.684 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:15.684 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:15.684 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-04e00d83-3286-c5e7-a595-94fd4e546f4c +49:15.684 [INFO] TestAgent.server: member joined, marking health alive: member=Node-04e00d83-3286-c5e7-a595-94fd4e546f4c +49:15.684 [INFO] TestAgent.server: New leader elected: payload=Node-04e00d83-3286-c5e7-a595-94fd4e546f4c +49:15.685 [INFO] TestAgent.server: federation state anti-entropy synced +49:15.836 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:15.836 [INFO] TestAgent: Synced node info +49:16.019 [INFO] TestAgent: Requesting shutdown +49:16.019 [INFO] TestAgent.server: shutting down server +49:16.019 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:16.019 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:16.019 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:16.019 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:16.019 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:16.019 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:16.019 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:16.019 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:16.019 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:16.019 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:16.019 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:16.019Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:16.019Z [INFO] agent.router.manager: shutting down +49:16.019 [INFO] TestAgent: consul server down +49:16.019 [INFO] TestAgent: shutdown complete +49:16.019 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23514 network=tcp +49:16.019 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23514 network=udp +49:16.019 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23515 network=tcp +49:16.520 [INFO] TestAgent: Waiting for endpoints to shut down +49:16.520 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{0_0_0_0_0_0_0_0_0_0_0} +--- PASS: TestUiServices (0.91s) + --- PASS: TestUiServices/Filtered (0.01s) + --- PASS: TestUiServices/No_Filter (0.00s) +2022-12-04T05:49:40.779Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.779Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:40.792Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.792Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:40.794 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:90708dc6-26aa-ea0d-3716-c351d6a2cd3d Address:127.0.0.1:23653}]" +49:40.794 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23653 [Follower]" leader= +49:40.795 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-90708dc6-26aa-ea0d-3716-c351d6a2cd3d.dc1 127.0.0.1 +49:40.795 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-90708dc6-26aa-ea0d-3716-c351d6a2cd3d 127.0.0.1 +2022-12-04T05:49:40.795Z [INFO] agent.router: Initializing LAN area manager +49:40.796 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23592 network=udp +49:40.796 [INFO] TestAgent.server: Adding LAN server: server="Node-90708dc6-26aa-ea0d-3716-c351d6a2cd3d (Addr: tcp/127.0.0.1:23653) (DC: dc1)" +49:40.796 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-90708dc6-26aa-ea0d-3716-c351d6a2cd3d.dc1 area=wan +49:40.796 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23592 network=tcp +49:40.796 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23593 network=tcp +49:40.796 [INFO] TestAgent: started state syncer +49:40.796 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23661 network=tcp +49:40.846 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:40.847 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:40.848 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23653 [Candidate]" term=2 +49:40.848 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:40.848 [DEBUG] TestAgent.server.raft: vote granted: from=90708dc6-26aa-ea0d-3716-c351d6a2cd3d term=2 tally=1 +49:40.848 [INFO] TestAgent.server.raft: election won: tally=1 +49:40.848 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23653 [Leader]" +49:40.848 [INFO] TestAgent.server: cluster leadership acquired +49:40.848 [INFO] TestAgent.server: initializing acls +49:40.848 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:40.848 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:40.849 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:40.849 [INFO] TestAgent.server: New leader elected: payload=Node-90708dc6-26aa-ea0d-3716-c351d6a2cd3d +49:40.849 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:40.849 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:40.849 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:40.849 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-90708dc6-26aa-ea0d-3716-c351d6a2cd3d +49:40.849 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-90708dc6-26aa-ea0d-3716-c351d6a2cd3d.dc1 +49:40.850 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:40.850 [INFO] TestAgent.server: Updating LAN server: server="Node-90708dc6-26aa-ea0d-3716-c351d6a2cd3d (Addr: tcp/127.0.0.1:23653) (DC: dc1)" +49:40.850 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-90708dc6-26aa-ea0d-3716-c351d6a2cd3d.dc1 area=wan +49:40.851 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:40.851 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:40.851 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:40.851 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:40.851 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:40.851 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-90708dc6-26aa-ea0d-3716-c351d6a2cd3d +49:40.851 [INFO] TestAgent.server: member joined, marking health alive: member=Node-90708dc6-26aa-ea0d-3716-c351d6a2cd3d +49:40.852 [INFO] TestAgent.server: federation state anti-entropy synced +49:40.852 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-90708dc6-26aa-ea0d-3716-c351d6a2cd3d +49:40.870 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-90708dc6-26aa-ea0d-3716-c351d6a2cd3d +49:40.870 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-90708dc6-26aa-ea0d-3716-c351d6a2cd3d +49:40.870 [INFO] TestAgent: Requesting shutdown +49:40.870 [INFO] TestAgent.server: shutting down server +49:40.870 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:40.870 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:40.870 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:40.870 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:40.870 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:40.870 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:40.870 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:40.870 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:40.870 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:40.870 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:40.870 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:40.870 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:40.870 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:40.870 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:40.870 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:40.870 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:40.870Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:40.870Z [INFO] agent.router.manager: shutting down +49:40.870 [INFO] TestAgent: consul server down +49:40.870 [INFO] TestAgent: shutdown complete +49:40.870 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23592 network=tcp +49:40.870 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23592 network=udp +49:40.870 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23593 network=tcp +49:41.370 [INFO] TestAgent: Waiting for endpoints to shut down +49:41.370 [INFO] TestAgent: Endpoints down +--- PASS: TestACLReplicationStatus (0.60s) +=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{6_6_6_6_6_6_6_5_6_6_-5} +[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:40.682Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.682Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:40.694Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.694Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:40.696 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6504bedc-592e-4fb0-e0ee-17d25d60f0db Address:127.0.0.1:23569}]" +49:40.696 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23569 [Follower]" leader= +49:40.696 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6504bedc-592e-4fb0-e0ee-17d25d60f0db.dc1 127.0.0.1 +49:40.697 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6504bedc-592e-4fb0-e0ee-17d25d60f0db 127.0.0.1 +2022-12-04T05:49:40.697Z [INFO] agent.router: Initializing LAN area manager +49:40.697 [INFO] TestAgent.server: Adding LAN server: server="Node-6504bedc-592e-4fb0-e0ee-17d25d60f0db (Addr: tcp/127.0.0.1:23569) (DC: dc1)" +49:40.697 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6504bedc-592e-4fb0-e0ee-17d25d60f0db.dc1 area=wan +49:40.697 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23683 network=udp +49:40.697 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23683 network=tcp +49:40.698 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23684 network=tcp +49:40.698 [INFO] TestAgent: started state syncer +49:40.698 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23591 network=tcp +49:40.747 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:40.764 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:40.764 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23569 [Candidate]" term=2 +49:40.765 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:40.765 [DEBUG] TestAgent.server.raft: vote granted: from=6504bedc-592e-4fb0-e0ee-17d25d60f0db term=2 tally=1 +49:40.765 [INFO] TestAgent.server.raft: election won: tally=1 +49:40.765 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23569 [Leader]" +49:40.765 [INFO] TestAgent.server: cluster leadership acquired +49:40.765 [INFO] TestAgent.server: initializing acls +49:40.765 [INFO] TestAgent.server: New leader elected: payload=Node-6504bedc-592e-4fb0-e0ee-17d25d60f0db +49:40.765 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:40.765 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:40.766 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:40.766 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:40.766 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:40.766 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:40.766 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-6504bedc-592e-4fb0-e0ee-17d25d60f0db +49:40.766 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-6504bedc-592e-4fb0-e0ee-17d25d60f0db.dc1 +49:40.766 [INFO] TestAgent.server: Updating LAN server: server="Node-6504bedc-592e-4fb0-e0ee-17d25d60f0db (Addr: tcp/127.0.0.1:23569) (DC: dc1)" +49:40.766 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-6504bedc-592e-4fb0-e0ee-17d25d60f0db.dc1 area=wan +49:40.767 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:40.768 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:40.768 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:40.768 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:40.768 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:40.768 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:40.768 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6504bedc-592e-4fb0-e0ee-17d25d60f0db +49:40.768 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6504bedc-592e-4fb0-e0ee-17d25d60f0db +49:40.769 [INFO] TestAgent.server: federation state anti-entropy synced +49:40.769 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6504bedc-592e-4fb0-e0ee-17d25d60f0db +49:40.819 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:40.819 [INFO] TestAgent: Synced node info +49:40.819 [DEBUG] TestAgent: Node info in sync +49:41.047 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-6504bedc-592e-4fb0-e0ee-17d25d60f0db +49:41.047 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-6504bedc-592e-4fb0-e0ee-17d25d60f0db +49:41.055 [INFO] TestAgent: Requesting shutdown +49:41.055 [INFO] TestAgent.server: shutting down server +49:41.055 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:41.055 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:41.055 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:41.055 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:41.055 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:41.055 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:41.055 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:41.055 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:41.055 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:41.055 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:41.055 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:41.055 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:41.055 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:41.055 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:41.055 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:41.055Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:41.055Z [INFO] agent.router.manager: shutting down +49:41.056 [INFO] TestAgent: consul server down +49:41.056 [INFO] TestAgent: shutdown complete +49:41.056 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23683 network=tcp +49:41.056 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23683 network=udp +49:41.056 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23684 network=tcp +49:41.556 [INFO] TestAgent: Waiting for endpoints to shut down +49:41.556 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_LoginProcedure_HTTP (0.89s) + --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/Create (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/Create_other (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/Update_Name_URL_Mismatch (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/Update (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/Invalid_payload (0.00s) - --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/List (0.04s) - --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/Delete (0.01s) + --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/List (0.00s) + --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/Delete (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/AuthMethod/Read (0.00s) - --- PASS: TestACL_LoginProcedure_HTTP/BindingRule (0.03s) + --- PASS: TestACL_LoginProcedure_HTTP/BindingRule (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/BindingRule/Create (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/BindingRule/Create_other (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/BindingRule/BindingRule_CRUD_Missing_ID_in_URL (0.00s) - --- PASS: TestACL_LoginProcedure_HTTP/BindingRule/Update (0.03s) + --- PASS: TestACL_LoginProcedure_HTTP/BindingRule/Update (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/BindingRule/ID_Supplied (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/BindingRule/Invalid_payload (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/BindingRule/List (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/BindingRule/Delete (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/BindingRule/Read (0.00s) - --- PASS: TestACL_LoginProcedure_HTTP/Login (0.01s) - --- PASS: TestACL_LoginProcedure_HTTP/Login/Create_Token_1 (0.01s) + --- PASS: TestACL_LoginProcedure_HTTP/Login (0.00s) + --- PASS: TestACL_LoginProcedure_HTTP/Login/Create_Token_1 (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/Login/Create_Token_2 (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/Login/List_Tokens_by_(incorrect)_Method (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/Login/List_Tokens_by_(correct)_Method (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/Login/Logout (0.00s) --- PASS: TestACL_LoginProcedure_HTTP/Login/Token_is_gone_after_Logout (0.00s) -=== CONT TestSessionDeleteDestroy -=== RUN TestACL_Disabled_Response/ACLBootstrap -=== RUN TestACL_Disabled_Response/ACLReplicationStatus -=== RUN TestACL_Disabled_Response/AgentToken -=== RUN TestACL_Disabled_Response/ACLRulesTranslate -=== RUN TestACL_Disabled_Response/ACLRulesTranslateLegacyToken -=== RUN TestACL_Disabled_Response/ACLPolicyList -=== RUN TestACL_Disabled_Response/ACLPolicyCRUD -=== RUN TestACL_Disabled_Response/ACLPolicyCreate -=== RUN TestACL_Disabled_Response/ACLTokenList -=== RUN TestACL_Disabled_Response/ACLTokenCreate -=== RUN TestACL_Disabled_Response/ACLTokenSelf -=== RUN TestACL_Disabled_Response/ACLTokenCRUD -=== RUN TestACL_Disabled_Response/ACLRoleList -=== RUN TestACL_Disabled_Response/ACLRoleCreate -=== RUN TestACL_Disabled_Response/ACLRoleCRUD -=== RUN TestACL_Disabled_Response/ACLBindingRuleList -=== RUN TestACL_Disabled_Response/ACLBindingRuleCreate -=== RUN TestACL_Disabled_Response/ACLBindingRuleCRUD -=== RUN TestACL_Disabled_Response/ACLAuthMethodList -=== RUN TestACL_Disabled_Response/ACLAuthMethodCreate -=== RUN TestACL_Disabled_Response/ACLAuthMethodCRUD -=== RUN TestACL_Disabled_Response/ACLLogin -=== RUN TestACL_Disabled_Response/ACLLogout -=== RUN TestACL_Disabled_Response/ACLAuthorize -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=33)_compressed=true -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:35.082Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:35.145Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:35.159 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:96919165-7352-e6eb-73f7-1930bcf71cb1 Address:127.0.0.1:17064}]" -13:35.160 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17064 [Follower]" leader= -13:35.160 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-96919165-7352-e6eb-73f7-1930bcf71cb1.dc1 127.0.0.1 -13:35.208 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-96919165-7352-e6eb-73f7-1930bcf71cb1 127.0.0.1 -2021-10-31T23:13:35.208Z [INFO] agent.router: Initializing LAN area manager -13:35.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17073 network=udp -13:35.209 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-96919165-7352-e6eb-73f7-1930bcf71cb1.dc1 area=wan -13:35.209 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17073 network=tcp -13:35.209 [INFO] TestAgent.server: Adding LAN server: server="Node-96919165-7352-e6eb-73f7-1930bcf71cb1 (Addr: tcp/127.0.0.1:17064) (DC: dc1)" -13:35.210 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:35.210 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17064 [Candidate]" term=2 -13:35.210 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:35.210 [DEBUG] TestAgent.server.raft: vote granted: from=96919165-7352-e6eb-73f7-1930bcf71cb1 term=2 tally=1 -13:35.210 [INFO] TestAgent.server.raft: election won: tally=1 -13:35.210 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17064 [Leader]" -13:35.210 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17074 network=tcp -13:35.210 [INFO] TestAgent.server: cluster leadership acquired -13:35.210 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17065 network=tcp -13:35.210 [INFO] TestAgent: started state syncer -13:35.210 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17064 -13:35.210 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:35.211 [INFO] TestAgent.server: New leader elected: payload=Node-96919165-7352-e6eb-73f7-1930bcf71cb1 -13:35.211 [INFO] TestAgent: Requesting shutdown -13:35.211 [INFO] TestAgent.server: shutting down server -13:35.211 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:35.211 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:35.211Z [INFO] agent.router.manager: shutting down -13:35.211 [ERROR] TestAgent.server: failed to establish leadership: error="raft is already shutdown" -2021-10-31T23:13:35.211Z [INFO] agent.router.manager: shutting down -13:35.211 [INFO] TestAgent: Synced node info -13:35.211 [DEBUG] TestAgent: Node info in sync -13:35.211 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=0 retry_limit=3 error="raft is already shutdown" -13:35.211 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=1 retry_limit=3 error="raft is already shutdown" -13:35.211 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=2 retry_limit=3 error="raft is already shutdown" -13:35.211 [ERROR] TestAgent.server: failed to transfer leadership: error="failed to transfer leadership in 3 attempts" -13:35.211 [INFO] TestAgent: consul server down -13:35.211 [INFO] TestAgent: shutdown complete -13:35.211 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17073 network=tcp -13:35.211 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17073 network=udp -13:35.211 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17074 network=tcp -13:35.713 [INFO] TestAgent: Waiting for endpoints to shut down -13:35.713 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_List (1.65s) - --- PASS: TestPreparedQuery_List/#00 (0.93s) - --- PASS: TestPreparedQuery_List/#01 (0.72s) -=== CONT TestTxnEndpoint_Bad_Size_Item -=== RUN TestTxnEndpoint_Bad_Size_Item/exceeds_default_limits -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=33)_compressed=false -2021-10-31T23:13:34.745Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:34.745Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:34.825Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:34.825Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:34.860 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4d53be39-019e-a323-2532-29ca1c4e75e0 Address:127.0.0.1:17015}]" -13:34.860 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4d53be39-019e-a323-2532-29ca1c4e75e0.dc1 127.0.0.1 -13:34.861 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4d53be39-019e-a323-2532-29ca1c4e75e0 127.0.0.1 -2021-10-31T23:13:34.861Z [INFO] agent.router: Initializing LAN area manager -13:34.861 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17024 network=udp -13:34.862 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17015 [Follower]" leader= -13:34.862 [INFO] TestAgent.server: Adding LAN server: server="Node-4d53be39-019e-a323-2532-29ca1c4e75e0 (Addr: tcp/127.0.0.1:17015) (DC: dc1)" -13:34.862 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4d53be39-019e-a323-2532-29ca1c4e75e0.dc1 area=wan -13:34.863 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17024 network=tcp -13:34.863 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17025 network=tcp -13:34.863 [INFO] TestAgent: started state syncer -13:34.863 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17016 network=tcp -13:34.904 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:34.904 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17015 [Candidate]" term=2 -13:34.905 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:34.905 [DEBUG] TestAgent.server.raft: vote granted: from=4d53be39-019e-a323-2532-29ca1c4e75e0 term=2 tally=1 -13:34.905 [INFO] TestAgent.server.raft: election won: tally=1 -13:34.905 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17015 [Leader]" -13:34.905 [INFO] TestAgent.server: cluster leadership acquired -13:34.905 [INFO] TestAgent.server: initializing acls -13:34.905 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:34.905 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:34.906 [INFO] TestAgent.server: New leader elected: payload=Node-4d53be39-019e-a323-2532-29ca1c4e75e0 -13:34.906 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:34.906 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:34.906 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:34.906 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:34.906 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4d53be39-019e-a323-2532-29ca1c4e75e0 -13:34.906 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4d53be39-019e-a323-2532-29ca1c4e75e0.dc1 -13:34.907 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:34.909 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:34.909 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:34.909 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:34.909 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:34.909 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:34.909 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4d53be39-019e-a323-2532-29ca1c4e75e0 -13:34.909 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4d53be39-019e-a323-2532-29ca1c4e75e0 -13:34.910 [INFO] TestAgent.server: Updating LAN server: server="Node-4d53be39-019e-a323-2532-29ca1c4e75e0 (Addr: tcp/127.0.0.1:17015) (DC: dc1)" -13:34.910 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4d53be39-019e-a323-2532-29ca1c4e75e0.dc1 area=wan -13:34.918 [INFO] TestAgent.server: federation state anti-entropy synced -13:34.918 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4d53be39-019e-a323-2532-29ca1c4e75e0 -13:34.957 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:34.957 [INFO] TestAgent: Synced node info -13:34.957 [DEBUG] TestAgent: Node info in sync -13:35.218 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4d53be39-019e-a323-2532-29ca1c4e75e0 -13:35.218 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4d53be39-019e-a323-2532-29ca1c4e75e0 -13:35.272 [INFO] TestAgent: Requesting shutdown -13:35.272 [INFO] TestAgent.server: shutting down server -13:35.272 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:35.272 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:35.272 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:35.272 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:35.272 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:35.272 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:35.273 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:35.273 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:35.273 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:35.273 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:35.273 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:35.273 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:35.273 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:35.273 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:35.273 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:35.273Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:35.273Z [INFO] agent.router.manager: shutting down -13:35.273 [INFO] TestAgent: consul server down -13:35.273 [INFO] TestAgent: shutdown complete -13:35.273 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17024 network=tcp -13:35.273 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17024 network=udp -13:35.273 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17025 network=tcp -13:35.774 [INFO] TestAgent: Waiting for endpoints to shut down -13:35.774 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_HTTP (1.10s) - --- PASS: TestACL_HTTP/Policy (0.01s) +=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{5_5_5_5_5_5_5_5_5_5_5} +2022-12-04T05:49:40.792Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.792Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:40.802Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.802Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:40.803 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4ca0e584-9941-49f4-541c-95656430cad4 Address:127.0.0.1:23667}]" +49:40.803 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23667 [Follower]" leader= +49:40.803 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4ca0e584-9941-49f4-541c-95656430cad4.dc1 127.0.0.1 +49:40.804 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4ca0e584-9941-49f4-541c-95656430cad4 127.0.0.1 +2022-12-04T05:49:40.804Z [INFO] agent.router: Initializing LAN area manager +49:40.804 [INFO] TestAgent.server: Adding LAN server: server="Node-4ca0e584-9941-49f4-541c-95656430cad4 (Addr: tcp/127.0.0.1:23667) (DC: dc1)" +49:40.804 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4ca0e584-9941-49f4-541c-95656430cad4.dc1 area=wan +49:40.804 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23662 network=udp +49:40.804 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23662 network=tcp +49:40.805 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23663 network=tcp +49:40.805 [INFO] TestAgent: started state syncer +49:40.805 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23633 network=tcp +49:40.841 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:40.841 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23667 [Candidate]" term=2 +49:40.841 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:40.841 [DEBUG] TestAgent.server.raft: vote granted: from=4ca0e584-9941-49f4-541c-95656430cad4 term=2 tally=1 +49:40.841 [INFO] TestAgent.server.raft: election won: tally=1 +49:40.841 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23667 [Leader]" +49:40.841 [INFO] TestAgent.server: cluster leadership acquired +49:40.841 [INFO] TestAgent.server: initializing acls +49:40.841 [INFO] TestAgent.server: New leader elected: payload=Node-4ca0e584-9941-49f4-541c-95656430cad4 +49:40.842 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:40.842 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:40.842 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:40.842 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:40.842 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:40.842 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:40.843 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4ca0e584-9941-49f4-541c-95656430cad4 +49:40.843 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4ca0e584-9941-49f4-541c-95656430cad4.dc1 +49:40.843 [INFO] TestAgent.server: Updating LAN server: server="Node-4ca0e584-9941-49f4-541c-95656430cad4 (Addr: tcp/127.0.0.1:23667) (DC: dc1)" +49:40.843 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4ca0e584-9941-49f4-541c-95656430cad4.dc1 area=wan +49:40.843 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:40.844 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:40.844 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:40.844 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:40.844 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:40.844 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:40.845 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4ca0e584-9941-49f4-541c-95656430cad4 +49:40.845 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4ca0e584-9941-49f4-541c-95656430cad4 +49:40.845 [INFO] TestAgent.server: federation state anti-entropy synced +49:40.845 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4ca0e584-9941-49f4-541c-95656430cad4 +49:40.846 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:40.846 [INFO] TestAgent: Synced node info +49:40.846 [DEBUG] TestAgent: Node info in sync +49:41.093 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4ca0e584-9941-49f4-541c-95656430cad4 +49:41.093 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-4ca0e584-9941-49f4-541c-95656430cad4 +49:41.106 [INFO] TestAgent: Requesting shutdown +49:41.106 [INFO] TestAgent.server: shutting down server +49:41.106 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:41.106 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:41.106 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:41.106 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:41.106 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:41.106 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:41.106 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:41.106 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:41.106 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:41.106 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:41.106 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:41.106 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:41.106 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:41.106 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:41.106 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:41.106Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:41.106Z [INFO] agent.router.manager: shutting down +49:41.106 [INFO] TestAgent: consul server down +49:41.106 [INFO] TestAgent: shutdown complete +49:41.106 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23662 network=tcp +49:41.106 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23662 network=udp +49:41.106 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23663 network=tcp +49:41.606 [INFO] TestAgent: Waiting for endpoints to shut down +49:41.606 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_HTTP (0.82s) + --- PASS: TestACL_HTTP/Policy (0.00s) --- PASS: TestACL_HTTP/Policy/Create (0.00s) --- PASS: TestACL_HTTP/Policy/Minimal (0.00s) --- PASS: TestACL_HTTP/Policy/Name_Chars (0.00s) @@ -48570,20 +52779,20 @@ --- PASS: TestACL_HTTP/Policy/List (0.00s) --- PASS: TestACL_HTTP/Policy/Read (0.00s) --- PASS: TestACL_HTTP/Policy/Read_Name (0.00s) - --- PASS: TestACL_HTTP/Role (0.01s) + --- PASS: TestACL_HTTP/Role (0.00s) --- PASS: TestACL_HTTP/Role/Create (0.00s) --- PASS: TestACL_HTTP/Role/Name_Chars (0.00s) --- PASS: TestACL_HTTP/Role/Update_Name_ID_Mismatch (0.00s) --- PASS: TestACL_HTTP/Role/Role_CRUD_Missing_ID_in_URL (0.00s) - --- PASS: TestACL_HTTP/Role/Update (0.01s) + --- PASS: TestACL_HTTP/Role/Update (0.00s) --- PASS: TestACL_HTTP/Role/ID_Supplied (0.00s) --- PASS: TestACL_HTTP/Role/Invalid_payload (0.00s) --- PASS: TestACL_HTTP/Role/Delete (0.00s) --- PASS: TestACL_HTTP/Role/List (0.00s) --- PASS: TestACL_HTTP/Role/Read (0.00s) - --- PASS: TestACL_HTTP/Token (0.04s) + --- PASS: TestACL_HTTP/Token (0.01s) --- PASS: TestACL_HTTP/Token/Create (0.00s) - --- PASS: TestACL_HTTP/Token/Create_Local (0.01s) + --- PASS: TestACL_HTTP/Token/Create_Local (0.00s) --- PASS: TestACL_HTTP/Token/Read (0.00s) --- PASS: TestACL_HTTP/Token/Self (0.00s) --- PASS: TestACL_HTTP/Token/Clone (0.00s) @@ -48595,7814 +52804,3731 @@ --- PASS: TestACL_HTTP/Token/List_by_Policy (0.00s) --- PASS: TestACL_HTTP/Token/Create_with_Accessor (0.00s) --- PASS: TestACL_HTTP/Token/Create_with_Secret (0.00s) - --- PASS: TestACL_HTTP/Token/Create_with_Accessor_and_Secret (0.01s) + --- PASS: TestACL_HTTP/Token/Create_with_Accessor_and_Secret (0.00s) --- PASS: TestACL_HTTP/Token/Create_with_Accessor_Dup (0.00s) --- PASS: TestACL_HTTP/Token/Create_with_Secret_as_Accessor_Dup (0.00s) --- PASS: TestACL_HTTP/Token/Create_with_Secret_Dup (0.00s) --- PASS: TestACL_HTTP/Token/Create_with_Accessor_as_Secret_Dup (0.00s) --- PASS: TestACL_HTTP/Token/Create_with_Reserved_Accessor (0.00s) --- PASS: TestACL_HTTP/Token/Create_with_Reserved_Secret (0.00s) -=== CONT TestTxnEndpoint_Bad_JSON -2021-10-31T23:13:34.927Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:34.927Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:35.000Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:35.000Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:35.038 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fe44001e-06c9-1c1b-6413-6006fd94abb4 Address:127.0.0.1:17036}]" -13:35.039 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17036 [Follower]" leader= -13:35.039 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fe44001e-06c9-1c1b-6413-6006fd94abb4.dc1 127.0.0.1 -13:35.040 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fe44001e-06c9-1c1b-6413-6006fd94abb4 127.0.0.1 -2021-10-31T23:13:35.040Z [INFO] agent.router: Initializing LAN area manager -13:35.040 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17017 network=udp -13:35.041 [INFO] TestAgent.server: Adding LAN server: server="Node-fe44001e-06c9-1c1b-6413-6006fd94abb4 (Addr: tcp/127.0.0.1:17036) (DC: dc1)" -13:35.041 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fe44001e-06c9-1c1b-6413-6006fd94abb4.dc1 area=wan -13:35.041 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17017 network=tcp -13:35.051 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17018 network=tcp -13:35.051 [INFO] TestAgent: started state syncer -13:35.051 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17037 network=tcp -13:35.086 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:35.086 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17036 [Candidate]" term=2 -13:35.086 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:35.086 [DEBUG] TestAgent.server.raft: vote granted: from=fe44001e-06c9-1c1b-6413-6006fd94abb4 term=2 tally=1 -13:35.086 [INFO] TestAgent.server.raft: election won: tally=1 -13:35.086 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17036 [Leader]" -13:35.086 [INFO] TestAgent.server: cluster leadership acquired -13:35.087 [INFO] TestAgent.server: initializing acls -13:35.087 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:35.088 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:35.088 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:35.088 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:35.088 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-fe44001e-06c9-1c1b-6413-6006fd94abb4 -13:35.088 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-fe44001e-06c9-1c1b-6413-6006fd94abb4.dc1 -13:35.089 [INFO] TestAgent.server: New leader elected: payload=Node-fe44001e-06c9-1c1b-6413-6006fd94abb4 -13:35.089 [INFO] TestAgent.server: Updating LAN server: server="Node-fe44001e-06c9-1c1b-6413-6006fd94abb4 (Addr: tcp/127.0.0.1:17036) (DC: dc1)" -13:35.089 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-fe44001e-06c9-1c1b-6413-6006fd94abb4.dc1 area=wan -13:35.089 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:35.092 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:35.092 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:35.092 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:35.092 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:35.092 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:35.092 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe44001e-06c9-1c1b-6413-6006fd94abb4 -13:35.092 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fe44001e-06c9-1c1b-6413-6006fd94abb4 -13:35.093 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe44001e-06c9-1c1b-6413-6006fd94abb4 -13:35.093 [INFO] TestAgent.server: federation state anti-entropy synced -13:35.154 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="ACL not found" -13:35.306 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fe44001e-06c9-1c1b-6413-6006fd94abb4 -13:35.306 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fe44001e-06c9-1c1b-6413-6006fd94abb4 -13:35.306 [WARN] TestAgent.server.acl: failed to remove bootstrap file: error="remove /tmp/consul-test/TestACL_Bootstrap-TestAgent-agent593180907/acl-bootstrap-reset: no such file or directory" -13:35.307 [INFO] TestAgent.server.acl: ACL bootstrap completed -13:35.308 [INFO] TestAgent: Requesting shutdown -13:35.308 [INFO] TestAgent.server: shutting down server -13:35.308 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:35.308 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:35.308 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:35.308 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:35.308 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:35.308 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:35.308 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:35.308 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:35.308 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:35.308 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:35.308 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:35.308 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:35.308 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:35.308 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:35.308 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:35.309Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:35.309Z [INFO] agent.router.manager: shutting down -13:35.309 [INFO] TestAgent: consul server down -13:35.309 [INFO] TestAgent: shutdown complete -13:35.309 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17017 network=tcp -13:35.309 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17017 network=udp -13:35.309 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17018 network=tcp -13:35.809 [INFO] TestAgent: Waiting for endpoints to shut down -13:35.809 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:34.981Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:34.981Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:35.049Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:35.049Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:35.061 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:afc487a7-07b3-5841-8e8f-6ed52ed082c5 Address:127.0.0.1:17043}]" -13:35.061 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17043 [Follower]" leader= -13:35.078 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-afc487a7-07b3-5841-8e8f-6ed52ed082c5.dc1 127.0.0.1 -13:35.079 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-afc487a7-07b3-5841-8e8f-6ed52ed082c5 127.0.0.1 -2021-10-31T23:13:35.079Z [INFO] agent.router: Initializing LAN area manager -13:35.079 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17031 network=udp -13:35.080 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-afc487a7-07b3-5841-8e8f-6ed52ed082c5.dc1 area=wan -13:35.080 [INFO] TestAgent.server: Adding LAN server: server="Node-afc487a7-07b3-5841-8e8f-6ed52ed082c5 (Addr: tcp/127.0.0.1:17043) (DC: dc1)" -13:35.080 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17031 network=tcp -13:35.081 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17032 network=tcp -13:35.081 [INFO] TestAgent: started state syncer -13:35.081 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17044 network=tcp -13:35.130 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:35.130 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:35.130 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17043 [Candidate]" term=2 -13:35.130 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:35.130 [DEBUG] TestAgent.server.raft: vote granted: from=afc487a7-07b3-5841-8e8f-6ed52ed082c5 term=2 tally=1 -13:35.130 [INFO] TestAgent.server.raft: election won: tally=1 -13:35.130 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17043 [Leader]" -13:35.130 [INFO] TestAgent.server: cluster leadership acquired -13:35.130 [INFO] TestAgent.server: initializing acls -13:35.131 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:35.131 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:35.131 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:35.132 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:35.132 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:35.132 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:35.132 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-afc487a7-07b3-5841-8e8f-6ed52ed082c5 -13:35.132 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-afc487a7-07b3-5841-8e8f-6ed52ed082c5.dc1 -13:35.133 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:35.134 [INFO] TestAgent.server: New leader elected: payload=Node-afc487a7-07b3-5841-8e8f-6ed52ed082c5 -13:35.134 [INFO] TestAgent.server: Updating LAN server: server="Node-afc487a7-07b3-5841-8e8f-6ed52ed082c5 (Addr: tcp/127.0.0.1:17043) (DC: dc1)" -13:35.134 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-afc487a7-07b3-5841-8e8f-6ed52ed082c5.dc1 area=wan -13:35.158 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:35.158 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:35.158 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:35.158 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:35.158 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:35.158 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-afc487a7-07b3-5841-8e8f-6ed52ed082c5 -13:35.158 [INFO] TestAgent.server: member joined, marking health alive: member=Node-afc487a7-07b3-5841-8e8f-6ed52ed082c5 -13:35.159 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-afc487a7-07b3-5841-8e8f-6ed52ed082c5 -13:35.159 [INFO] TestAgent.server: federation state anti-entropy synced -13:35.286 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:35.287 [INFO] TestAgent: Synced node info -13:35.287 [DEBUG] TestAgent: Node info in sync -13:35.306 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-afc487a7-07b3-5841-8e8f-6ed52ed082c5 -13:35.306 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-afc487a7-07b3-5841-8e8f-6ed52ed082c5 -13:35.306 [INFO] TestAgent: Requesting shutdown -13:35.306 [INFO] TestAgent.server: shutting down server -13:35.306 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:35.306 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:35.306 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:35.306 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:35.306 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:35.306 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:35.306 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:35.306 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:35.306 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:35.308 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:35.308 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:35.308 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:35.308 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:35.308 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:35.308 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:35.308Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:35.308Z [INFO] agent.router.manager: shutting down -13:35.309 [INFO] TestAgent: consul server down -13:35.309 [INFO] TestAgent: shutdown complete -13:35.309 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17031 network=tcp -13:35.309 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17031 network=udp -13:35.309 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17032 network=tcp -13:35.809 [INFO] TestAgent: Waiting for endpoints to shut down -13:35.810 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Bootstrap (0.94s) - --- PASS: TestACL_Bootstrap/bootstrap (0.00s) - --- PASS: TestACL_Bootstrap/not_again (0.00s) -=== CONT TestStatusPeersSecondary ---- PASS: TestACLReplicationStatus (0.91s) -=== CONT TestStatusPeers -2021-10-31T23:13:35.100Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:35.204Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:35.220 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c8abcdff-6598-1dc3-ae7c-6858b1557b30 Address:127.0.0.1:17050}]" -13:35.220 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c8abcdff-6598-1dc3-ae7c-6858b1557b30.dc1 127.0.0.1 -13:35.223 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17050 [Follower]" leader= -13:35.225 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c8abcdff-6598-1dc3-ae7c-6858b1557b30 127.0.0.1 -2021-10-31T23:13:35.226Z [INFO] agent.router: Initializing LAN area manager -13:35.226 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c8abcdff-6598-1dc3-ae7c-6858b1557b30.dc1 area=wan -13:35.226 [INFO] TestAgent.server: Adding LAN server: server="Node-c8abcdff-6598-1dc3-ae7c-6858b1557b30 (Addr: tcp/127.0.0.1:17050) (DC: dc1)" -13:35.226 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17066 network=udp -13:35.226 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17066 network=tcp -13:35.227 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17067 network=tcp -13:35.227 [INFO] TestAgent: started state syncer -13:35.227 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17051 network=tcp -13:35.277 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:35.277 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17050 [Candidate]" term=2 -13:35.278 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:35.278 [DEBUG] TestAgent.server.raft: vote granted: from=c8abcdff-6598-1dc3-ae7c-6858b1557b30 term=2 tally=1 -13:35.278 [INFO] TestAgent.server.raft: election won: tally=1 -13:35.278 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17050 [Leader]" -13:35.278 [INFO] TestAgent.server: cluster leadership acquired -13:35.278 [INFO] TestAgent.server: New leader elected: payload=Node-c8abcdff-6598-1dc3-ae7c-6858b1557b30 -13:35.278 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17050 -13:35.280 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:35.291 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:35.291 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:35.291 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:35.291 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:35.291 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:35.291 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c8abcdff-6598-1dc3-ae7c-6858b1557b30 -13:35.291 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c8abcdff-6598-1dc3-ae7c-6858b1557b30 -13:35.292 [INFO] TestAgent.server: federation state anti-entropy synced -13:35.311 [INFO] TestAgent: Requesting shutdown -13:35.311 [INFO] TestAgent.server: shutting down server -13:35.312 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:35.312 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:35.312 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:35.312 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:35.312 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:35.312 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:35.312 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:35.312 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:35.312 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:35.312 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:35.312 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:35.312 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:35.312Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:35.312Z [INFO] agent.router.manager: shutting down -13:35.312 [INFO] TestAgent: consul server down -13:35.312 [INFO] TestAgent: shutdown complete -13:35.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17066 network=tcp -13:35.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17066 network=udp -13:35.312 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17067 network=tcp -13:35.813 [INFO] TestAgent: Waiting for endpoints to shut down -13:35.813 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_ListKeys (0.77s) -=== CONT TestStatusLeaderSecondary -2021-10-31T23:13:35.106Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:35.204Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:35.219 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0c6ec3ec-d15b-303a-a94f-9d34068e6e7f Address:127.0.0.1:17120}]" -13:35.220 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0c6ec3ec-d15b-303a-a94f-9d34068e6e7f.dc1 127.0.0.1 -13:35.220 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0c6ec3ec-d15b-303a-a94f-9d34068e6e7f 127.0.0.1 -2021-10-31T23:13:35.220Z [INFO] agent.router: Initializing LAN area manager -13:35.221 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17052 network=udp -13:35.221 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17120 [Follower]" leader= -13:35.223 [INFO] TestAgent.server: Adding LAN server: server="Node-0c6ec3ec-d15b-303a-a94f-9d34068e6e7f (Addr: tcp/127.0.0.1:17120) (DC: dc1)" -13:35.224 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0c6ec3ec-d15b-303a-a94f-9d34068e6e7f.dc1 area=wan -13:35.224 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17052 network=tcp -13:35.225 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17053 network=tcp -13:35.225 [INFO] TestAgent: started state syncer -13:35.226 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17121 network=tcp -13:35.278 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:35.278 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17120 [Candidate]" term=2 -13:35.279 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:35.279 [DEBUG] TestAgent.server.raft: vote granted: from=0c6ec3ec-d15b-303a-a94f-9d34068e6e7f term=2 tally=1 -13:35.279 [INFO] TestAgent.server.raft: election won: tally=1 -13:35.279 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17120 [Leader]" -13:35.279 [INFO] TestAgent.server: cluster leadership acquired -13:35.279 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17120 -13:35.280 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:35.281 [INFO] TestAgent.server: New leader elected: payload=Node-0c6ec3ec-d15b-303a-a94f-9d34068e6e7f -13:35.291 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:35.291 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:35.291 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:35.291 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:35.291 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:35.291 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0c6ec3ec-d15b-303a-a94f-9d34068e6e7f -13:35.291 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0c6ec3ec-d15b-303a-a94f-9d34068e6e7f -13:35.291 [INFO] TestAgent.server: federation state anti-entropy synced -13:35.309 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:35.314 [INFO] TestAgent: Requesting shutdown -13:35.314 [INFO] TestAgent.server: shutting down server -13:35.314 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:35.314 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:35.314 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:35.314 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:35.314 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:35.314 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:35.314 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:35.314 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:35.314 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:35.314 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:35.314 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:35.314 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:35.314Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:35.314Z [INFO] agent.router.manager: shutting down -13:35.314 [INFO] TestAgent: consul server down -13:35.314 [INFO] TestAgent: shutdown complete -13:35.314 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17052 network=tcp -13:35.314 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17052 network=udp -13:35.315 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17053 network=tcp -13:35.817 [INFO] TestAgent: Waiting for endpoints to shut down -13:35.817 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Execute/#02 -[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:34.945Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:35.066Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:35.069 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:166a455a-3ef5-8a18-7edf-51c23ca34589 Address:127.0.0.1:17029}]" -13:35.070 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17029 [Follower]" leader= -13:35.079 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-166a455a-3ef5-8a18-7edf-51c23ca34589.dc1 127.0.0.1 -13:35.080 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-166a455a-3ef5-8a18-7edf-51c23ca34589 127.0.0.1 -2021-10-31T23:13:35.080Z [INFO] agent.router: Initializing LAN area manager -13:35.081 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17038 network=udp -13:35.081 [INFO] TestAgent.server: Adding LAN server: server="Node-166a455a-3ef5-8a18-7edf-51c23ca34589 (Addr: tcp/127.0.0.1:17029) (DC: dc1)" -13:35.081 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-166a455a-3ef5-8a18-7edf-51c23ca34589.dc1 area=wan -13:35.081 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17038 network=tcp -13:35.082 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17039 network=tcp -13:35.082 [INFO] TestAgent: started state syncer -13:35.083 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17030 network=tcp -13:35.109 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:35.110 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17029 [Candidate]" term=2 -13:35.110 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:35.110 [DEBUG] TestAgent.server.raft: vote granted: from=166a455a-3ef5-8a18-7edf-51c23ca34589 term=2 tally=1 -13:35.110 [INFO] TestAgent.server.raft: election won: tally=1 -13:35.110 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17029 [Leader]" -13:35.110 [INFO] TestAgent.server: cluster leadership acquired -13:35.110 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17029 -13:35.111 [INFO] TestAgent.server: New leader elected: payload=Node-166a455a-3ef5-8a18-7edf-51c23ca34589 -13:35.111 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:35.135 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:35.135 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:35.135 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:35.135 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:35.135 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:35.135 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-166a455a-3ef5-8a18-7edf-51c23ca34589 -13:35.135 [INFO] TestAgent.server: member joined, marking health alive: member=Node-166a455a-3ef5-8a18-7edf-51c23ca34589 -13:35.138 [INFO] TestAgent.server: federation state anti-entropy synced -13:35.366 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:35.366 [INFO] TestAgent: Synced node info -13:35.461 [INFO] TestAgent: Requesting shutdown -13:35.461 [INFO] TestAgent.server: shutting down server -13:35.461 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:35.461 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:35.461 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:35.461 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:35.461 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:35.461 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:35.461 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:35.461 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:35.461 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:35.461 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:35.461 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:35.461Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:35.461Z [INFO] agent.router.manager: shutting down -13:35.462 [INFO] TestAgent: consul server down -13:35.462 [INFO] TestAgent: shutdown complete -13:35.462 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17038 network=tcp -13:35.462 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17038 network=udp -13:35.462 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17039 network=tcp -13:35.965 [INFO] TestAgent: Waiting for endpoints to shut down -13:35.965 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Disabled_Response (1.08s) - --- PASS: TestACL_Disabled_Response/ACLBootstrap (0.00s) - --- PASS: TestACL_Disabled_Response/ACLReplicationStatus (0.00s) - --- PASS: TestACL_Disabled_Response/AgentToken (0.00s) - --- PASS: TestACL_Disabled_Response/ACLRulesTranslate (0.00s) - --- PASS: TestACL_Disabled_Response/ACLRulesTranslateLegacyToken (0.00s) - --- PASS: TestACL_Disabled_Response/ACLPolicyList (0.00s) - --- PASS: TestACL_Disabled_Response/ACLPolicyCRUD (0.00s) - --- PASS: TestACL_Disabled_Response/ACLPolicyCreate (0.00s) - --- PASS: TestACL_Disabled_Response/ACLTokenList (0.00s) - --- PASS: TestACL_Disabled_Response/ACLTokenCreate (0.00s) - --- PASS: TestACL_Disabled_Response/ACLTokenSelf (0.00s) - --- PASS: TestACL_Disabled_Response/ACLTokenCRUD (0.00s) - --- PASS: TestACL_Disabled_Response/ACLRoleList (0.00s) - --- PASS: TestACL_Disabled_Response/ACLRoleCreate (0.00s) - --- PASS: TestACL_Disabled_Response/ACLRoleCRUD (0.00s) - --- PASS: TestACL_Disabled_Response/ACLBindingRuleList (0.00s) - --- PASS: TestACL_Disabled_Response/ACLBindingRuleCreate (0.00s) - --- PASS: TestACL_Disabled_Response/ACLBindingRuleCRUD (0.00s) - --- PASS: TestACL_Disabled_Response/ACLAuthMethodList (0.00s) - --- PASS: TestACL_Disabled_Response/ACLAuthMethodCreate (0.00s) - --- PASS: TestACL_Disabled_Response/ACLAuthMethodCRUD (0.00s) - --- PASS: TestACL_Disabled_Response/ACLLogin (0.00s) - --- PASS: TestACL_Disabled_Response/ACLLogout (0.02s) - --- PASS: TestACL_Disabled_Response/ACLAuthorize (0.00s) -=== CONT TestStatusLeader -2021-10-31T23:13:24.081Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:24.081Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:24.119Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:24.119Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:24.121 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:998516e8-7474-94cf-535c-da07a800ce15 Address:127.0.0.1:16217}]" -13:24.121 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16217 [Follower]" leader= -13:24.135 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-998516e8-7474-94cf-535c-da07a800ce15.dc1 127.0.0.1 -13:24.136 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-998516e8-7474-94cf-535c-da07a800ce15 127.0.0.1 -2021-10-31T23:13:24.136Z [INFO] agent.router: Initializing LAN area manager -13:24.136 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16156 network=udp -13:24.136 [INFO] TestAgent.server: Adding LAN server: server="Node-998516e8-7474-94cf-535c-da07a800ce15 (Addr: tcp/127.0.0.1:16217) (DC: dc1)" -13:24.136 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-998516e8-7474-94cf-535c-da07a800ce15.dc1 area=wan -13:24.137 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16156 network=tcp -13:24.146 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16213 network=tcp -13:24.146 [INFO] TestAgent: started state syncer -13:24.146 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16218 network=tcp -13:24.189 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:24.190 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16217 [Candidate]" term=2 -13:24.189 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:24.190 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:24.190 [DEBUG] TestAgent.server.raft: vote granted: from=998516e8-7474-94cf-535c-da07a800ce15 term=2 tally=1 -13:24.190 [INFO] TestAgent.server.raft: election won: tally=1 -13:24.190 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16217 [Leader]" -13:24.190 [INFO] TestAgent.server: cluster leadership acquired -13:24.190 [INFO] TestAgent.server: initializing acls -13:24.191 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:24.191 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:24.191 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:24.192 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:24.192 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:24.192 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:24.192 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-998516e8-7474-94cf-535c-da07a800ce15 -13:24.192 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-998516e8-7474-94cf-535c-da07a800ce15.dc1 -13:24.193 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:24.193 [INFO] TestAgent.server: New leader elected: payload=Node-998516e8-7474-94cf-535c-da07a800ce15 -13:24.194 [INFO] TestAgent.server: Updating LAN server: server="Node-998516e8-7474-94cf-535c-da07a800ce15 (Addr: tcp/127.0.0.1:16217) (DC: dc1)" -13:24.194 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-998516e8-7474-94cf-535c-da07a800ce15.dc1 area=wan -13:24.200 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:24.200 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:24.200 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:24.200 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:24.200 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:24.200 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-998516e8-7474-94cf-535c-da07a800ce15 -13:24.200 [INFO] TestAgent.server: member joined, marking health alive: member=Node-998516e8-7474-94cf-535c-da07a800ce15 -13:24.201 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-998516e8-7474-94cf-535c-da07a800ce15 -13:24.201 [INFO] TestAgent.server: federation state anti-entropy synced -13:24.450 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-998516e8-7474-94cf-535c-da07a800ce15 -13:24.450 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-998516e8-7474-94cf-535c-da07a800ce15 -13:24.451 [INFO] TestAgent.server: server starting leave -13:24.451 [INFO] TestAgent.server.serf.wan: serf: EventMemberLeave: Node-998516e8-7474-94cf-535c-da07a800ce15.dc1 127.0.0.1 -13:24.451 [INFO] TestAgent.server: Handled event for server in area: event=member-leave server=Node-998516e8-7474-94cf-535c-da07a800ce15.dc1 area=wan -2021-10-31T23:13:24.451Z [INFO] agent.router.manager: shutting down -13:24.482 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:24.483 [INFO] TestAgent: Synced node info -13:24.483 [DEBUG] TestAgent: Node info in sync -13:25.357 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:25.357 [DEBUG] TestAgent: Node info in sync -13:27.491 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-998516e8-7474-94cf-535c-da07a800ce15 127.0.0.1 -13:27.491 [INFO] TestAgent.server: Removing LAN server: server="Node-998516e8-7474-94cf-535c-da07a800ce15 (Addr: tcp/127.0.0.1:16217) (DC: dc1)" -13:27.491 [WARN] TestAgent.server: deregistering self should be done by follower: name=Node-998516e8-7474-94cf-535c-da07a800ce15 -13:28.195 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -13:30.219 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -13:30.493 [INFO] TestAgent.server: Waiting to drain RPC traffic: drain_time=5s -13:32.211 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -13:34.194 [ERROR] TestAgent.server.autopilot: Error promoting servers: error="error getting server raft protocol versions: No servers found" -13:34.194 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -13:35.494 [INFO] TestAgent: Requesting shutdown -13:35.494 [INFO] TestAgent.server: shutting down server -13:35.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:35.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:35.494 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:35.494 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:35.494 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:35.494 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:35.495 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:35.495 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:35.495 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:35.495 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:35.495 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:35.495 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:35.495 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -2021-10-31T23:13:35.495Z [INFO] agent.router.manager: shutting down -13:35.495 [INFO] TestAgent: consul server down -13:35.495 [INFO] TestAgent: shutdown complete -13:35.495 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16156 network=tcp -13:35.495 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16156 network=udp -13:35.495 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16213 network=tcp -13:35.997 [INFO] TestAgent: Waiting for endpoints to shut down -13:35.997 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_Leave_ACLDeny (11.94s) - --- PASS: TestAgent_Leave_ACLDeny/no_token (0.00s) - --- PASS: TestAgent_Leave_ACLDeny/read-only_token (0.00s) - --- PASS: TestAgent_Leave_ACLDeny/agent_master_token (11.04s) -=== CONT TestSnapshot_Options -=== RUN TestSnapshot_Options/GET -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=true#01 -2021-10-31T23:13:35.068Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:35.141Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:35.160 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7c9e705e-77ec-b6ca-3e01-3822dda87f98 Address:127.0.0.1:17071}]" -13:35.161 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17071 [Follower]" leader= -13:35.161 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7c9e705e-77ec-b6ca-3e01-3822dda87f98.dc1 127.0.0.1 -13:35.171 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7c9e705e-77ec-b6ca-3e01-3822dda87f98 127.0.0.1 -2021-10-31T23:13:35.171Z [INFO] agent.router: Initializing LAN area manager -13:35.171 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17045 network=udp -13:35.172 [INFO] TestAgent.server: Adding LAN server: server="Node-7c9e705e-77ec-b6ca-3e01-3822dda87f98 (Addr: tcp/127.0.0.1:17071) (DC: dc1)" -13:35.172 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7c9e705e-77ec-b6ca-3e01-3822dda87f98.dc1 area=wan -13:35.172 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17045 network=tcp -13:35.173 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17046 network=tcp -13:35.173 [INFO] TestAgent: started state syncer -13:35.173 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17072 network=tcp -13:35.226 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:35.226 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17071 [Candidate]" term=2 -13:35.226 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:35.226 [DEBUG] TestAgent.server.raft: vote granted: from=7c9e705e-77ec-b6ca-3e01-3822dda87f98 term=2 tally=1 -13:35.226 [INFO] TestAgent.server.raft: election won: tally=1 -13:35.226 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17071 [Leader]" -13:35.226 [INFO] TestAgent.server: cluster leadership acquired -13:35.227 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17071 -13:35.227 [INFO] TestAgent.server: New leader elected: payload=Node-7c9e705e-77ec-b6ca-3e01-3822dda87f98 -13:35.236 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:35.271 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:35.271 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:35.271 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:35.271 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:35.271 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:35.271 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7c9e705e-77ec-b6ca-3e01-3822dda87f98 -13:35.271 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7c9e705e-77ec-b6ca-3e01-3822dda87f98 -13:35.272 [INFO] TestAgent.server: federation state anti-entropy synced -13:35.454 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:35.466 [INFO] TestAgent: Synced node info -13:35.466 [DEBUG] TestAgent: Node info in sync -13:35.573 [INFO] TestAgent: Requesting shutdown -13:35.573 [INFO] TestAgent.server: shutting down server -13:35.573 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:35.573 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:35.573 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:35.573 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:35.573 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:35.573 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:35.573 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:35.573 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:35.573 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:35.573 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:35.573 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:35.574Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:35.574Z [INFO] agent.router.manager: shutting down -13:35.574 [INFO] TestAgent: consul server down -13:35.574 [INFO] TestAgent: shutdown complete -13:35.574 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17045 network=tcp -13:35.574 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17045 network=udp -13:35.574 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17046 network=tcp -13:36.076 [INFO] TestAgent: Waiting for endpoints to shut down -13:36.076 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_CAS (1.09s) -=== CONT TestACL_Legacy_Destroy -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:35.407Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:35.459Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:35.483 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:18ad9913-243d-06ee-a201-523275465248 Address:127.0.0.1:17113}]" -13:35.484 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-18ad9913-243d-06ee-a201-523275465248.dc1 127.0.0.1 -13:35.484 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-18ad9913-243d-06ee-a201-523275465248 127.0.0.1 -2021-10-31T23:13:35.484Z [INFO] agent.router: Initializing LAN area manager -13:35.485 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17122 network=udp -13:35.485 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17113 [Follower]" leader= -13:35.485 [INFO] TestAgent.server: Adding LAN server: server="Node-18ad9913-243d-06ee-a201-523275465248 (Addr: tcp/127.0.0.1:17113) (DC: dc1)" -13:35.485 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-18ad9913-243d-06ee-a201-523275465248.dc1 area=wan -13:35.486 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17122 network=tcp -13:35.487 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17123 network=tcp -13:35.487 [INFO] TestAgent: started state syncer -13:35.487 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17114 network=tcp -13:35.537 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:35.537 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17113 [Candidate]" term=2 -13:35.546 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:35.546 [DEBUG] TestAgent.server.raft: vote granted: from=18ad9913-243d-06ee-a201-523275465248 term=2 tally=1 -13:35.546 [INFO] TestAgent.server.raft: election won: tally=1 -13:35.546 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17113 [Leader]" -13:35.546 [INFO] TestAgent.server: cluster leadership acquired -13:35.546 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17113 -13:35.547 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:35.548 [INFO] TestAgent.server: New leader elected: payload=Node-18ad9913-243d-06ee-a201-523275465248 -13:35.548 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:35.548 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:35.548 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:35.548 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:35.548 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:35.548 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-18ad9913-243d-06ee-a201-523275465248 -13:35.548 [INFO] TestAgent.server: member joined, marking health alive: member=Node-18ad9913-243d-06ee-a201-523275465248 -13:35.548 [INFO] TestAgent.server: federation state anti-entropy synced -13:35.610 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:35.610 [INFO] TestAgent: Synced node info -13:35.610 [DEBUG] TestAgent: Node info in sync -13:35.775 [INFO] TestAgent: Requesting shutdown -13:35.775 [INFO] TestAgent.server: shutting down server -13:35.775 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:35.775 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:35.775 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:35.775 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:35.775 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:35.775 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:35.775 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:35.775 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:35.775 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:35.775 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:35.775 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:35.775Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:35.775Z [INFO] agent.router.manager: shutting down -13:35.776 [INFO] TestAgent: consul server down -13:35.776 [INFO] TestAgent: shutdown complete -13:35.776 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17122 network=tcp -13:35.776 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17122 network=udp -13:35.776 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17123 network=tcp -13:36.278 [INFO] TestAgent: Waiting for endpoints to shut down -13:36.278 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionDeleteDestroy (0.92s) -=== CONT TestACL_Legacy_Get -=== RUN TestACL_Legacy_Get/wrong_id -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:35.800Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:35.851Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:35.853 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7333d226-5c0d-d731-7e8a-726ed4d754b1 Address:127.0.0.1:17099}]" -13:35.853 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17099 [Follower]" leader= -13:35.856 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7333d226-5c0d-d731-7e8a-726ed4d754b1.dc1 127.0.0.1 -13:35.856 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7333d226-5c0d-d731-7e8a-726ed4d754b1 127.0.0.1 -2021-10-31T23:13:35.856Z [INFO] agent.router: Initializing LAN area manager -13:35.857 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17087 network=udp -13:35.857 [INFO] TestAgent.server: Adding LAN server: server="Node-7333d226-5c0d-d731-7e8a-726ed4d754b1 (Addr: tcp/127.0.0.1:17099) (DC: dc1)" -13:35.857 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7333d226-5c0d-d731-7e8a-726ed4d754b1.dc1 area=wan -13:35.857 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17087 network=tcp -13:35.858 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17088 network=tcp -13:35.858 [INFO] TestAgent: started state syncer -13:35.858 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17100 network=tcp -13:35.900 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:35.900 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17099 [Candidate]" term=2 -13:35.900 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:35.900 [DEBUG] TestAgent.server.raft: vote granted: from=7333d226-5c0d-d731-7e8a-726ed4d754b1 term=2 tally=1 -13:35.900 [INFO] TestAgent.server.raft: election won: tally=1 -13:35.900 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17099 [Leader]" -13:35.900 [INFO] TestAgent.server: cluster leadership acquired -13:35.900 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17099 -13:35.901 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:35.901 [INFO] TestAgent.server: New leader elected: payload=Node-7333d226-5c0d-d731-7e8a-726ed4d754b1 -13:35.903 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:35.903 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:35.903 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:35.903 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:35.903 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:35.903 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7333d226-5c0d-d731-7e8a-726ed4d754b1 -13:35.903 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7333d226-5c0d-d731-7e8a-726ed4d754b1 -13:35.904 [INFO] TestAgent.server: federation state anti-entropy synced -13:36.066 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:36.067 [INFO] TestAgent: Synced node info -13:36.067 [DEBUG] TestAgent: Node info in sync -13:36.150 [INFO] TestAgent: Requesting shutdown -13:36.150 [INFO] TestAgent.server: shutting down server -13:36.150 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:36.150 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:36.150 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:36.150 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:36.150 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:36.150 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:36.150 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:36.150 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:36.150 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:36.150 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:36.150 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:36.150Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:36.150Z [INFO] agent.router.manager: shutting down -13:36.150 [INFO] TestAgent: consul server down -13:36.150 [INFO] TestAgent: shutdown complete -13:36.150 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17087 network=tcp -13:36.150 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17087 network=udp -13:36.150 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17088 network=tcp -13:36.651 [INFO] TestAgent: Waiting for endpoints to shut down -13:36.651 [INFO] TestAgent: Endpoints down ---- PASS: TestTxnEndpoint_Bad_JSON (0.88s) -=== CONT TestACL_Legacy_Clone -2021-10-31T23:13:35.871Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:35.907Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:35.909 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:38c3ad15-3927-ebd3-7dad-411bf18cf878 Address:127.0.0.1:17127}]" -13:35.909 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17127 [Follower]" leader= -13:35.922 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-38c3ad15-3927-ebd3-7dad-411bf18cf878.dc1 127.0.0.1 -13:35.922 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-38c3ad15-3927-ebd3-7dad-411bf18cf878 127.0.0.1 -2021-10-31T23:13:35.922Z [INFO] agent.router: Initializing LAN area manager -13:35.923 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17094 network=udp -13:35.923 [INFO] TestAgent.server: Adding LAN server: server="Node-38c3ad15-3927-ebd3-7dad-411bf18cf878 (Addr: tcp/127.0.0.1:17127) (DC: dc1)" -13:35.923 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-38c3ad15-3927-ebd3-7dad-411bf18cf878.dc1 area=wan -13:35.923 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17094 network=tcp -13:35.924 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17095 network=tcp -13:35.924 [INFO] TestAgent: started state syncer -13:35.924 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17128 network=tcp -13:35.993 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:35.993 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17127 [Candidate]" term=2 -13:35.994 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:35.994 [DEBUG] TestAgent.server.raft: vote granted: from=38c3ad15-3927-ebd3-7dad-411bf18cf878 term=2 tally=1 -13:35.994 [INFO] TestAgent.server.raft: election won: tally=1 -13:35.994 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17127 [Leader]" -13:35.994 [INFO] TestAgent.server: cluster leadership acquired -13:35.994 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17127 -13:35.995 [INFO] TestAgent.server: New leader elected: payload=Node-38c3ad15-3927-ebd3-7dad-411bf18cf878 -13:35.995 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:36.006 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:36.006 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:36.006 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:36.006 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:36.006 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:36.006 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-38c3ad15-3927-ebd3-7dad-411bf18cf878 -13:36.006 [INFO] TestAgent.server: member joined, marking health alive: member=Node-38c3ad15-3927-ebd3-7dad-411bf18cf878 -13:36.007 [INFO] TestAgent.server: federation state anti-entropy synced -13:36.158 [INFO] TestAgent: Requesting shutdown -13:36.158 [INFO] TestAgent.server: shutting down server -13:36.158 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:36.158 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:36.158 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:36.158 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:36.158 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:36.158 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:36.158 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:36.158 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:36.158 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:36.158 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:36.158 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:36.158 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:36.158Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:36.158Z [INFO] agent.router.manager: shutting down -13:36.158 [INFO] TestAgent: consul server down -13:36.158 [INFO] TestAgent: shutdown complete -13:36.158 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17094 network=tcp -13:36.158 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17094 network=udp -13:36.158 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17095 network=tcp -13:36.661 [INFO] TestAgent: Waiting for endpoints to shut down -13:36.661 [INFO] TestAgent: Endpoints down ---- PASS: TestStatusPeers (0.85s) -=== CONT TestOperator_Keyring_LocalOnly -2021-10-31T23:13:35.772Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:35.827Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:35.829 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a163cc60-c541-f85a-07a5-45b0de5c4163 Address:127.0.0.1:17085}]" -13:35.829 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17085 [Follower]" leader= -13:35.838 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a163cc60-c541-f85a-07a5-45b0de5c4163.dc1 127.0.0.1 -13:35.839 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a163cc60-c541-f85a-07a5-45b0de5c4163 127.0.0.1 -2021-10-31T23:13:35.839Z [INFO] agent.router: Initializing LAN area manager -13:35.839 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17115 network=udp -13:35.839 [INFO] TestAgent.server: Adding LAN server: server="Node-a163cc60-c541-f85a-07a5-45b0de5c4163 (Addr: tcp/127.0.0.1:17085) (DC: dc1)" -13:35.839 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a163cc60-c541-f85a-07a5-45b0de5c4163.dc1 area=wan -13:35.839 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17115 network=tcp -13:35.840 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17116 network=tcp -13:35.840 [INFO] TestAgent: started state syncer -13:35.840 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17086 network=tcp -13:35.897 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:35.897 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17085 [Candidate]" term=2 -13:35.898 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:35.898 [DEBUG] TestAgent.server.raft: vote granted: from=a163cc60-c541-f85a-07a5-45b0de5c4163 term=2 tally=1 -13:35.898 [INFO] TestAgent.server.raft: election won: tally=1 -13:35.898 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17085 [Leader]" -13:35.898 [INFO] TestAgent.server: cluster leadership acquired -13:35.898 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17085 -13:35.899 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:35.900 [INFO] TestAgent.server: New leader elected: payload=Node-a163cc60-c541-f85a-07a5-45b0de5c4163 -13:35.900 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:35.901 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:35.901 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:35.901 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:35.901 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:35.901 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a163cc60-c541-f85a-07a5-45b0de5c4163 -13:35.901 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a163cc60-c541-f85a-07a5-45b0de5c4163 -13:35.906 [INFO] TestAgent.server: federation state anti-entropy synced -13:36.177 [INFO] TestAgent: Requesting shutdown -13:36.177 [INFO] TestAgent.server: shutting down server -13:36.177 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:36.177 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:36.177 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:36.177 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:36.177 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:36.177 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:36.177 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:36.177 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:36.177 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:36.177 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:36.177 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:36.177 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:36.178Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:36.178Z [INFO] agent.router.manager: shutting down -13:36.178 [INFO] TestAgent: consul server down -13:36.178 [INFO] TestAgent: shutdown complete -13:36.178 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17115 network=tcp -13:36.178 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17115 network=udp -13:36.178 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17116 network=tcp -13:36.679 [INFO] TestAgent: Waiting for endpoints to shut down -13:36.679 [INFO] TestAgent: Endpoints down -=== RUN TestTxnEndpoint_Bad_Size_Item/exceeds_configured_max_txn_len -2021-10-31T23:13:35.855Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:35.901Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:35.915 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:66027648-3fde-9299-f49e-26d360d00427 Address:127.0.0.1:17134}]" -13:35.916 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-66027648-3fde-9299-f49e-26d360d00427.dc1 127.0.0.1 -13:35.916 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-66027648-3fde-9299-f49e-26d360d00427 127.0.0.1 -2021-10-31T23:13:35.916Z [INFO] agent.router: Initializing LAN area manager -13:35.916 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17134 [Follower]" leader= -13:35.916 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17129 network=udp -13:35.917 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-66027648-3fde-9299-f49e-26d360d00427.dc1 area=wan -13:35.917 [INFO] TestAgent.server: Adding LAN server: server="Node-66027648-3fde-9299-f49e-26d360d00427 (Addr: tcp/127.0.0.1:17134) (DC: dc1)" -13:35.917 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17129 network=tcp -13:35.919 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17130 network=tcp -13:35.919 [INFO] TestAgent: started state syncer -13:35.919 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17135 network=tcp -13:35.961 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:35.961 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17134 [Candidate]" term=2 -13:35.962 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:35.962 [DEBUG] TestAgent.server.raft: vote granted: from=66027648-3fde-9299-f49e-26d360d00427 term=2 tally=1 -13:35.962 [INFO] TestAgent.server.raft: election won: tally=1 -13:35.962 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17134 [Leader]" -13:35.962 [INFO] TestAgent.server: cluster leadership acquired -13:35.962 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17134 -13:35.963 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:35.964 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:35.964 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:35.964 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:35.964 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:35.964 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:35.964 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-66027648-3fde-9299-f49e-26d360d00427 -13:35.964 [INFO] TestAgent.server: member joined, marking health alive: member=Node-66027648-3fde-9299-f49e-26d360d00427 -13:35.965 [INFO] TestAgent.server: New leader elected: payload=Node-66027648-3fde-9299-f49e-26d360d00427 -13:35.965 [INFO] TestAgent.server: federation state anti-entropy synced -13:35.994 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:35.995 [INFO] TestAgent: Synced node info -13:35.995 [DEBUG] TestAgent: Node info in sync -13:36.297 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:36.297 [INFO] TestAgent: Requesting shutdown -13:36.297 [INFO] TestAgent.server: shutting down server -13:36.297 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:36.297 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:36.297 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:36.297 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:36.297 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:36.297 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:36.297 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:36.297 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:36.297 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:36.297 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:36.297 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:36.298Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:36.298Z [INFO] agent.router.manager: shutting down -13:36.298 [INFO] TestAgent: consul server down -13:36.298 [INFO] TestAgent: shutdown complete -13:36.298 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17129 network=tcp -13:36.298 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17129 network=udp -13:36.298 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17130 network=tcp -13:36.800 [INFO] TestAgent: Waiting for endpoints to shut down -13:36.841 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Execute/#03 +=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{5_5_5_5_5_5_5_5_5_5_5} +2022-12-04T05:49:40.984Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.984Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:41.005Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:41.005Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:41.006 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a1956d3d-77f6-7fc0-0c44-ac6fba977b1f Address:127.0.0.1:23604}]" +49:41.006 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23604 [Follower]" leader= +49:41.006 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a1956d3d-77f6-7fc0-0c44-ac6fba977b1f.dc1 127.0.0.1 +49:41.007 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a1956d3d-77f6-7fc0-0c44-ac6fba977b1f 127.0.0.1 +2022-12-04T05:49:41.007Z [INFO] agent.router: Initializing LAN area manager +49:41.007 [INFO] TestAgent.server: Adding LAN server: server="Node-a1956d3d-77f6-7fc0-0c44-ac6fba977b1f (Addr: tcp/127.0.0.1:23604) (DC: dc1)" +49:41.007 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a1956d3d-77f6-7fc0-0c44-ac6fba977b1f.dc1 area=wan +49:41.007 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23676 network=udp +49:41.007 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23676 network=tcp +49:41.008 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23677 network=tcp +49:41.008 [INFO] TestAgent: started state syncer +49:41.008 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23605 network=tcp +49:41.054 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:41.054 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23604 [Candidate]" term=2 +49:41.055 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:41.055 [DEBUG] TestAgent.server.raft: vote granted: from=a1956d3d-77f6-7fc0-0c44-ac6fba977b1f term=2 tally=1 +49:41.055 [INFO] TestAgent.server.raft: election won: tally=1 +49:41.055 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23604 [Leader]" +49:41.055 [INFO] TestAgent.server: cluster leadership acquired +49:41.055 [INFO] TestAgent.server: initializing acls +49:41.055 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:41.055 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:41.055 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:41.056 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:41.056 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:41.056 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:41.056 [INFO] TestAgent.server: New leader elected: payload=Node-a1956d3d-77f6-7fc0-0c44-ac6fba977b1f +49:41.056 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a1956d3d-77f6-7fc0-0c44-ac6fba977b1f +49:41.056 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a1956d3d-77f6-7fc0-0c44-ac6fba977b1f.dc1 +49:41.056 [INFO] TestAgent.server: Updating LAN server: server="Node-a1956d3d-77f6-7fc0-0c44-ac6fba977b1f (Addr: tcp/127.0.0.1:23604) (DC: dc1)" +49:41.056 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a1956d3d-77f6-7fc0-0c44-ac6fba977b1f.dc1 area=wan +49:41.056 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:41.058 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:41.058 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:41.058 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:41.058 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:41.058 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:41.058 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1956d3d-77f6-7fc0-0c44-ac6fba977b1f +49:41.058 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a1956d3d-77f6-7fc0-0c44-ac6fba977b1f +49:41.058 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1956d3d-77f6-7fc0-0c44-ac6fba977b1f +49:41.058 [INFO] TestAgent.server: federation state anti-entropy synced +49:41.200 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a1956d3d-77f6-7fc0-0c44-ac6fba977b1f +49:41.201 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a1956d3d-77f6-7fc0-0c44-ac6fba977b1f +49:41.201 [INFO] TestAgent: Requesting shutdown +49:41.201 [INFO] TestAgent.server: shutting down server +49:41.201 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:41.201 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:41.201 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:41.201 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:41.201 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:41.201 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:41.201 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:41.201 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:41.201 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:41.201 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:41.201 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:41.201 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:41.201 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:41.201 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:41.201 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:41.201 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:41.201Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:41.201Z [INFO] agent.router.manager: shutting down +49:41.201 [INFO] TestAgent: consul server down +49:41.201 [INFO] TestAgent: shutdown complete +49:41.201 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23676 network=tcp +49:41.201 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23676 network=udp +49:41.201 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23677 network=tcp +49:41.701 [INFO] TestAgent: Waiting for endpoints to shut down +49:41.701 [INFO] TestAgent: Endpoints down +=== RUN TestACL_Legacy_Get/right_id +2022-12-04T05:49:41.087Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:41.087Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:41.099Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:41.099Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:41.101 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9793efb9-ce2c-46e8-6687-f93eebea24b2 Address:127.0.0.1:23646}]" +49:41.101 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23646 [Follower]" leader= +49:41.102 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9793efb9-ce2c-46e8-6687-f93eebea24b2.dc1 127.0.0.1 +49:41.102 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9793efb9-ce2c-46e8-6687-f93eebea24b2 127.0.0.1 +2022-12-04T05:49:41.102Z [INFO] agent.router: Initializing LAN area manager +49:41.102 [INFO] TestAgent.server: Adding LAN server: server="Node-9793efb9-ce2c-46e8-6687-f93eebea24b2 (Addr: tcp/127.0.0.1:23646) (DC: dc1)" +49:41.102 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9793efb9-ce2c-46e8-6687-f93eebea24b2.dc1 area=wan +49:41.102 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23725 network=udp +49:41.102 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23725 network=tcp +49:41.103 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23726 network=tcp +49:41.103 [INFO] TestAgent: started state syncer +49:41.103 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23647 network=tcp +49:41.146 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:41.146 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23646 [Candidate]" term=2 +49:41.146 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:41.146 [DEBUG] TestAgent.server.raft: vote granted: from=9793efb9-ce2c-46e8-6687-f93eebea24b2 term=2 tally=1 +49:41.146 [INFO] TestAgent.server.raft: election won: tally=1 +49:41.146 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23646 [Leader]" +49:41.146 [INFO] TestAgent.server: cluster leadership acquired +49:41.146 [INFO] TestAgent.server: New leader elected: payload=Node-9793efb9-ce2c-46e8-6687-f93eebea24b2 +49:41.146 [INFO] TestAgent.server: initializing acls +49:41.147 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:41.147 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:41.147 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:41.147 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:41.147 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:41.147 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:41.147 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9793efb9-ce2c-46e8-6687-f93eebea24b2 +49:41.147 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9793efb9-ce2c-46e8-6687-f93eebea24b2.dc1 +49:41.147 [INFO] TestAgent.server: Updating LAN server: server="Node-9793efb9-ce2c-46e8-6687-f93eebea24b2 (Addr: tcp/127.0.0.1:23646) (DC: dc1)" +49:41.147 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9793efb9-ce2c-46e8-6687-f93eebea24b2.dc1 area=wan +49:41.148 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:41.149 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:41.149 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:41.149 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:41.149 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:41.149 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:41.149 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9793efb9-ce2c-46e8-6687-f93eebea24b2 +49:41.149 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9793efb9-ce2c-46e8-6687-f93eebea24b2 +49:41.155 [INFO] TestAgent.server: federation state anti-entropy synced +49:41.155 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9793efb9-ce2c-46e8-6687-f93eebea24b2 +49:41.251 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:41.252 [INFO] TestAgent: Synced node info +49:41.252 [DEBUG] TestAgent: Node info in sync +49:41.275 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9793efb9-ce2c-46e8-6687-f93eebea24b2 +49:41.275 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9793efb9-ce2c-46e8-6687-f93eebea24b2 +49:41.276 [INFO] TestAgent: Requesting shutdown +49:41.276 [INFO] TestAgent.server: shutting down server +49:41.276 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:41.276 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:41.276 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:41.276 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:41.276 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:41.276 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:41.276 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:41.276 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:41.276 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:41.276 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:41.276 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:41.276 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:41.276 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:41.276 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:41.276 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:41.276Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:41.276Z [INFO] agent.router.manager: shutting down +49:41.276 [INFO] TestAgent: consul server down +49:41.276 [INFO] TestAgent: shutdown complete +49:41.276 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23725 network=tcp +49:41.276 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23725 network=udp +49:41.276 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23726 network=tcp +49:41.776 [INFO] TestAgent: Waiting for endpoints to shut down +49:41.776 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Legacy_Destroy (0.70s) +=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{5_5_5_5_5_5_5_5_5_5_5} [WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:36.153Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:36.153Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:36.208Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:36.208Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:36.210 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2c0a4859-64ea-c332-79b5-f36283a8cfb2 Address:127.0.0.1:16128}]" -13:36.210 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16128 [Follower]" leader= -13:36.211 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2c0a4859-64ea-c332-79b5-f36283a8cfb2.dc1 127.0.0.1 -13:36.212 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2c0a4859-64ea-c332-79b5-f36283a8cfb2 127.0.0.1 -2021-10-31T23:13:36.212Z [INFO] agent.router: Initializing LAN area manager -13:36.212 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16242 network=udp -13:36.212 [INFO] TestAgent.server: Adding LAN server: server="Node-2c0a4859-64ea-c332-79b5-f36283a8cfb2 (Addr: tcp/127.0.0.1:16128) (DC: dc1)" -13:36.212 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2c0a4859-64ea-c332-79b5-f36283a8cfb2.dc1 area=wan -13:36.212 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16242 network=tcp -13:36.213 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16243 network=tcp -13:36.213 [INFO] TestAgent: started state syncer -13:36.213 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16129 network=tcp -13:36.257 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:36.257 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16128 [Candidate]" term=2 -13:36.258 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:36.258 [DEBUG] TestAgent.server.raft: vote granted: from=2c0a4859-64ea-c332-79b5-f36283a8cfb2 term=2 tally=1 -13:36.258 [INFO] TestAgent.server.raft: election won: tally=1 -13:36.258 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16128 [Leader]" -13:36.258 [INFO] TestAgent.server: cluster leadership acquired -13:36.258 [INFO] TestAgent.server: initializing acls -13:36.258 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:36.258 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:36.259 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:36.259 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:36.259 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:36.259 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:36.259 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-2c0a4859-64ea-c332-79b5-f36283a8cfb2 -13:36.259 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-2c0a4859-64ea-c332-79b5-f36283a8cfb2.dc1 -13:36.260 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:36.261 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:36.262 [INFO] TestAgent.server: New leader elected: payload=Node-2c0a4859-64ea-c332-79b5-f36283a8cfb2 -13:36.262 [INFO] TestAgent.server: Updating LAN server: server="Node-2c0a4859-64ea-c332-79b5-f36283a8cfb2 (Addr: tcp/127.0.0.1:16128) (DC: dc1)" -13:36.262 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-2c0a4859-64ea-c332-79b5-f36283a8cfb2.dc1 area=wan -13:36.269 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:36.269 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:36.269 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:36.269 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:36.270 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2c0a4859-64ea-c332-79b5-f36283a8cfb2 -13:36.270 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2c0a4859-64ea-c332-79b5-f36283a8cfb2 -13:36.271 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2c0a4859-64ea-c332-79b5-f36283a8cfb2 -13:36.271 [INFO] TestAgent.server: federation state anti-entropy synced -13:36.298 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:36.299 [INFO] TestAgent: Synced node info -13:36.360 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-2c0a4859-64ea-c332-79b5-f36283a8cfb2 -13:36.360 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-2c0a4859-64ea-c332-79b5-f36283a8cfb2 -13:36.366 [INFO] TestAgent: Requesting shutdown -13:36.367 [INFO] TestAgent.server: shutting down server -13:36.367 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:36.367 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:36.367 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:36.367 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:36.367 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:36.367 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:36.367 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:36.367 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:36.367 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:36.367 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:36.367 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:36.367 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:36.367 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:36.367 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:36.367 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:36.367Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:36.367Z [INFO] agent.router.manager: shutting down -13:36.367 [INFO] TestAgent: consul server down -13:36.367 [INFO] TestAgent: shutdown complete -13:36.367 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16242 network=tcp -13:36.367 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16242 network=udp -13:36.367 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16243 network=tcp -13:36.869 [INFO] TestAgent: Waiting for endpoints to shut down -13:36.869 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Legacy_Destroy (0.79s) -=== CONT TestOperator_AutopilotSetConfiguration -2021-10-31T23:13:34.243Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:34.295Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:34.314 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:58b2b996-3c38-67fd-ffa9-ab74d99ac15c Address:127.0.0.1:17001}]" -13:34.315 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-58b2b996-3c38-67fd-ffa9-ab74d99ac15c.dc1 127.0.0.1 -13:34.315 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-58b2b996-3c38-67fd-ffa9-ab74d99ac15c 127.0.0.1 -2021-10-31T23:13:34.315Z [INFO] agent.router: Initializing LAN area manager -13:34.317 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17001 [Follower]" leader= -13:34.318 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-58b2b996-3c38-67fd-ffa9-ab74d99ac15c.dc1 area=wan -13:34.318 [INFO] TestAgent.server: Adding LAN server: server="Node-58b2b996-3c38-67fd-ffa9-ab74d99ac15c (Addr: tcp/127.0.0.1:17001) (DC: dc1)" -13:34.318 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16972 network=tcp -13:34.318 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16972 network=udp -13:34.318 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16973 network=tcp -13:34.318 [INFO] TestAgent: started state syncer -13:34.319 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17002 network=tcp -13:34.373 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:34.373 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17001 [Candidate]" term=2 -13:34.374 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:34.374 [DEBUG] TestAgent.server.raft: vote granted: from=58b2b996-3c38-67fd-ffa9-ab74d99ac15c term=2 tally=1 -13:34.374 [INFO] TestAgent.server.raft: election won: tally=1 -13:34.374 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17001 [Leader]" -13:34.374 [INFO] TestAgent.server: cluster leadership acquired -13:34.374 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17001 -13:34.375 [INFO] TestAgent.server: New leader elected: payload=Node-58b2b996-3c38-67fd-ffa9-ab74d99ac15c -13:34.375 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:34.376 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:34.376 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:34.376 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:34.376 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:34.376 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:34.376 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-58b2b996-3c38-67fd-ffa9-ab74d99ac15c -13:34.376 [INFO] TestAgent.server: member joined, marking health alive: member=Node-58b2b996-3c38-67fd-ffa9-ab74d99ac15c -13:34.377 [INFO] TestAgent.server: federation state anti-entropy synced -13:34.402 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:34.402 [INFO] TestAgent: Synced node info -13:36.422 [INFO] TestAgent: Requesting shutdown -13:36.422 [INFO] TestAgent.server: shutting down server -13:36.422 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:36.422 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:36.422 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:36.422 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:36.422 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:36.422 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:36.422 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:36.422 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:36.422 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:36.422 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:36.422 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:36.422Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:36.422Z [INFO] agent.router.manager: shutting down -13:36.422 [INFO] TestAgent: consul server down -13:36.422 [INFO] TestAgent: shutdown complete -13:36.422 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16972 network=tcp -13:36.422 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16972 network=udp -13:36.422 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16973 network=tcp -13:36.929 [INFO] TestAgent: Waiting for endpoints to shut down -13:36.930 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_ServerHealth (2.72s) -2021-10-31T23:13:34.240Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:34.303Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:34.317 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:641e6391-506d-961f-8802-d4fcca54cb62 Address:127.0.0.1:16963}]" -13:34.318 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-641e6391-506d-961f-8802-d4fcca54cb62.dc1 127.0.0.1 -13:34.318 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16963 [Follower]" leader= -13:34.318 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-641e6391-506d-961f-8802-d4fcca54cb62 127.0.0.1 -2021-10-31T23:13:34.318Z [INFO] agent.router: Initializing LAN area manager -13:34.318 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16821 network=udp -13:34.319 [INFO] TestAgent.server: Adding LAN server: server="Node-641e6391-506d-961f-8802-d4fcca54cb62 (Addr: tcp/127.0.0.1:16963) (DC: dc1)" -13:34.319 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-641e6391-506d-961f-8802-d4fcca54cb62.dc1 area=wan -13:34.319 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16821 network=tcp -13:34.319 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16822 network=tcp -13:34.319 [INFO] TestAgent: started state syncer -13:34.319 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16964 network=tcp -13:34.377 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:34.377 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16963 [Candidate]" term=2 -13:34.378 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:34.378 [DEBUG] TestAgent.server.raft: vote granted: from=641e6391-506d-961f-8802-d4fcca54cb62 term=2 tally=1 -13:34.378 [INFO] TestAgent.server.raft: election won: tally=1 -13:34.378 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16963 [Leader]" -13:34.378 [INFO] TestAgent.server: cluster leadership acquired -13:34.378 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16963 -13:34.379 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:34.380 [INFO] TestAgent.server: New leader elected: payload=Node-641e6391-506d-961f-8802-d4fcca54cb62 -13:34.380 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:34.380 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:34.380 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:34.380 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:34.380 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:34.380 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-641e6391-506d-961f-8802-d4fcca54cb62 -13:34.380 [INFO] TestAgent.server: member joined, marking health alive: member=Node-641e6391-506d-961f-8802-d4fcca54cb62 -13:34.381 [INFO] TestAgent.server: federation state anti-entropy synced -13:34.494 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:34.495 [INFO] TestAgent: Synced node info -13:36.434 [INFO] TestAgent: Requesting shutdown -13:36.434 [INFO] TestAgent.server: shutting down server -13:36.434 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:36.434 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:36.434 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:36.434 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:36.434 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:36.434 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:36.434 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:36.434 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:36.434 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:36.434 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:36.434 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:36.434Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:36.434Z [INFO] agent.router.manager: shutting down -13:36.434 [INFO] TestAgent: consul server down -13:36.434 [INFO] TestAgent: shutdown complete -13:36.434 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16821 network=tcp -13:36.434 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16821 network=udp -13:36.434 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16822 network=tcp -13:36.934 [INFO] TestAgent: Waiting for endpoints to shut down -13:36.934 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_ServerHealth_Unhealthy (2.74s) -=== CONT TestSessionCreate_NoCheck -2021-10-31T23:13:36.062Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:36.062Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:36.142Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:36.142Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:36.144 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ba107d56-ccc3-1b3c-675b-711c1f689d75 Address:127.0.0.1:17057}]" -13:36.145 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ba107d56-ccc3-1b3c-675b-711c1f689d75.dc1 127.0.0.1 -13:36.145 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ba107d56-ccc3-1b3c-675b-711c1f689d75 127.0.0.1 -13:36.145 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17057 [Follower]" leader= -2021-10-31T23:13:36.150Z [INFO] agent.router: Initializing LAN area manager -13:36.151 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16137 network=udp -13:36.151 [INFO] TestAgent.server: Adding LAN server: server="Node-ba107d56-ccc3-1b3c-675b-711c1f689d75 (Addr: tcp/127.0.0.1:17057) (DC: dc1)" -13:36.151 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ba107d56-ccc3-1b3c-675b-711c1f689d75.dc1 area=wan -13:36.151 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16137 network=tcp -13:36.152 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16138 network=tcp -13:36.152 [INFO] TestAgent: started state syncer -13:36.152 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17058 network=tcp -13:36.200 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:36.200 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17057 [Candidate]" term=2 -13:36.201 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:36.201 [DEBUG] TestAgent.server.raft: vote granted: from=ba107d56-ccc3-1b3c-675b-711c1f689d75 term=2 tally=1 -13:36.201 [INFO] TestAgent.server.raft: election won: tally=1 -13:36.201 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17057 [Leader]" -13:36.201 [INFO] TestAgent.server: cluster leadership acquired -13:36.201 [INFO] TestAgent.server: initializing acls -13:36.201 [INFO] TestAgent.server: initializing acls -13:36.201 [INFO] TestAgent.server: New leader elected: payload=Node-ba107d56-ccc3-1b3c-675b-711c1f689d75 -13:36.201 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:36.201 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:36.201 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:36.201 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:36.202 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:36.202 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:36.202 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:36.202 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:36.202 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ba107d56-ccc3-1b3c-675b-711c1f689d75 -13:36.202 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ba107d56-ccc3-1b3c-675b-711c1f689d75.dc1 -13:36.203 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:36.205 [INFO] TestAgent.server: Updating LAN server: server="Node-ba107d56-ccc3-1b3c-675b-711c1f689d75 (Addr: tcp/127.0.0.1:17057) (DC: dc1)" -13:36.205 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:36.205 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -13:36.205 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:36.205 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:36.205 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:36.205 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:36.205 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:36.205 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba107d56-ccc3-1b3c-675b-711c1f689d75 -13:36.205 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ba107d56-ccc3-1b3c-675b-711c1f689d75 -13:36.205 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ba107d56-ccc3-1b3c-675b-711c1f689d75 -13:36.205 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ba107d56-ccc3-1b3c-675b-711c1f689d75.dc1 -13:36.205 [INFO] TestAgent.server: Updating LAN server: server="Node-ba107d56-ccc3-1b3c-675b-711c1f689d75 (Addr: tcp/127.0.0.1:17057) (DC: dc1)" -13:36.206 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ba107d56-ccc3-1b3c-675b-711c1f689d75.dc1 area=wan -13:36.206 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ba107d56-ccc3-1b3c-675b-711c1f689d75.dc1 area=wan -13:36.214 [INFO] TestAgent.server: federation state anti-entropy synced -13:36.214 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba107d56-ccc3-1b3c-675b-711c1f689d75 -13:36.214 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba107d56-ccc3-1b3c-675b-711c1f689d75 -13:36.418 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ba107d56-ccc3-1b3c-675b-711c1f689d75 -13:36.418 [INFO] TestAgent: Requesting shutdown -13:36.418 [INFO] TestAgent.server: shutting down server -13:36.418 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:36.418 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:36.418 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:36.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:36.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:36.418 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:36.418 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:36.418 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:36.418 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:36.418 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:36.418 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:36.418 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:36.418 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:36.418 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:36.418 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:36.418 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:36.418Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:36.418Z [INFO] agent.router.manager: shutting down -13:36.419 [INFO] TestAgent: consul server down -13:36.419 [INFO] TestAgent: shutdown complete -13:36.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16137 network=tcp -13:36.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16137 network=udp -13:36.419 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16138 network=tcp -13:36.938 [INFO] TestAgent: Waiting for endpoints to shut down -13:36.938 [INFO] TestAgent: Endpoints down -=== RUN TestSnapshot_Options/GET#01 -=== CONT TestOperator_AutopilotGetConfiguration -2021-10-31T23:13:36.061Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:36.100Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:36.131 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0a26cb04-fdf6-acab-3b1f-0cc08b82972d Address:127.0.0.1:16135}]" -13:36.131 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0a26cb04-fdf6-acab-3b1f-0cc08b82972d.dc1 127.0.0.1 -13:36.132 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0a26cb04-fdf6-acab-3b1f-0cc08b82972d 127.0.0.1 -2021-10-31T23:13:36.132Z [INFO] agent.router: Initializing LAN area manager -13:36.132 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17136 network=udp -13:36.132 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16135 [Follower]" leader= -13:36.133 [INFO] TestAgent.server: Adding LAN server: server="Node-0a26cb04-fdf6-acab-3b1f-0cc08b82972d (Addr: tcp/127.0.0.1:16135) (DC: dc1)" -13:36.133 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0a26cb04-fdf6-acab-3b1f-0cc08b82972d.dc1 area=wan -13:36.138 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17136 network=tcp -13:36.138 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17137 network=tcp -13:36.138 [INFO] TestAgent: started state syncer -13:36.138 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16136 network=tcp -13:36.201 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:36.201 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16135 [Candidate]" term=2 -13:36.202 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:36.202 [DEBUG] TestAgent.server.raft: vote granted: from=0a26cb04-fdf6-acab-3b1f-0cc08b82972d term=2 tally=1 -13:36.202 [INFO] TestAgent.server.raft: election won: tally=1 -13:36.202 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16135 [Leader]" -13:36.202 [INFO] TestAgent.server: cluster leadership acquired -13:36.202 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16135 -13:36.203 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:36.204 [INFO] TestAgent.server: New leader elected: payload=Node-0a26cb04-fdf6-acab-3b1f-0cc08b82972d -13:36.205 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:36.205 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:36.205 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:36.205 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:36.205 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:36.205 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a26cb04-fdf6-acab-3b1f-0cc08b82972d -13:36.205 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0a26cb04-fdf6-acab-3b1f-0cc08b82972d -13:36.206 [INFO] TestAgent.server: federation state anti-entropy synced -13:36.378 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:36.418 [INFO] TestAgent: Synced node info -13:36.562 [INFO] TestAgent: Requesting shutdown -13:36.562 [INFO] TestAgent.server: shutting down server -13:36.562 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:36.562 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:36.562 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:36.562 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:36.562 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:36.562 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:36.562 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:36.562 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:36.562 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:36.562 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:36.562 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:36.562Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:36.562Z [INFO] agent.router.manager: shutting down -13:36.563 [INFO] TestAgent: consul server down -13:36.563 [INFO] TestAgent: shutdown complete -13:36.563 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17136 network=tcp -13:36.563 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17136 network=udp -13:36.563 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17137 network=tcp -13:37.067 [INFO] TestAgent: Waiting for endpoints to shut down -13:37.067 [INFO] TestAgent: Endpoints down ---- PASS: TestStatusLeader (1.10s) -=== CONT TestSessionGet -=== RUN TestSessionGet/#00 -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=false#01 +2022-12-04T05:49:40.891Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.891Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:40.904Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.904Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:40.906 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5200b93c-db74-70c5-0ac5-63c87d9ace3e Address:127.0.0.1:23625}]" +49:40.906 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23625 [Follower]" leader= +49:40.907 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5200b93c-db74-70c5-0ac5-63c87d9ace3e.dc1 127.0.0.1 +49:40.907 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5200b93c-db74-70c5-0ac5-63c87d9ace3e 127.0.0.1 +2022-12-04T05:49:40.907Z [INFO] agent.router: Initializing LAN area manager +49:40.907 [INFO] TestAgent.server: Adding LAN server: server="Node-5200b93c-db74-70c5-0ac5-63c87d9ace3e (Addr: tcp/127.0.0.1:23625) (DC: dc1)" +49:40.907 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5200b93c-db74-70c5-0ac5-63c87d9ace3e.dc1 area=wan +49:40.908 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23634 network=udp +49:40.908 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23634 network=tcp +49:40.908 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23635 network=tcp +49:40.908 [INFO] TestAgent: started state syncer +49:40.908 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23675 network=tcp +49:40.947 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:40.947 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23625 [Candidate]" term=2 +49:40.947 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:40.947 [DEBUG] TestAgent.server.raft: vote granted: from=5200b93c-db74-70c5-0ac5-63c87d9ace3e term=2 tally=1 +49:40.947 [INFO] TestAgent.server.raft: election won: tally=1 +49:40.947 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23625 [Leader]" +49:40.947 [INFO] TestAgent.server: cluster leadership acquired +49:40.947 [INFO] TestAgent.server: New leader elected: payload=Node-5200b93c-db74-70c5-0ac5-63c87d9ace3e +49:40.947 [INFO] TestAgent.server: initializing acls +49:40.947 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:40.947 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:40.948 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:40.948 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:40.948 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:40.948 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:40.948 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5200b93c-db74-70c5-0ac5-63c87d9ace3e +49:40.948 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5200b93c-db74-70c5-0ac5-63c87d9ace3e.dc1 +49:40.948 [INFO] TestAgent.server: Updating LAN server: server="Node-5200b93c-db74-70c5-0ac5-63c87d9ace3e (Addr: tcp/127.0.0.1:23625) (DC: dc1)" +49:40.948 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5200b93c-db74-70c5-0ac5-63c87d9ace3e.dc1 area=wan +49:40.949 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:40.950 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:40.950 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:40.950 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:40.950 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:40.950 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:40.950 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5200b93c-db74-70c5-0ac5-63c87d9ace3e +49:40.950 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5200b93c-db74-70c5-0ac5-63c87d9ace3e +49:40.950 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5200b93c-db74-70c5-0ac5-63c87d9ace3e +49:40.951 [INFO] TestAgent.server: federation state anti-entropy synced +49:41.014 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:41.014 [INFO] TestAgent: Synced node info +49:41.014 [DEBUG] TestAgent: Node info in sync +49:41.320 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5200b93c-db74-70c5-0ac5-63c87d9ace3e +49:41.320 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5200b93c-db74-70c5-0ac5-63c87d9ace3e +49:41.323 [INFO] TestAgent: Requesting shutdown +49:41.323 [INFO] TestAgent.server: shutting down server +49:41.323 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:41.323 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:41.323 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:41.323 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:41.323 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:41.323 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:41.323 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:41.323 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:41.323 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:41.323 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:41.323 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:41.323 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:41.323 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:41.323 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:41.323 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:49:41.324Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:41.324Z [INFO] agent.router.manager: shutting down +49:41.324 [INFO] TestAgent: consul server down +49:41.324 [INFO] TestAgent: shutdown complete +49:41.324 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23634 network=tcp +49:41.324 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23634 network=udp +49:41.324 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23635 network=tcp +49:41.824 [INFO] TestAgent: Waiting for endpoints to shut down +49:41.824 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Legacy_List (0.95s) +=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{4_4_4_4_4_4_4_4_4_4_4} +2022-12-04T05:49:40.981Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.981Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:40.994Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:40.994Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:40.995 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:52bd650b-cc57-a059-0901-6f66213b4675 Address:127.0.0.1:23674}]" +49:40.995 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23674 [Follower]" leader= +49:40.996 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-52bd650b-cc57-a059-0901-6f66213b4675.dc1 127.0.0.1 +49:40.996 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-52bd650b-cc57-a059-0901-6f66213b4675 127.0.0.1 +2022-12-04T05:49:40.996Z [INFO] agent.router: Initializing LAN area manager +49:40.996 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23606 network=udp +49:40.996 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-52bd650b-cc57-a059-0901-6f66213b4675.dc1 area=wan +49:40.997 [INFO] TestAgent.server: Adding LAN server: server="Node-52bd650b-cc57-a059-0901-6f66213b4675 (Addr: tcp/127.0.0.1:23674) (DC: dc1)" +49:40.997 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23606 network=tcp +49:40.997 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23607 network=tcp +49:40.997 [INFO] TestAgent: started state syncer +49:40.997 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23724 network=tcp +49:41.047 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:41.052 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:41.052 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23674 [Candidate]" term=2 +49:41.052 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:41.052 [DEBUG] TestAgent.server.raft: vote granted: from=52bd650b-cc57-a059-0901-6f66213b4675 term=2 tally=1 +49:41.052 [INFO] TestAgent.server.raft: election won: tally=1 +49:41.052 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23674 [Leader]" +49:41.053 [INFO] TestAgent.server: cluster leadership acquired +49:41.053 [INFO] TestAgent.server: New leader elected: payload=Node-52bd650b-cc57-a059-0901-6f66213b4675 +49:41.053 [INFO] TestAgent.server: initializing acls +49:41.053 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:41.053 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:41.053 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:41.053 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:41.053 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:41.053 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:41.054 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-52bd650b-cc57-a059-0901-6f66213b4675 +49:41.054 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-52bd650b-cc57-a059-0901-6f66213b4675.dc1 +49:41.054 [INFO] TestAgent.server: Updating LAN server: server="Node-52bd650b-cc57-a059-0901-6f66213b4675 (Addr: tcp/127.0.0.1:23674) (DC: dc1)" +49:41.054 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-52bd650b-cc57-a059-0901-6f66213b4675.dc1 area=wan +49:41.054 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:41.056 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:41.056 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:41.056 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:41.056 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:41.056 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:41.056 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-52bd650b-cc57-a059-0901-6f66213b4675 +49:41.056 [INFO] TestAgent.server: member joined, marking health alive: member=Node-52bd650b-cc57-a059-0901-6f66213b4675 +49:41.057 [INFO] TestAgent.server: federation state anti-entropy synced +49:41.057 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-52bd650b-cc57-a059-0901-6f66213b4675 +49:41.062 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:41.063 [INFO] TestAgent: Synced node info +49:41.340 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-52bd650b-cc57-a059-0901-6f66213b4675 +49:41.340 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-52bd650b-cc57-a059-0901-6f66213b4675 +49:41.341 [INFO] TestAgent: Requesting shutdown +49:41.341 [INFO] TestAgent.server: shutting down server +49:41.341 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:41.341 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:41.341 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:41.341 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:41.341 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:41.341 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:41.341 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:41.341 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:41.341 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:41.341 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:41.341 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:41.341 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:41.341 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:41.341 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:41.341 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +2022-12-04T05:49:41.341Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:41.341Z [INFO] agent.router.manager: shutting down +49:41.341 [INFO] TestAgent: consul server down +49:41.341 [INFO] TestAgent: shutdown complete +49:41.341 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23606 network=tcp +49:41.341 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23606 network=udp +49:41.341 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23607 network=tcp +49:41.844 [INFO] TestAgent: Waiting for endpoints to shut down +49:41.844 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Legacy_UpdateUpsert (0.88s) +=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{4_4_4_4_4_4_4_4_4_4_4} +2022-12-04T05:49:40.477Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:40.487Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:40.489 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d9cafbf2-ecc2-83a7-84c4-257f8d0886ca Address:127.0.0.1:23617}]" +49:40.489 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23617 [Follower]" leader= +49:40.489 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d9cafbf2-ecc2-83a7-84c4-257f8d0886ca.dc2 127.0.0.1 +49:40.489 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d9cafbf2-ecc2-83a7-84c4-257f8d0886ca 127.0.0.1 +2022-12-04T05:49:40.489Z [INFO] agent.router: Initializing LAN area manager +49:40.490 [INFO] TestAgent.server: Adding LAN server: server="Node-d9cafbf2-ecc2-83a7-84c4-257f8d0886ca (Addr: tcp/127.0.0.1:23617) (DC: dc2)" +49:40.490 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d9cafbf2-ecc2-83a7-84c4-257f8d0886ca.dc2 area=wan +49:40.490 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23570 network=udp +49:40.490 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23570 network=tcp +49:40.490 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23571 network=tcp +49:40.490 [INFO] TestAgent: started state syncer +49:40.490 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23583 network=tcp +49:40.540 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false +49:40.550 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:40.551 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23617 [Candidate]" term=2 +49:40.551 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:40.551 [DEBUG] TestAgent.server.raft: vote granted: from=d9cafbf2-ecc2-83a7-84c4-257f8d0886ca term=2 tally=1 +49:40.551 [INFO] TestAgent.server.raft: election won: tally=1 +49:40.551 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23617 [Leader]" +49:40.551 [INFO] TestAgent.server: cluster leadership acquired +49:40.551 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false +49:40.551 [INFO] TestAgent.leader: started routine: routine="legacy ACL replication" +49:40.551 [INFO] TestAgent.server: started legacy ACL replication +49:40.551 [WARN] TestAgent.server.connect: primary datacenter is configured but unreachable - deferring initialization of the secondary datacenter CA +49:40.551 [INFO] TestAgent.leader: started routine: routine="config entry replication" +49:40.551 [INFO] TestAgent.leader: started routine: routine="federation state replication" +49:40.551 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:40.551 [INFO] TestAgent.leader: started routine: routine="secondary CA roots watch" +49:40.551 [INFO] TestAgent.leader: started routine: routine="intention replication" +49:40.551 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:40.551 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:40.551 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d9cafbf2-ecc2-83a7-84c4-257f8d0886ca +49:40.551 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d9cafbf2-ecc2-83a7-84c4-257f8d0886ca +49:40.551 [DEBUG] TestAgent.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 +49:40.551 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=Intention.List +49:40.551 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc +49:40.551 [DEBUG] TestAgent.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 +49:40.551 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConnectCA.Roots +49:40.551 [ERROR] TestAgent.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: No path to datacenter" +49:40.551 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc +49:40.551 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc +49:40.551 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc +49:40.551 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc +49:40.551 [DEBUG] TestAgent.server: federation states are not enabled in the primary dc +49:40.551 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConfigEntry.ListAll +49:40.551 [ERROR] TestAgent.server.connect: error replicating intentions: routine="intention replication" error="No path to datacenter" +49:40.551 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List +49:40.552 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" +49:40.552 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List +49:40.552 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" +49:40.552 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List +49:40.552 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" +49:40.552 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List +49:40.552 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" +49:40.551 [INFO] TestAgent.server: New leader elected: payload=Node-d9cafbf2-ecc2-83a7-84c4-257f8d0886ca +49:40.552 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.List +49:40.552 [WARN] TestAgent.server.replication.acl.legacy: Legacy ACL replication error (will retry if still leader): error="failed to retrieve remote ACLs: No path to datacenter" +49:40.640 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false +49:40.651 [WARN] TestAgent.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ACL.GetPolicy +49:40.651 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d9cafbf2-ecc2-83a7-84c4-257f8d0886ca +49:40.651 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:23567] +49:40.651 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:23567 +49:40.652 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5847e3d6-832b-132b-bb94-24dc9adb05c6.dc1 127.0.0.1 +49:40.652 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +49:40.652 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5847e3d6-832b-132b-bb94-24dc9adb05c6.dc1 area=wan +49:40.717 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:40.718 [INFO] TestAgent: Synced node info +49:40.840 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL replication" +49:40.840 [INFO] TestAgent.leader: started routine: routine="ACL policy replication" +49:40.840 [INFO] TestAgent.leader: started routine: routine="ACL role replication" +49:40.840 [INFO] TestAgent.leader: started routine: routine="ACL token replication" +49:40.840 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:40.840 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +49:40.840 [INFO] TestAgent.server.replication.acl.policy: started ACL Policy replication +49:40.840 [INFO] TestAgent.server.replication.acl.role: started ACL Role replication +49:40.840 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL replication" +49:40.840 [INFO] TestAgent.server.replication.acl.token: started ACL Token replication +49:40.840 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d9cafbf2-ecc2-83a7-84c4-257f8d0886ca +49:40.841 [INFO] TestAgent.server: Updating LAN server: server="Node-d9cafbf2-ecc2-83a7-84c4-257f8d0886ca (Addr: tcp/127.0.0.1:23617) (DC: dc2)" +49:40.841 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d9cafbf2-ecc2-83a7-84c4-257f8d0886ca +49:40.841 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d9cafbf2-ecc2-83a7-84c4-257f8d0886ca.dc2 +49:40.841 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d9cafbf2-ecc2-83a7-84c4-257f8d0886ca.dc2 area=wan +49:40.841 [DEBUG] TestAgent.server.replication.acl.role: finished fetching acls: amount=0 +49:40.841 [DEBUG] TestAgent.server.replication.acl.role: acl replication: local=0 remote=0 +49:40.841 [DEBUG] TestAgent.server.replication.acl.role: acl replication: deletions=0 updates=0 +49:40.841 [DEBUG] TestAgent.server.replication.acl.role: ACL replication completed through remote index: index=1 +49:40.841 [DEBUG] TestAgent.server.replication.acl.policy: finished fetching acls: amount=2 +49:40.841 [DEBUG] TestAgent.server.replication.acl.policy: acl replication: local=0 remote=2 +49:40.841 [DEBUG] TestAgent.server.replication.acl.policy: acl replication: deletions=0 updates=2 +49:40.841 [DEBUG] TestAgent.server.replication.acl.token: finished fetching acls: amount=3 +49:40.841 [DEBUG] TestAgent.server.replication.acl.token: acl replication: local=0 remote=3 +49:40.841 [DEBUG] TestAgent.server.replication.acl.token: acl replication: deletions=0 updates=3 +49:40.845 [DEBUG] TestAgent.server.replication.acl.token: acl replication - downloaded updates: amount=3 +49:40.845 [DEBUG] TestAgent.server.replication.acl.token: acl replication - performing updates +49:40.846 [DEBUG] TestAgent.server.replication.acl.token: acl replication - upserted batch: number_upserted=3 batch_size=442 +49:40.846 [DEBUG] TestAgent.server.replication.acl.token: acl replication - finished updates +49:40.846 [DEBUG] TestAgent.server.replication.acl.token: ACL replication completed through remote index: index=16 +49:40.846 [DEBUG] TestAgent.server.replication.acl.policy: acl replication - downloaded updates: amount=2 +49:40.846 [DEBUG] TestAgent.server.replication.acl.policy: acl replication - performing updates +49:40.846 [DEBUG] TestAgent.server.replication.acl.policy: acl replication - upserted batch: number_upserted=2 batch_size=675 +49:40.846 [DEBUG] TestAgent.server.replication.acl.policy: acl replication - finished updates +49:40.846 [DEBUG] TestAgent.server.replication.acl.policy: ACL replication completed through remote index: index=15 +49:40.864 [INFO] TestAgent: Requesting shutdown +49:40.864 [INFO] TestAgent.server: shutting down server +49:40.864 [DEBUG] TestAgent.leader: stopping routine: routine="federation state replication" +49:40.864 [DEBUG] TestAgent.leader: stopping routine: routine="ACL policy replication" +49:40.864 [DEBUG] TestAgent.leader: stopping routine: routine="config entry replication" +49:40.864 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:40.864 [DEBUG] TestAgent.leader: stopping routine: routine="secondary CA roots watch" +49:40.864 [DEBUG] TestAgent.leader: stopping routine: routine="intention replication" +49:40.864 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:40.864 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:40.864 [DEBUG] TestAgent.leader: stopping routine: routine="ACL role replication" +49:40.864 [DEBUG] TestAgent.leader: stopping routine: routine="ACL token replication" +49:40.864 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:40.864 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:40.864 [INFO] TestAgent.replication.federation_state: stopped replication +49:40.864 [DEBUG] TestAgent.leader: stopped routine: routine="federation state replication" +49:40.864 [INFO] TestAgent.server.replication.config_entry: stopped replication +49:40.864 [DEBUG] TestAgent.leader: stopped routine: routine="config entry replication" +49:40.864 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:40.864 [DEBUG] TestAgent.leader: stopped routine: routine="secondary CA roots watch" +49:40.864 [DEBUG] TestAgent.leader: stopped routine: routine="intention replication" +49:40.864 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:40.864 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:40.864 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:40.864 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:40.865Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:40.865Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:40.865Z [INFO] agent.router.manager: shutting down +49:40.865 [INFO] TestAgent: consul server down +49:40.865 [INFO] TestAgent: shutdown complete +49:40.865 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23570 network=tcp +49:40.865 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23570 network=udp +49:40.865 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23571 network=tcp +49:40.865 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:23596 datacenter=dc1 method=ACL.PolicyList error="rpc error making call: EOF" +49:40.865 [WARN] TestAgent.server.replication.acl.policy: ACL replication error (will retry if still leader): error="failed to retrieve remote ACL policies: rpc error making call: EOF" +49:40.865 [DEBUG] TestAgent.leader: stopped routine: routine="ACL policy replication" +49:40.865 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:23596 datacenter=dc1 method=ACL.TokenList error="rpc error making call: EOF" +49:40.865 [ERROR] TestAgent.server.rpc: RPC failed to server in DC: server=127.0.0.1:23596 datacenter=dc1 method=ACL.RoleList error="rpc error making call: EOF" +49:40.865 [WARN] TestAgent.server.replication.acl.role: ACL replication error (will retry if still leader): error="failed to retrieve remote ACL roles: rpc error making call: EOF" +49:40.865 [DEBUG] TestAgent.leader: stopped routine: routine="ACL role replication" +49:40.865 [WARN] TestAgent.server.replication.acl.token: ACL replication error (will retry if still leader): error="failed to retrieve remote ACL tokens: rpc error making call: EOF" +49:40.865 [DEBUG] TestAgent.leader: stopped routine: routine="ACL token replication" +49:41.365 [INFO] TestAgent: Waiting for endpoints to shut down +49:41.365 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:40.198Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:40.207Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:40.209 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5847e3d6-832b-132b-bb94-24dc9adb05c6 Address:127.0.0.1:23596}]" +49:40.209 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23596 [Follower]" leader= +49:40.209 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5847e3d6-832b-132b-bb94-24dc9adb05c6.dc1 127.0.0.1 +49:40.210 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5847e3d6-832b-132b-bb94-24dc9adb05c6 127.0.0.1 +2022-12-04T05:49:40.210Z [INFO] agent.router: Initializing LAN area manager +49:40.210 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23563 network=udp +49:40.210 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5847e3d6-832b-132b-bb94-24dc9adb05c6.dc1 area=wan +49:40.210 [INFO] TestAgent.server: Adding LAN server: server="Node-5847e3d6-832b-132b-bb94-24dc9adb05c6 (Addr: tcp/127.0.0.1:23596) (DC: dc1)" +49:40.210 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23563 network=tcp +49:40.210 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23564 network=tcp +49:40.210 [INFO] TestAgent: started state syncer +49:40.211 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23576 network=tcp +49:40.260 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:40.265 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:40.265 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23596 [Candidate]" term=2 +49:40.265 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:40.265 [DEBUG] TestAgent.server.raft: vote granted: from=5847e3d6-832b-132b-bb94-24dc9adb05c6 term=2 tally=1 +49:40.265 [INFO] TestAgent.server.raft: election won: tally=1 +49:40.265 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23596 [Leader]" +49:40.266 [INFO] TestAgent.server: cluster leadership acquired +49:40.266 [INFO] TestAgent.server: New leader elected: payload=Node-5847e3d6-832b-132b-bb94-24dc9adb05c6 +49:40.266 [INFO] TestAgent.server: initializing acls +49:40.266 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:40.266 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:40.267 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:40.267 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:40.267 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:40.267 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5847e3d6-832b-132b-bb94-24dc9adb05c6 +49:40.267 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5847e3d6-832b-132b-bb94-24dc9adb05c6.dc1 +49:40.267 [INFO] TestAgent.server: Updating LAN server: server="Node-5847e3d6-832b-132b-bb94-24dc9adb05c6 (Addr: tcp/127.0.0.1:23596) (DC: dc1)" +49:40.267 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5847e3d6-832b-132b-bb94-24dc9adb05c6.dc1 area=wan +49:40.267 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:40.269 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:40.269 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:40.269 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:40.269 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:40.269 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:40.269 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5847e3d6-832b-132b-bb94-24dc9adb05c6 +49:40.269 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5847e3d6-832b-132b-bb94-24dc9adb05c6 +49:40.269 [INFO] TestAgent.server: federation state anti-entropy synced +49:40.269 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5847e3d6-832b-132b-bb94-24dc9adb05c6 +49:40.466 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5847e3d6-832b-132b-bb94-24dc9adb05c6 +49:40.495 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:40.495 [INFO] TestAgent: Synced node info +49:40.651 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:41682 +49:40.652 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d9cafbf2-ecc2-83a7-84c4-257f8d0886ca.dc2 127.0.0.1 +49:40.652 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d9cafbf2-ecc2-83a7-84c4-257f8d0886ca.dc2 area=wan +49:41.365 [INFO] TestAgent: Requesting shutdown +49:41.365 [INFO] TestAgent.server: shutting down server +49:41.365 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:41.365 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:41.365 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:41.365 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:41.365 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:41.365 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:41.365 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:41.365 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:41.365 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:41.365 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:41.365 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:41.365 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:41.365 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:41.365 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:41.365 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +2022-12-04T05:49:41.365Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:41.365Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:41.365Z [INFO] agent.router.manager: shutting down +49:41.366 [INFO] TestAgent: consul server down +49:41.366 [INFO] TestAgent: shutdown complete +49:41.366 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23563 network=tcp +49:41.366 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23563 network=udp +49:41.366 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23564 network=tcp +49:41.866 [INFO] TestAgent: Waiting for endpoints to shut down +49:41.866 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Authorize (1.68s) + --- PASS: TestACL_Authorize/master-token (0.00s) + --- PASS: TestACL_Authorize/master-token/dc1 (0.00s) + --- PASS: TestACL_Authorize/master-token/dc2 (0.00s) + --- PASS: TestACL_Authorize/custom-token (0.00s) + --- PASS: TestACL_Authorize/custom-token/dc1 (0.00s) + --- PASS: TestACL_Authorize/custom-token/dc2 (0.00s) + --- PASS: TestACL_Authorize/too-many-requests (0.00s) + --- PASS: TestACL_Authorize/decode-failure (0.00s) + --- PASS: TestACL_Authorize/acl-not-found (0.00s) + --- PASS: TestACL_Authorize/local-token-in-secondary-dc (0.00s) + --- PASS: TestACL_Authorize/local-token-wrong-dc (0.00s) +=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{4_4_4_4_4_4_4_4_4_4_4} +2022-12-04T05:49:41.117Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:41.117Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:41.126Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:41.126Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:41.128 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5229a5d8-435c-894d-9a1b-c01443eb3eba Address:127.0.0.1:23827}]" +49:41.128 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23827 [Follower]" leader= +49:41.128 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5229a5d8-435c-894d-9a1b-c01443eb3eba.dc1 127.0.0.1 +49:41.129 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5229a5d8-435c-894d-9a1b-c01443eb3eba 127.0.0.1 +2022-12-04T05:49:41.129Z [INFO] agent.router: Initializing LAN area manager +49:41.129 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5229a5d8-435c-894d-9a1b-c01443eb3eba.dc1 area=wan +49:41.129 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23648 network=udp +49:41.129 [INFO] TestAgent.server: Adding LAN server: server="Node-5229a5d8-435c-894d-9a1b-c01443eb3eba (Addr: tcp/127.0.0.1:23827) (DC: dc1)" +49:41.129 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23648 network=tcp +49:41.129 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23649 network=tcp +49:41.129 [INFO] TestAgent: started state syncer +49:41.129 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23703 network=tcp +49:41.180 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:41.194 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:41.194 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23827 [Candidate]" term=2 +49:41.194 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:41.194 [DEBUG] TestAgent.server.raft: vote granted: from=5229a5d8-435c-894d-9a1b-c01443eb3eba term=2 tally=1 +49:41.194 [INFO] TestAgent.server.raft: election won: tally=1 +49:41.194 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23827 [Leader]" +49:41.195 [INFO] TestAgent.server: cluster leadership acquired +49:41.195 [INFO] TestAgent.server: initializing acls +49:41.195 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:41.195 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:41.195 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:41.196 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:41.196 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:41.196 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:41.196 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5229a5d8-435c-894d-9a1b-c01443eb3eba +49:41.196 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5229a5d8-435c-894d-9a1b-c01443eb3eba.dc1 +49:41.196 [INFO] TestAgent.server: New leader elected: payload=Node-5229a5d8-435c-894d-9a1b-c01443eb3eba +49:41.196 [INFO] TestAgent.server: Updating LAN server: server="Node-5229a5d8-435c-894d-9a1b-c01443eb3eba (Addr: tcp/127.0.0.1:23827) (DC: dc1)" +49:41.196 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5229a5d8-435c-894d-9a1b-c01443eb3eba.dc1 area=wan +49:41.197 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:41.198 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:41.198 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:41.198 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:41.198 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:41.198 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:41.198 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5229a5d8-435c-894d-9a1b-c01443eb3eba +49:41.198 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5229a5d8-435c-894d-9a1b-c01443eb3eba +49:41.198 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5229a5d8-435c-894d-9a1b-c01443eb3eba +49:41.199 [INFO] TestAgent.server: federation state anti-entropy synced +49:41.385 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:41.385 [INFO] TestAgent: Synced node info +49:41.385 [DEBUG] TestAgent: Node info in sync +49:41.443 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5229a5d8-435c-894d-9a1b-c01443eb3eba +49:41.443 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5229a5d8-435c-894d-9a1b-c01443eb3eba +49:41.444 [INFO] TestAgent: Requesting shutdown +49:41.444 [INFO] TestAgent.server: shutting down server +49:41.444 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:41.444 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:41.444 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:41.444 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:41.444 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:41.444 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:41.444 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:41.444 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:41.444 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:41.444 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:41.444 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:41.444 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:41.444 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:41.444 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:41.444 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:41.444Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:41.444Z [INFO] agent.router.manager: shutting down +49:41.444 [INFO] TestAgent: consul server down +49:41.444 [INFO] TestAgent: shutdown complete +49:41.444 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23648 network=tcp +49:41.444 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23648 network=udp +49:41.444 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23649 network=tcp +49:41.944 [INFO] TestAgent: Waiting for endpoints to shut down +49:41.944 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Legacy_Update (0.84s) +=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{3_3_3_3_3_3_3_3_3_3_3} [WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:36.367Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:36.367Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:36.449Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:36.449Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:36.455 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:604e5cb9-2859-3b4d-9005-7dcfbaef333c Address:127.0.0.1:16177}]" -13:36.456 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-604e5cb9-2859-3b4d-9005-7dcfbaef333c.dc1 127.0.0.1 -13:36.456 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-604e5cb9-2859-3b4d-9005-7dcfbaef333c 127.0.0.1 -2021-10-31T23:13:36.456Z [INFO] agent.router: Initializing LAN area manager -13:36.457 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16151 network=udp -13:36.457 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16177 [Follower]" leader= -13:36.457 [INFO] TestAgent.server: Adding LAN server: server="Node-604e5cb9-2859-3b4d-9005-7dcfbaef333c (Addr: tcp/127.0.0.1:16177) (DC: dc1)" -13:36.457 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-604e5cb9-2859-3b4d-9005-7dcfbaef333c.dc1 area=wan -13:36.457 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16151 network=tcp -13:36.466 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16152 network=tcp -13:36.466 [INFO] TestAgent: started state syncer -13:36.466 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16164 network=tcp -13:36.510 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:36.510 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16177 [Candidate]" term=2 -13:36.510 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:36.510 [DEBUG] TestAgent.server.raft: vote granted: from=604e5cb9-2859-3b4d-9005-7dcfbaef333c term=2 tally=1 -13:36.510 [INFO] TestAgent.server.raft: election won: tally=1 -13:36.510 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16177 [Leader]" -13:36.510 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=2 mode=2 found=true leader=127.0.0.1:16177 -13:36.510 [INFO] TestAgent.server: cluster leadership acquired -13:36.511 [INFO] TestAgent.server: initializing acls -13:36.512 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:36.512 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:36.512 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:36.513 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:36.513 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:36.513 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:36.513 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-604e5cb9-2859-3b4d-9005-7dcfbaef333c -13:36.513 [INFO] TestAgent.server: New leader elected: payload=Node-604e5cb9-2859-3b4d-9005-7dcfbaef333c -13:36.514 [INFO] TestAgent.server: Updating LAN server: server="Node-604e5cb9-2859-3b4d-9005-7dcfbaef333c (Addr: tcp/127.0.0.1:16177) (DC: dc1)" -13:36.522 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-604e5cb9-2859-3b4d-9005-7dcfbaef333c.dc1 -13:36.523 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:36.525 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:36.525 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:36.525 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:36.525 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:36.525 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:36.525 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-604e5cb9-2859-3b4d-9005-7dcfbaef333c -13:36.525 [INFO] TestAgent.server: member joined, marking health alive: member=Node-604e5cb9-2859-3b4d-9005-7dcfbaef333c -13:36.526 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-604e5cb9-2859-3b4d-9005-7dcfbaef333c.dc1 area=wan -13:36.542 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-604e5cb9-2859-3b4d-9005-7dcfbaef333c -13:36.544 [INFO] TestAgent.server: federation state anti-entropy synced -13:36.761 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:36.762 [INFO] TestAgent: Synced node info -13:36.762 [DEBUG] TestAgent: Node info in sync -13:36.842 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-604e5cb9-2859-3b4d-9005-7dcfbaef333c -13:36.842 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-604e5cb9-2859-3b4d-9005-7dcfbaef333c -13:36.842 [INFO] TestAgent: Requesting shutdown -13:36.842 [INFO] TestAgent.server: shutting down server -13:36.842 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:36.842 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:36.842 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:36.842 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:36.842 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:36.842 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:36.842 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:36.845 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:36.845 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:36.846 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:36.846 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:36.846 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:36.846 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:36.846 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:36.846 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -2021-10-31T23:13:36.846Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:36.846Z [INFO] agent.router.manager: shutting down -13:36.846 [INFO] TestAgent: consul server down -13:36.846 [INFO] TestAgent: shutdown complete -13:36.846 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16151 network=tcp -13:36.846 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16151 network=udp -13:36.846 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16152 network=tcp -13:37.349 [INFO] TestAgent: Waiting for endpoints to shut down -13:37.350 [INFO] TestAgent: Endpoints down -=== RUN TestACL_Legacy_Get/right_id -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:36.148Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:36.176Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:36.180 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2d647e11-7efc-e389-3db9-821735db4048 Address:127.0.0.1:16226}]" -13:36.181 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2d647e11-7efc-e389-3db9-821735db4048.secondary 127.0.0.1 -13:36.182 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2d647e11-7efc-e389-3db9-821735db4048 127.0.0.1 -2021-10-31T23:13:36.182Z [INFO] agent.router: Initializing LAN area manager -13:36.182 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17059 network=udp -13:36.182 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16226 [Follower]" leader= -13:36.182 [INFO] TestAgent.server: Adding LAN server: server="Node-2d647e11-7efc-e389-3db9-821735db4048 (Addr: tcp/127.0.0.1:16226) (DC: secondary)" -13:36.182 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2d647e11-7efc-e389-3db9-821735db4048.secondary area=wan -13:36.182 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17059 network=tcp -13:36.183 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17060 network=tcp -13:36.183 [INFO] TestAgent: started state syncer -13:36.183 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16241 network=tcp -13:36.221 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:36.221 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16226 [Candidate]" term=2 -13:36.222 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:36.222 [DEBUG] TestAgent.server.raft: vote granted: from=2d647e11-7efc-e389-3db9-821735db4048 term=2 tally=1 -13:36.222 [INFO] TestAgent.server.raft: election won: tally=1 -13:36.222 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16226 [Leader]" -13:36.222 [INFO] TestAgent.server: cluster leadership acquired -13:36.222 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16226 -13:36.223 [INFO] TestAgent.server: New leader elected: payload=Node-2d647e11-7efc-e389-3db9-821735db4048 -13:36.223 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:36.225 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:36.225 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:36.225 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:36.225 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:36.225 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:36.225 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2d647e11-7efc-e389-3db9-821735db4048 -13:36.225 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2d647e11-7efc-e389-3db9-821735db4048 -13:36.230 [INFO] TestAgent.server: federation state anti-entropy synced -13:36.358 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:17112] -13:36.359 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:17112 -13:36.360 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-443df186-bf17-54c8-6067-ba60d6b06807.primary 127.0.0.1 -13:36.360 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -13:36.361 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-443df186-bf17-54c8-6067-ba60d6b06807.primary area=wan -13:36.419 [INFO] TestAgent: Requesting shutdown -13:36.419 [INFO] TestAgent.server: shutting down server -13:36.419 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:36.419 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:36.419 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:36.419 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:36.419 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:36.419 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:36.419 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:36.419 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:36.419 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:36.419 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:36.419 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:36.419 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:36.419Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:36.419Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:36.419Z [INFO] agent.router.manager: shutting down -13:36.419 [INFO] TestAgent: consul server down -13:36.419 [INFO] TestAgent: shutdown complete -13:36.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17059 network=tcp -13:36.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17059 network=udp -13:36.419 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17060 network=tcp -13:36.938 [INFO] TestAgent: Waiting for endpoints to shut down -13:36.938 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:35.876Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:35.938Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:35.946 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:443df186-bf17-54c8-6067-ba60d6b06807 Address:127.0.0.1:17106}]" -13:35.947 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-443df186-bf17-54c8-6067-ba60d6b06807.primary 127.0.0.1 -13:35.947 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-443df186-bf17-54c8-6067-ba60d6b06807 127.0.0.1 -2021-10-31T23:13:35.947Z [INFO] agent.router: Initializing LAN area manager -13:35.947 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17101 network=udp -13:35.947 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17106 [Follower]" leader= -13:35.947 [INFO] TestAgent.server: Adding LAN server: server="Node-443df186-bf17-54c8-6067-ba60d6b06807 (Addr: tcp/127.0.0.1:17106) (DC: primary)" -13:35.948 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-443df186-bf17-54c8-6067-ba60d6b06807.primary area=wan -13:35.948 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17101 network=tcp -13:35.948 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17102 network=tcp -13:35.948 [INFO] TestAgent: started state syncer -13:35.948 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17107 network=tcp -13:36.002 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:36.002 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17106 [Candidate]" term=2 -13:36.002 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:36.002 [DEBUG] TestAgent.server.raft: vote granted: from=443df186-bf17-54c8-6067-ba60d6b06807 term=2 tally=1 -13:36.002 [INFO] TestAgent.server.raft: election won: tally=1 -13:36.002 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17106 [Leader]" -13:36.002 [INFO] TestAgent.server: cluster leadership acquired -13:36.002 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17106 -13:36.002 [INFO] TestAgent.server: New leader elected: payload=Node-443df186-bf17-54c8-6067-ba60d6b06807 -13:36.003 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:36.018 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:36.018 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:36.018 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:36.018 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:36.018 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:36.018 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-443df186-bf17-54c8-6067-ba60d6b06807 -13:36.018 [INFO] TestAgent.server: member joined, marking health alive: member=Node-443df186-bf17-54c8-6067-ba60d6b06807 -13:36.019 [INFO] TestAgent.server: federation state anti-entropy synced -13:36.067 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:36.067 [INFO] TestAgent: Synced node info -13:36.359 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:59356 -13:36.360 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2d647e11-7efc-e389-3db9-821735db4048.secondary 127.0.0.1 -13:36.360 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2d647e11-7efc-e389-3db9-821735db4048.secondary area=wan -13:36.366 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:36.366 [DEBUG] TestAgent: Node info in sync -13:36.366 [DEBUG] TestAgent: Node info in sync -13:36.938 [INFO] TestAgent: Requesting shutdown -13:36.938 [INFO] TestAgent.server: shutting down server -13:36.938 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:36.938 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:36.938 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:36.938 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:36.938 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:36.938 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:36.938 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:36.938 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:36.938 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:36.938 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:36.938 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:36.938Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:36.938Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:36.938Z [INFO] agent.router.manager: shutting down -13:36.939 [INFO] TestAgent: consul server down -13:36.939 [INFO] TestAgent: shutdown complete -13:36.939 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17101 network=tcp -13:36.939 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17101 network=udp -13:36.939 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17102 network=tcp -13:37.441 [INFO] TestAgent: Waiting for endpoints to shut down -13:37.441 [INFO] TestAgent: Endpoints down ---- PASS: TestStatusPeersSecondary (1.63s) -=== CONT TestSessionCustomTTL -=== RUN TestSessionCreate_NoCheck/no_check_fields_should_yield_default_serfHealth -=== RUN TestSessionCreate_NoCheck/overwrite_nodechecks_to_associate_with_no_checks -=== RUN TestSessionCreate_NoCheck/overwrite_checks_to_associate_with_no_checks +2022-12-04T05:49:41.381Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:41.392Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:41.394 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7609f0e0-5860-d51b-c799-11da943993fd Address:127.0.0.1:23751}]" +49:41.394 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23751 [Follower]" leader= +49:41.394 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:41.395 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:41.395Z [INFO] agent.router: Initializing LAN area manager +49:41.395 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23751) (DC: dc1)" +49:41.395 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:41.395 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23704 network=udp +49:41.395 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23704 network=tcp +49:41.396 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23705 network=tcp +49:41.396 [INFO] TestAgent: started state syncer +49:41.396 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23772 network=tcp +49:41.464 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:41.464 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23751 [Candidate]" term=2 +49:41.465 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:41.465 [DEBUG] TestAgent.server.raft: vote granted: from=7609f0e0-5860-d51b-c799-11da943993fd term=2 tally=1 +49:41.465 [INFO] TestAgent.server.raft: election won: tally=1 +49:41.465 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23751 [Leader]" +49:41.465 [INFO] TestAgent.server: cluster leadership acquired +49:41.465 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23751 +49:41.466 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:41.467 [INFO] TestAgent.server: New leader elected: payload=test-node +49:41.468 [INFO] TestAgent: Synced node info +49:41.472 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:41.472 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:41.472 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:41.472 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:41.472 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:41.472 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:41.472 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:41.473 [INFO] TestAgent.server: federation state anti-entropy synced +49:41.617 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=983.08µs client=127.0.0.1:43502 client_network=udp +49:41.617 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=729.454µs client=127.0.0.1:34995 client_network=udp +49:41.618 [DEBUG] TestAgent.dns: request served from client: name=22fe4b5d-0773-a3ae-3860-ca41dc6d7761.query.consul. type=A class=IN latency=749.747µs client=127.0.0.1:55087 client_network=udp +49:41.618 [INFO] TestAgent: Requesting shutdown +49:41.618 [INFO] TestAgent.server: shutting down server +49:41.618 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:41.619 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:41.619 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:41.619 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:41.619 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:41.619 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:41.619 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:41.619 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:41.619 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:41.619 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:41.619 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:41.619Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:41.619Z [INFO] agent.router.manager: shutting down +49:41.619 [INFO] TestAgent: consul server down +49:41.619 [INFO] TestAgent: shutdown complete +49:41.619 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23704 network=tcp +49:41.619 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23704 network=udp +49:41.619 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23705 network=tcp +49:42.119 [INFO] TestAgent: Waiting for endpoints to shut down +49:42.119 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{3_3_3_3_3_3_3_3_3_3_3} +2022-12-04T05:49:41.383Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:41.396Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:41.398 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:298111c0-63ec-86f5-ea50-2df0185fc46d Address:127.0.0.1:23632}]" +49:41.398 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23632 [Follower]" leader= +49:41.398 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:41.399 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:41.399Z [INFO] agent.router: Initializing LAN area manager +49:41.399 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23632) (DC: dc1)" +49:41.399 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:41.399 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23773 network=udp +49:41.399 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23773 network=tcp +49:41.400 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23774 network=tcp +49:41.400 [INFO] TestAgent: started state syncer +49:41.400 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23612 network=tcp +49:41.439 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:41.439 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23632 [Candidate]" term=2 +49:41.439 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:41.439 [DEBUG] TestAgent.server.raft: vote granted: from=298111c0-63ec-86f5-ea50-2df0185fc46d term=2 tally=1 +49:41.439 [INFO] TestAgent.server.raft: election won: tally=1 +49:41.439 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23632 [Leader]" +49:41.440 [INFO] TestAgent.server: cluster leadership acquired +49:41.440 [INFO] TestAgent.server: New leader elected: payload=test-node +49:41.440 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23632 +49:41.441 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:41.442 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:41.442 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:41.442 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:41.442 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:41.442 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:41.442 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:41.442 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:41.443 [INFO] TestAgent.server: federation state anti-entropy synced +49:41.722 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:41.723 [INFO] TestAgent: Synced node info +49:41.756 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=917.397µs client=127.0.0.1:60860 client_network=udp +49:41.757 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=690.504µs client=127.0.0.1:38527 client_network=udp +49:41.758 [DEBUG] TestAgent.dns: request served from client: name=6b125c8e-950f-cb29-f218-b35545a03ddb.query.consul. type=A class=IN latency=728.66µs client=127.0.0.1:53593 client_network=udp +49:41.758 [INFO] TestAgent: Requesting shutdown +49:41.758 [INFO] TestAgent.server: shutting down server +49:41.758 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:41.758 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:41.758 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:41.758 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:41.758 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:41.758 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:41.758 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:41.758 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:41.758 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:41.758 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:41.758 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:41.758Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:41.758Z [INFO] agent.router.manager: shutting down +49:41.759 [INFO] TestAgent: consul server down +49:41.759 [INFO] TestAgent: shutdown complete +49:41.759 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23773 network=tcp +49:41.759 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23773 network=udp +49:41.759 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23774 network=tcp +49:42.259 [INFO] TestAgent: Waiting for endpoints to shut down +49:42.259 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{3_3_3_3_3_3_3_3_3_3_3} [WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:36.908Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:36.999Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:37.004 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8563eb25-740a-4ca0-bbe4-41f19ca60f45 Address:127.0.0.1:16205}]" -13:37.004 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16205 [Follower]" leader= -13:37.004 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8563eb25-740a-4ca0-bbe4-41f19ca60f45.dc1 127.0.0.1 -13:37.005 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8563eb25-740a-4ca0-bbe4-41f19ca60f45 127.0.0.1 -2021-10-31T23:13:37.005Z [INFO] agent.router: Initializing LAN area manager -13:37.006 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16256 network=udp -13:37.012 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8563eb25-740a-4ca0-bbe4-41f19ca60f45.dc1 area=wan -13:37.065 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16256 network=tcp -13:37.066 [INFO] TestAgent.server: Adding LAN server: server="Node-8563eb25-740a-4ca0-bbe4-41f19ca60f45 (Addr: tcp/127.0.0.1:16205) (DC: dc1)" -13:37.066 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16257 network=tcp -13:37.066 [INFO] TestAgent: started state syncer -13:37.066 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16234 network=tcp -13:37.066 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:37.066 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16205 [Candidate]" term=2 -13:37.066 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:37.066 [DEBUG] TestAgent.server.raft: vote granted: from=8563eb25-740a-4ca0-bbe4-41f19ca60f45 term=2 tally=1 -13:37.066 [INFO] TestAgent.server.raft: election won: tally=1 -13:37.066 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16205 [Leader]" -13:37.067 [INFO] TestAgent.server: cluster leadership acquired -13:37.067 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16205 -13:37.068 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:37.069 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:37.069 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:37.069 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:37.069 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:37.069 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:37.069 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8563eb25-740a-4ca0-bbe4-41f19ca60f45 -13:37.069 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8563eb25-740a-4ca0-bbe4-41f19ca60f45 -13:37.070 [INFO] TestAgent.server: New leader elected: payload=Node-8563eb25-740a-4ca0-bbe4-41f19ca60f45 -13:37.074 [INFO] TestAgent.server: federation state anti-entropy synced -13:37.122 [INFO] TestAgent: Requesting shutdown -13:37.122 [INFO] TestAgent.server: shutting down server -13:37.122 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:37.122 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:37.122 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:37.122 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:37.122 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:37.125 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:37.125 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:37.125 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:37.126 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:37.126 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:37.126 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:37.126 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:37.126Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:37.126Z [INFO] agent.router.manager: shutting down -13:37.126 [INFO] TestAgent: consul server down -13:37.126 [INFO] TestAgent: shutdown complete -13:37.126 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16256 network=tcp -13:37.126 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16256 network=udp -13:37.126 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16257 network=tcp -13:37.626 [INFO] TestAgent: Waiting for endpoints to shut down -13:37.626 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_AutopilotSetConfiguration (0.76s) -=== CONT TestSessionDestroy -2021-10-31T23:13:36.882Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:36.980Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:36.999 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:98613dda-a3d2-33e6-aa74-bf54681af535 Address:127.0.0.1:16275}]" -13:36.999 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16275 [Follower]" leader= -13:36.999 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-98613dda-a3d2-33e6-aa74-bf54681af535.dc1 127.0.0.1 -13:37.000 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-98613dda-a3d2-33e6-aa74-bf54681af535 127.0.0.1 -2021-10-31T23:13:37.000Z [INFO] agent.router: Initializing LAN area manager -13:37.000 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16228 network=udp -13:37.001 [INFO] TestAgent.server: Adding LAN server: server="Node-98613dda-a3d2-33e6-aa74-bf54681af535 (Addr: tcp/127.0.0.1:16275) (DC: dc1)" -13:37.001 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-98613dda-a3d2-33e6-aa74-bf54681af535.dc1 area=wan -13:37.001 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16228 network=tcp -13:37.016 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16229 network=tcp -13:37.016 [INFO] TestAgent: started state syncer -13:37.016 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16255 network=tcp -13:37.064 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:37.064 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16275 [Candidate]" term=2 -13:37.064 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:37.064 [DEBUG] TestAgent.server.raft: vote granted: from=98613dda-a3d2-33e6-aa74-bf54681af535 term=2 tally=1 -13:37.064 [INFO] TestAgent.server.raft: election won: tally=1 -13:37.064 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16275 [Leader]" -13:37.065 [INFO] TestAgent.server: cluster leadership acquired -13:37.065 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16275 -13:37.065 [INFO] TestAgent.server: New leader elected: payload=Node-98613dda-a3d2-33e6-aa74-bf54681af535 -13:37.066 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:37.067 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:37.067 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:37.067 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:37.067 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:37.067 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:37.068 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-98613dda-a3d2-33e6-aa74-bf54681af535 -13:37.068 [INFO] TestAgent.server: member joined, marking health alive: member=Node-98613dda-a3d2-33e6-aa74-bf54681af535 -13:37.068 [INFO] TestAgent.server: federation state anti-entropy synced -13:37.070 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:37.070 [INFO] TestAgent: Synced node info -13:37.150 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:37.150 [INFO] TestAgent: Requesting shutdown -13:37.150 [INFO] TestAgent.server: shutting down server -13:37.150 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:37.150 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:37.150 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:37.150 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:37.150 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:37.150 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:37.150 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:37.150 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:37.150 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:37.150 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:37.150 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:37.150Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:37.150Z [INFO] agent.router.manager: shutting down -13:37.150 [INFO] TestAgent: consul server down -13:37.150 [INFO] TestAgent: shutdown complete -13:37.150 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16228 network=tcp -13:37.150 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16228 network=udp -13:37.150 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16229 network=tcp -13:37.657 [INFO] TestAgent: Waiting for endpoints to shut down -13:37.657 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Execute/#04 -2021-10-31T23:13:36.773Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:36.856Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:36.858 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a46c7083-00eb-9987-5dcd-3afdad6e4682 Address:127.0.0.1:16240}]" -13:36.860 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16240 [Follower]" leader= -13:36.863 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a46c7083-00eb-9987-5dcd-3afdad6e4682.dc1 127.0.0.1 -13:36.864 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a46c7083-00eb-9987-5dcd-3afdad6e4682 127.0.0.1 -2021-10-31T23:13:36.864Z [INFO] agent.router: Initializing LAN area manager -13:36.864 [INFO] TestAgent.server: Adding LAN server: server="Node-a46c7083-00eb-9987-5dcd-3afdad6e4682 (Addr: tcp/127.0.0.1:16240) (DC: dc1)" -13:36.864 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16193 network=udp -13:36.864 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a46c7083-00eb-9987-5dcd-3afdad6e4682.dc1 area=wan -13:36.865 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16193 network=tcp -13:36.865 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16194 network=tcp -13:36.865 [INFO] TestAgent: started state syncer -13:36.865 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16227 network=tcp -13:36.928 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:36.928 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16240 [Candidate]" term=2 -13:36.928 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:36.929 [DEBUG] TestAgent.server.raft: vote granted: from=a46c7083-00eb-9987-5dcd-3afdad6e4682 term=2 tally=1 -13:36.929 [INFO] TestAgent.server.raft: election won: tally=1 -13:36.929 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16240 [Leader]" -13:36.934 [INFO] TestAgent.server: cluster leadership acquired -13:36.934 [INFO] TestAgent.server: New leader elected: payload=Node-a46c7083-00eb-9987-5dcd-3afdad6e4682 -13:36.945 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16240 -13:36.947 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:36.949 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:36.949 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:36.949 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:36.949 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:36.949 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:36.949 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a46c7083-00eb-9987-5dcd-3afdad6e4682 -13:36.949 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a46c7083-00eb-9987-5dcd-3afdad6e4682 -13:36.954 [INFO] TestAgent.server: federation state anti-entropy synced -13:37.014 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:37.016 [INFO] TestAgent: Synced node info -13:37.016 [DEBUG] TestAgent: Node info in sync -13:37.192 [INFO] TestAgent: Requesting shutdown -13:37.192 [INFO] TestAgent.server: shutting down server -13:37.192 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:37.192 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:37.192 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:37.192 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:37.192 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:37.192 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:37.192 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:37.192 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:37.192 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:37.192 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:37.192 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:37.192Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:37.192Z [INFO] agent.router.manager: shutting down -13:37.192 [INFO] TestAgent: consul server down -13:37.192 [INFO] TestAgent: shutdown complete -13:37.192 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16193 network=tcp -13:37.192 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16193 network=udp -13:37.192 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16194 network=tcp -13:37.693 [INFO] TestAgent: Waiting for endpoints to shut down -13:37.693 [INFO] TestAgent: Endpoints down -=== RUN TestTxnEndpoint_Bad_Size_Item/exceeds_default_max_kv_value_size -2021-10-31T23:13:36.306Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:36.415Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:36.417 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7c49023a-4ca3-95f0-65a1-086b69f3c466 Address:127.0.0.1:16247}]" -13:36.417 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16247 [Follower]" leader= -13:36.426 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7c49023a-4ca3-95f0-65a1-086b69f3c466.secondary 127.0.0.1 -13:36.427 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7c49023a-4ca3-95f0-65a1-086b69f3c466 127.0.0.1 -2021-10-31T23:13:36.427Z [INFO] agent.router: Initializing LAN area manager -13:36.427 [INFO] TestAgent.server: Adding LAN server: server="Node-7c49023a-4ca3-95f0-65a1-086b69f3c466 (Addr: tcp/127.0.0.1:16247) (DC: secondary)" -13:36.427 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7c49023a-4ca3-95f0-65a1-086b69f3c466.secondary area=wan -13:36.427 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16130 network=udp -13:36.427 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16130 network=tcp -13:36.428 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16131 network=tcp -13:36.428 [INFO] TestAgent: started state syncer -13:36.428 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16150 network=tcp -13:36.489 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:36.489 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16247 [Candidate]" term=2 -13:36.490 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:36.490 [DEBUG] TestAgent.server.raft: vote granted: from=7c49023a-4ca3-95f0-65a1-086b69f3c466 term=2 tally=1 -13:36.490 [INFO] TestAgent.server.raft: election won: tally=1 -13:36.490 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16247 [Leader]" -13:36.490 [INFO] TestAgent.server: cluster leadership acquired -13:36.490 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16247 -13:36.491 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:36.492 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:36.492 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:36.492 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:36.492 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:36.492 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:36.492 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7c49023a-4ca3-95f0-65a1-086b69f3c466 -13:36.492 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7c49023a-4ca3-95f0-65a1-086b69f3c466 -13:36.493 [INFO] TestAgent.server: New leader elected: payload=Node-7c49023a-4ca3-95f0-65a1-086b69f3c466 -13:36.493 [INFO] TestAgent.server: federation state anti-entropy synced -13:36.662 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:36.662 [INFO] TestAgent: Synced node info -13:36.662 [DEBUG] TestAgent: Node info in sync -13:36.723 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:17133] -13:36.723 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:17133 -13:36.724 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6f1e0823-73aa-a459-51db-a8cd871f61c5.primary 127.0.0.1 -13:36.725 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -13:36.725 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6f1e0823-73aa-a459-51db-a8cd871f61c5.primary area=wan -13:36.734 [INFO] TestAgent: Requesting shutdown -13:36.734 [INFO] TestAgent.server: shutting down server -13:36.734 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:36.734 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:36.734 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:36.734 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:36.734 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:36.735 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:36.735 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:36.735 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:36.735 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:36.735 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:36.735 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:36.735Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:36.735Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:36.735Z [INFO] agent.router.manager: shutting down -13:36.735 [INFO] TestAgent: consul server down -13:36.735 [INFO] TestAgent: shutdown complete -13:36.735 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16130 network=tcp -13:36.735 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16130 network=udp -13:36.735 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16131 network=tcp -13:37.240 [INFO] TestAgent: Waiting for endpoints to shut down -13:37.240 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:35.849Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:35.896Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:35.907 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6f1e0823-73aa-a459-51db-a8cd871f61c5 Address:127.0.0.1:17092}]" -13:35.907 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6f1e0823-73aa-a459-51db-a8cd871f61c5.primary 127.0.0.1 -13:35.908 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6f1e0823-73aa-a459-51db-a8cd871f61c5 127.0.0.1 -2021-10-31T23:13:35.908Z [INFO] agent.router: Initializing LAN area manager -13:35.908 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17108 network=udp -13:35.908 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17092 [Follower]" leader= -13:35.908 [INFO] TestAgent.server: Adding LAN server: server="Node-6f1e0823-73aa-a459-51db-a8cd871f61c5 (Addr: tcp/127.0.0.1:17092) (DC: primary)" -13:35.908 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6f1e0823-73aa-a459-51db-a8cd871f61c5.primary area=wan -13:35.909 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17108 network=tcp -13:35.909 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17109 network=tcp -13:35.909 [INFO] TestAgent: started state syncer -13:35.909 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17093 network=tcp -13:35.991 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:35.991 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17092 [Candidate]" term=2 -13:35.991 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:35.991 [DEBUG] TestAgent.server.raft: vote granted: from=6f1e0823-73aa-a459-51db-a8cd871f61c5 term=2 tally=1 -13:35.991 [INFO] TestAgent.server.raft: election won: tally=1 -13:35.991 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17092 [Leader]" -13:35.992 [INFO] TestAgent.server: cluster leadership acquired -13:35.992 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17092 -13:35.993 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:35.994 [INFO] TestAgent.server: New leader elected: payload=Node-6f1e0823-73aa-a459-51db-a8cd871f61c5 -13:35.997 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:35.997 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:35.997 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:35.997 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:35.997 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:35.997 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6f1e0823-73aa-a459-51db-a8cd871f61c5 -13:35.997 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6f1e0823-73aa-a459-51db-a8cd871f61c5 -13:35.998 [INFO] TestAgent.server: federation state anti-entropy synced -13:36.185 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:36.186 [INFO] TestAgent: Synced node info -13:36.186 [DEBUG] TestAgent: Node info in sync -13:36.723 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:58730 -13:36.724 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7c49023a-4ca3-95f0-65a1-086b69f3c466.secondary 127.0.0.1 -13:36.724 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7c49023a-4ca3-95f0-65a1-086b69f3c466.secondary area=wan -13:37.098 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:37.098 [DEBUG] TestAgent: Node info in sync -13:37.240 [INFO] TestAgent: Requesting shutdown -13:37.240 [INFO] TestAgent.server: shutting down server -13:37.240 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:37.240 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:37.240 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:37.240 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:37.240 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:37.240 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:37.240 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:37.240 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:37.240 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:37.240 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:37.240 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:37.240Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:37.240Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:37.240Z [INFO] agent.router.manager: shutting down -13:37.240 [INFO] TestAgent: consul server down -13:37.240 [INFO] TestAgent: shutdown complete -13:37.240 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17108 network=tcp -13:37.240 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17108 network=udp -13:37.240 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17109 network=tcp -13:37.757 [INFO] TestAgent: Waiting for endpoints to shut down -13:37.757 [INFO] TestAgent: Endpoints down ---- PASS: TestStatusLeaderSecondary (1.94s) -=== CONT TestKVSEndpoint_DELETE_CAS -2021-10-31T23:13:36.764Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:36.852Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:36.859 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b316ca9c-e578-adcc-eef1-39abad1bca66 Address:127.0.0.1:16184}]" -13:36.859 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b316ca9c-e578-adcc-eef1-39abad1bca66.dc1 127.0.0.1 -13:36.860 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b316ca9c-e578-adcc-eef1-39abad1bca66 127.0.0.1 -2021-10-31T23:13:36.860Z [INFO] agent.router: Initializing LAN area manager -13:36.860 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16277 network=udp -13:36.860 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16184 [Follower]" leader= -13:36.860 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b316ca9c-e578-adcc-eef1-39abad1bca66.dc1 area=wan -13:36.860 [INFO] TestAgent.server: Adding LAN server: server="Node-b316ca9c-e578-adcc-eef1-39abad1bca66 (Addr: tcp/127.0.0.1:16184) (DC: dc1)" -13:36.860 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16277 network=tcp -13:36.861 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16278 network=tcp -13:36.861 [INFO] TestAgent: started state syncer -13:36.861 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16192 network=tcp -13:36.935 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:36.935 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16184 [Candidate]" term=2 -13:36.935 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:36.935 [DEBUG] TestAgent.server.raft: vote granted: from=b316ca9c-e578-adcc-eef1-39abad1bca66 term=2 tally=1 -13:36.935 [INFO] TestAgent.server.raft: election won: tally=1 -13:36.935 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16184 [Leader]" -13:36.935 [INFO] TestAgent.server: cluster leadership acquired -13:36.936 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16184 -13:36.937 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:36.937 [INFO] TestAgent.server: New leader elected: payload=Node-b316ca9c-e578-adcc-eef1-39abad1bca66 -13:36.939 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:36.939 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:36.939 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:36.939 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:36.939 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:36.939 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b316ca9c-e578-adcc-eef1-39abad1bca66 -13:36.939 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b316ca9c-e578-adcc-eef1-39abad1bca66 -13:36.940 [INFO] TestAgent.server: federation state anti-entropy synced -13:37.070 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:37.070 [INFO] TestAgent: Synced node info -13:37.234 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -13:37.234 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-b316ca9c-e578-adcc-eef1-39abad1bca66 -13:37.235 [INFO] TestAgent: Requesting shutdown -13:37.235 [INFO] TestAgent.server: shutting down server -13:37.235 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:37.235 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:37.235 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:37.235 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:37.235 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:37.235 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:37.235 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:37.235 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:37.235 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:37.235 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:37.235 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:37.235Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:37.235Z [INFO] agent.router.manager: shutting down -13:37.235 [INFO] TestAgent: consul server down -13:37.235 [INFO] TestAgent: shutdown complete -13:37.235 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16277 network=tcp -13:37.235 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16277 network=udp -13:37.235 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16278 network=tcp -13:37.758 [INFO] TestAgent: Waiting for endpoints to shut down -13:37.758 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_Keyring_LocalOnly (1.10s) -=== CONT TestUiIndex -2021-10-31T23:13:36.725Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:36.725Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:36.833Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:36.833Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:36.842 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d3bbda0b-6dc0-855f-1a92-cc1cd69d9711 Address:127.0.0.1:16191}]" -13:36.843 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d3bbda0b-6dc0-855f-1a92-cc1cd69d9711.dc1 127.0.0.1 -13:36.843 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d3bbda0b-6dc0-855f-1a92-cc1cd69d9711 127.0.0.1 -2021-10-31T23:13:36.844Z [INFO] agent.router: Initializing LAN area manager -13:36.844 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16165 network=udp -13:36.844 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16191 [Follower]" leader= -13:36.844 [INFO] TestAgent.server: Adding LAN server: server="Node-d3bbda0b-6dc0-855f-1a92-cc1cd69d9711 (Addr: tcp/127.0.0.1:16191) (DC: dc1)" -13:36.844 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d3bbda0b-6dc0-855f-1a92-cc1cd69d9711.dc1 area=wan -13:36.844 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16165 network=tcp -13:36.845 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16166 network=tcp -13:36.845 [INFO] TestAgent: started state syncer -13:36.845 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16276 network=tcp -13:36.895 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:36.933 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:36.934 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16191 [Candidate]" term=2 -13:36.934 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:36.934 [DEBUG] TestAgent.server.raft: vote granted: from=d3bbda0b-6dc0-855f-1a92-cc1cd69d9711 term=2 tally=1 -13:36.934 [INFO] TestAgent.server.raft: election won: tally=1 -13:36.934 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16191 [Leader]" -13:36.934 [INFO] TestAgent.server: cluster leadership acquired -13:36.934 [INFO] TestAgent.server: initializing acls -13:36.935 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:36.935 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:36.935 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:36.936 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:36.936 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:36.936 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:36.936 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d3bbda0b-6dc0-855f-1a92-cc1cd69d9711 -13:36.936 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d3bbda0b-6dc0-855f-1a92-cc1cd69d9711.dc1 -13:36.937 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:36.938 [INFO] TestAgent.server: New leader elected: payload=Node-d3bbda0b-6dc0-855f-1a92-cc1cd69d9711 -13:36.938 [INFO] TestAgent.server: Updating LAN server: server="Node-d3bbda0b-6dc0-855f-1a92-cc1cd69d9711 (Addr: tcp/127.0.0.1:16191) (DC: dc1)" -13:36.939 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d3bbda0b-6dc0-855f-1a92-cc1cd69d9711.dc1 area=wan -13:36.944 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:36.944 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:36.944 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:36.944 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:36.944 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:36.944 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d3bbda0b-6dc0-855f-1a92-cc1cd69d9711 -13:36.944 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d3bbda0b-6dc0-855f-1a92-cc1cd69d9711 -13:36.945 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d3bbda0b-6dc0-855f-1a92-cc1cd69d9711 -13:36.962 [INFO] TestAgent.server: federation state anti-entropy synced -13:37.234 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:37.235 [INFO] TestAgent: Synced node info -13:37.257 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d3bbda0b-6dc0-855f-1a92-cc1cd69d9711 -13:37.258 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d3bbda0b-6dc0-855f-1a92-cc1cd69d9711 -13:37.265 [INFO] TestAgent: Requesting shutdown -13:37.265 [INFO] TestAgent.server: shutting down server -13:37.265 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:37.265 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:37.265 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:37.265 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:37.265 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:37.265 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:37.265 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:37.265 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:37.265 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:37.265 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:37.265 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:37.265 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:37.265 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:37.265 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:37.265 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:37.265Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:37.265Z [INFO] agent.router.manager: shutting down -13:37.265 [INFO] TestAgent: consul server down -13:37.265 [INFO] TestAgent: shutdown complete -13:37.265 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16165 network=tcp -13:37.265 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16165 network=udp -13:37.265 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16166 network=tcp -13:37.765 [INFO] TestAgent: Waiting for endpoints to shut down -13:37.765 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Legacy_Clone (1.11s) -=== CONT TestUiNodeInfo -2021-10-31T23:13:37.097Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:37.139Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:37.140 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:41262b33-aa52-8d55-13f9-3b1c4de6a075 Address:127.0.0.1:16561}]" -13:37.141 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-41262b33-aa52-8d55-13f9-3b1c4de6a075.dc1 127.0.0.1 -13:37.141 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-41262b33-aa52-8d55-13f9-3b1c4de6a075 127.0.0.1 -13:37.141 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16561 [Follower]" leader= -2021-10-31T23:13:37.153Z [INFO] agent.router: Initializing LAN area manager -13:37.154 [INFO] TestAgent.server: Adding LAN server: server="Node-41262b33-aa52-8d55-13f9-3b1c4de6a075 (Addr: tcp/127.0.0.1:16561) (DC: dc1)" -13:37.154 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-41262b33-aa52-8d55-13f9-3b1c4de6a075.dc1 area=wan -13:37.154 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16591 network=udp -13:37.154 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16591 network=tcp -13:37.155 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16592 network=tcp -13:37.155 [INFO] TestAgent: started state syncer -13:37.155 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16562 network=tcp -13:37.198 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:37.198 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16561 [Candidate]" term=2 -13:37.198 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:37.198 [DEBUG] TestAgent.server.raft: vote granted: from=41262b33-aa52-8d55-13f9-3b1c4de6a075 term=2 tally=1 -13:37.198 [INFO] TestAgent.server.raft: election won: tally=1 -13:37.198 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16561 [Leader]" -13:37.198 [INFO] TestAgent.server: cluster leadership acquired -13:37.198 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16561 -13:37.199 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:37.199 [INFO] TestAgent.server: New leader elected: payload=Node-41262b33-aa52-8d55-13f9-3b1c4de6a075 -13:37.201 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:37.201 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:37.201 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:37.201 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:37.201 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:37.201 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-41262b33-aa52-8d55-13f9-3b1c4de6a075 -13:37.201 [INFO] TestAgent.server: member joined, marking health alive: member=Node-41262b33-aa52-8d55-13f9-3b1c4de6a075 -13:37.202 [INFO] TestAgent.server: federation state anti-entropy synced -13:37.269 [INFO] TestAgent: Requesting shutdown -13:37.269 [INFO] TestAgent.server: shutting down server -13:37.269 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:37.269 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:37.269 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:37.269 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:37.269 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:37.269 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:37.270 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:37.269 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:37.269 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:37.269 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:37.269 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:37.270 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:37.270Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:37.270Z [INFO] agent.router.manager: shutting down -13:37.270 [INFO] TestAgent: consul server down -13:37.270 [INFO] TestAgent: shutdown complete -13:37.270 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16591 network=tcp -13:37.270 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16591 network=udp -13:37.270 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16592 network=tcp -13:37.774 [INFO] TestAgent: Waiting for endpoints to shut down -13:37.774 [INFO] TestAgent: Endpoints down -=== RUN TestSessionGet/#01 -[WARN] freeport: 10 out of 66 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:37.070Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:37.070Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:37.131Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:37.131Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:37.133 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9178b25e-6bfc-ca90-72c5-b19cad762275 Address:127.0.0.1:16282}]" -13:37.133 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9178b25e-6bfc-ca90-72c5-b19cad762275.dc1 127.0.0.1 -13:37.133 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16282 [Follower]" leader= -13:37.134 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9178b25e-6bfc-ca90-72c5-b19cad762275 127.0.0.1 -2021-10-31T23:13:37.134Z [INFO] agent.router: Initializing LAN area manager -13:37.134 [INFO] TestAgent.server: Adding LAN server: server="Node-9178b25e-6bfc-ca90-72c5-b19cad762275 (Addr: tcp/127.0.0.1:16282) (DC: dc1)" -13:37.134 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16263 network=udp -13:37.134 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9178b25e-6bfc-ca90-72c5-b19cad762275.dc1 area=wan -13:37.134 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16263 network=tcp -13:37.135 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16264 network=tcp -13:37.135 [INFO] TestAgent: started state syncer -13:37.135 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16269 network=tcp -13:37.185 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:37.201 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:37.201 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16282 [Candidate]" term=2 -13:37.202 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:37.202 [DEBUG] TestAgent.server.raft: vote granted: from=9178b25e-6bfc-ca90-72c5-b19cad762275 term=2 tally=1 -13:37.202 [INFO] TestAgent.server.raft: election won: tally=1 -13:37.202 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16282 [Leader]" -13:37.202 [INFO] TestAgent.server: cluster leadership acquired -13:37.202 [INFO] TestAgent.server: initializing acls -13:37.202 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:37.202 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:37.203 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:37.204 [INFO] TestAgent: Synced node info -13:37.204 [DEBUG] TestAgent: Node info in sync -13:37.204 [INFO] TestAgent.server: New leader elected: payload=Node-9178b25e-6bfc-ca90-72c5-b19cad762275 -13:37.204 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:37.204 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:37.204 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:37.204 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9178b25e-6bfc-ca90-72c5-b19cad762275 -13:37.204 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9178b25e-6bfc-ca90-72c5-b19cad762275.dc1 -13:37.205 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:37.205 [INFO] TestAgent.server: Updating LAN server: server="Node-9178b25e-6bfc-ca90-72c5-b19cad762275 (Addr: tcp/127.0.0.1:16282) (DC: dc1)" -13:37.205 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9178b25e-6bfc-ca90-72c5-b19cad762275.dc1 area=wan -13:37.213 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:37.213 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:37.213 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:37.213 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:37.213 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:37.213 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9178b25e-6bfc-ca90-72c5-b19cad762275 -13:37.213 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9178b25e-6bfc-ca90-72c5-b19cad762275 -13:37.214 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9178b25e-6bfc-ca90-72c5-b19cad762275 -13:37.214 [INFO] TestAgent.server: federation state anti-entropy synced -13:37.454 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9178b25e-6bfc-ca90-72c5-b19cad762275 -13:37.454 [INFO] TestAgent: Requesting shutdown -13:37.454 [INFO] TestAgent.server: shutting down server -13:37.454 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:37.454 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:37.454 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:37.454 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:37.454 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:37.454 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:37.454 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:37.454 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:37.454 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:37.454 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:37.454 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:37.454 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:37.454 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:37.454 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:37.454 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:37.454Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:37.455Z [INFO] agent.router.manager: shutting down -13:37.462 [INFO] TestAgent: consul server down -13:37.462 [INFO] TestAgent: shutdown complete -13:37.462 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16263 network=tcp -13:37.462 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16263 network=udp -13:37.462 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16264 network=tcp -13:37.962 [INFO] TestAgent: Waiting for endpoints to shut down -13:37.962 [INFO] TestAgent: Endpoints down -=== RUN TestSnapshot_Options/GET#02 -2021-10-31T23:13:36.973Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:37.043Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:37.062 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f6b5d91c-b416-b267-1df3-30e495d394c1 Address:127.0.0.1:16296}]" -13:37.063 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f6b5d91c-b416-b267-1df3-30e495d394c1.dc1 127.0.0.1 -13:37.063 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f6b5d91c-b416-b267-1df3-30e495d394c1 127.0.0.1 -2021-10-31T23:13:37.063Z [INFO] agent.router: Initializing LAN area manager -13:37.064 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16235 network=udp -13:37.064 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16296 [Follower]" leader= -13:37.064 [INFO] TestAgent.server: Adding LAN server: server="Node-f6b5d91c-b416-b267-1df3-30e495d394c1 (Addr: tcp/127.0.0.1:16296) (DC: dc1)" -13:37.064 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f6b5d91c-b416-b267-1df3-30e495d394c1.dc1 area=wan -13:37.064 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16235 network=tcp -13:37.065 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16236 network=tcp -13:37.065 [INFO] TestAgent: started state syncer -13:37.065 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16262 network=tcp -13:37.121 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:37.121 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16296 [Candidate]" term=2 -13:37.122 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:37.122 [DEBUG] TestAgent.server.raft: vote granted: from=f6b5d91c-b416-b267-1df3-30e495d394c1 term=2 tally=1 -13:37.122 [INFO] TestAgent.server.raft: election won: tally=1 -13:37.122 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16296 [Leader]" -13:37.122 [INFO] TestAgent.server: cluster leadership acquired -13:37.123 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16296 -13:37.123 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:37.125 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:37.125 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:37.125 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:37.125 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:37.125 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:37.125 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f6b5d91c-b416-b267-1df3-30e495d394c1 -13:37.125 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f6b5d91c-b416-b267-1df3-30e495d394c1 -13:37.126 [INFO] TestAgent.server: New leader elected: payload=Node-f6b5d91c-b416-b267-1df3-30e495d394c1 -13:37.130 [INFO] TestAgent.server: federation state anti-entropy synced -13:37.130 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:37.130 [INFO] TestAgent: Synced node info -13:37.130 [DEBUG] TestAgent: Node info in sync -13:37.504 [INFO] TestAgent: Requesting shutdown -13:37.504 [INFO] TestAgent.server: shutting down server -13:37.504 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:37.504 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:37.504 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:37.504 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:37.504 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:37.504 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:37.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:37.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:37.504 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:37.504 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:37.504 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:37.504Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:37.504Z [INFO] agent.router.manager: shutting down -13:37.504 [INFO] TestAgent: consul server down -13:37.504 [INFO] TestAgent: shutdown complete -13:37.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16235 network=tcp -13:37.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16235 network=udp -13:37.504 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16236 network=tcp -13:38.006 [INFO] TestAgent: Waiting for endpoints to shut down -13:38.006 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionCreate_NoCheck (1.07s) - --- PASS: TestSessionCreate_NoCheck/no_check_fields_should_yield_default_serfHealth (0.00s) - --- PASS: TestSessionCreate_NoCheck/overwrite_nodechecks_to_associate_with_no_checks (0.00s) - --- PASS: TestSessionCreate_NoCheck/overwrite_checks_to_associate_with_no_checks (0.00s) -=== CONT TestUiNodes_Filter -2021-10-31T23:13:37.068Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:37.114Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:37.116 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b4c44b67-849e-a9a9-a619-4c14b1efc379 Address:127.0.0.1:16568}]" -13:37.117 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b4c44b67-849e-a9a9-a619-4c14b1efc379.dc1 127.0.0.1 -13:37.117 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b4c44b67-849e-a9a9-a619-4c14b1efc379 127.0.0.1 -2021-10-31T23:13:37.117Z [INFO] agent.router: Initializing LAN area manager -13:37.117 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16568 [Follower]" leader= -13:37.118 [INFO] TestAgent.server: Adding LAN server: server="Node-b4c44b67-849e-a9a9-a619-4c14b1efc379 (Addr: tcp/127.0.0.1:16568) (DC: dc1)" -13:37.118 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b4c44b67-849e-a9a9-a619-4c14b1efc379.dc1 area=wan -13:37.118 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16270 network=tcp -13:37.126 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16270 network=udp -13:37.126 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16271 network=tcp -13:37.126 [INFO] TestAgent: started state syncer -13:37.126 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16569 network=tcp -13:37.161 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:37.161 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16568 [Candidate]" term=2 -13:37.162 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:37.162 [DEBUG] TestAgent.server.raft: vote granted: from=b4c44b67-849e-a9a9-a619-4c14b1efc379 term=2 tally=1 -13:37.162 [INFO] TestAgent.server.raft: election won: tally=1 -13:37.162 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16568 [Leader]" -13:37.162 [INFO] TestAgent.server: cluster leadership acquired -13:37.162 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16568 -13:37.162 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:37.164 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:37.164 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:37.164 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:37.164 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:37.164 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:37.164 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b4c44b67-849e-a9a9-a619-4c14b1efc379 -13:37.164 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b4c44b67-849e-a9a9-a619-4c14b1efc379 -13:37.164 [INFO] TestAgent.server: New leader elected: payload=Node-b4c44b67-849e-a9a9-a619-4c14b1efc379 -13:37.164 [INFO] TestAgent.server: federation state anti-entropy synced -13:37.382 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:37.388 [INFO] TestAgent: Synced node info -13:37.388 [DEBUG] TestAgent: Node info in sync -13:37.518 [INFO] TestAgent: Requesting shutdown -13:37.518 [INFO] TestAgent.server: shutting down server -13:37.518 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:37.518 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:37.518 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:37.518 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:37.518 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:37.518 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:37.518 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:37.518 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:37.518 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:37.518 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:37.518 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:37.519Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:37.519Z [INFO] agent.router.manager: shutting down -13:37.519 [INFO] TestAgent: consul server down -13:37.519 [INFO] TestAgent: shutdown complete -13:37.519 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16270 network=tcp -13:37.519 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16270 network=udp -13:37.519 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16271 network=tcp -13:38.019 [INFO] TestAgent: Waiting for endpoints to shut down -13:38.019 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_AutopilotGetConfiguration (1.07s) -=== CONT TestUiNodes -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=true#01 -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=false#01 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:37.692Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:37.764Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:37.768 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7a4cd906-805f-bde2-fee8-0f692935e79c Address:127.0.0.1:16198}]" -13:37.769 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16198 [Follower]" leader= -13:37.769 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7a4cd906-805f-bde2-fee8-0f692935e79c.dc1 127.0.0.1 -13:37.778 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7a4cd906-805f-bde2-fee8-0f692935e79c 127.0.0.1 -2021-10-31T23:13:37.778Z [INFO] agent.router: Initializing LAN area manager -13:37.778 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16326 network=udp -13:37.778 [INFO] TestAgent.server: Adding LAN server: server="Node-7a4cd906-805f-bde2-fee8-0f692935e79c (Addr: tcp/127.0.0.1:16198) (DC: dc1)" -13:37.778 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7a4cd906-805f-bde2-fee8-0f692935e79c.dc1 area=wan -13:37.779 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16326 network=tcp -13:37.780 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16327 network=tcp -13:37.780 [INFO] TestAgent: started state syncer -13:37.780 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16178 network=tcp -13:37.817 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:37.817 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16198 [Candidate]" term=2 -13:37.818 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:37.818 [DEBUG] TestAgent.server.raft: vote granted: from=7a4cd906-805f-bde2-fee8-0f692935e79c term=2 tally=1 -13:37.818 [INFO] TestAgent.server.raft: election won: tally=1 -13:37.818 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16198 [Leader]" -13:37.818 [INFO] TestAgent.server: cluster leadership acquired -13:37.818 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16198 -13:37.819 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:37.819 [INFO] TestAgent.server: New leader elected: payload=Node-7a4cd906-805f-bde2-fee8-0f692935e79c -13:37.821 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:37.821 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:37.827 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:37.827 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:37.827 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:37.827 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7a4cd906-805f-bde2-fee8-0f692935e79c -13:37.828 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7a4cd906-805f-bde2-fee8-0f692935e79c -13:37.862 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:37.862 [INFO] TestAgent: Requesting shutdown -13:37.862 [INFO] TestAgent.server: shutting down server -13:37.862 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:37.862 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:37.862 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:37.862 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:37.862 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:37.862 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" -13:37.862 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:37.862 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:37.862 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:37.862 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:37.862 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:37.862 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:37.862Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:37.862Z [INFO] agent.router.manager: shutting down -13:37.863 [INFO] TestAgent: consul server down -13:37.863 [INFO] TestAgent: shutdown complete -13:37.863 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16326 network=tcp -13:37.863 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16326 network=udp -13:37.863 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16327 network=tcp -13:38.365 [INFO] TestAgent: Waiting for endpoints to shut down -13:38.365 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Execute/#05 -2021-10-31T23:13:37.441Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:37.441Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:37.483Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:37.483Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:37.485 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d369a267-5366-ccf9-1b09-a4d486817404 Address:127.0.0.1:16170}]" -13:37.485 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d369a267-5366-ccf9-1b09-a4d486817404.dc1 127.0.0.1 -13:37.485 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16170 [Follower]" leader= -13:37.498 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d369a267-5366-ccf9-1b09-a4d486817404 127.0.0.1 -2021-10-31T23:13:37.498Z [INFO] agent.router: Initializing LAN area manager -13:37.502 [INFO] TestAgent.server: Adding LAN server: server="Node-d369a267-5366-ccf9-1b09-a4d486817404 (Addr: tcp/127.0.0.1:16170) (DC: dc1)" -13:37.502 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d369a267-5366-ccf9-1b09-a4d486817404.dc1 area=wan -13:37.505 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16500 network=udp -13:37.505 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16500 network=tcp -13:37.506 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16501 network=tcp -13:37.506 [INFO] TestAgent: started state syncer -13:37.506 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16318 network=tcp -13:37.550 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:37.550 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16170 [Candidate]" term=2 -13:37.550 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:37.550 [DEBUG] TestAgent.server.raft: vote granted: from=d369a267-5366-ccf9-1b09-a4d486817404 term=2 tally=1 -13:37.550 [INFO] TestAgent.server.raft: election won: tally=1 -13:37.550 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16170 [Leader]" -13:37.553 [INFO] TestAgent.server: cluster leadership acquired -13:37.554 [INFO] TestAgent.server: initializing acls -13:37.554 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:37.554 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:37.555 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:37.555 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:37.555 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:37.555 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:37.555 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -13:37.556 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d369a267-5366-ccf9-1b09-a4d486817404 -13:37.556 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d369a267-5366-ccf9-1b09-a4d486817404.dc1 -13:37.556 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d369a267-5366-ccf9-1b09-a4d486817404.dc1 area=wan -13:37.556 [INFO] TestAgent.server: Updating LAN server: server="Node-d369a267-5366-ccf9-1b09-a4d486817404 (Addr: tcp/127.0.0.1:16170) (DC: dc1)" -13:37.562 [INFO] TestAgent.server: initializing acls -13:37.562 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:37.563 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:37.565 [INFO] TestAgent.server: New leader elected: payload=Node-d369a267-5366-ccf9-1b09-a4d486817404 -13:37.570 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:37.570 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:37.570 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:37.570 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:37.570 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:37.570 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d369a267-5366-ccf9-1b09-a4d486817404 -13:37.570 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d369a267-5366-ccf9-1b09-a4d486817404 -13:37.571 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d369a267-5366-ccf9-1b09-a4d486817404 -13:37.571 [INFO] TestAgent.server: federation state anti-entropy synced -13:37.819 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:37.819 [INFO] TestAgent: Synced node info -13:37.883 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d369a267-5366-ccf9-1b09-a4d486817404 -13:37.883 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-d369a267-5366-ccf9-1b09-a4d486817404 -13:37.884 [INFO] TestAgent: Requesting shutdown -13:37.884 [INFO] TestAgent.server: shutting down server -13:37.884 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:37.884 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:37.884 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:37.884 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:37.884 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:37.884 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:37.884 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:37.884 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:37.884 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:37.884 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:37.884 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:37.884 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:37.884 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:37.884 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:37.884 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:37.885Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:37.885Z [INFO] agent.router.manager: shutting down -13:37.885 [INFO] TestAgent: consul server down -13:37.885 [INFO] TestAgent: shutdown complete -13:37.885 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16500 network=tcp -13:37.885 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16500 network=udp -13:37.885 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16501 network=tcp -13:38.385 [INFO] TestAgent: Waiting for endpoints to shut down -13:38.385 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Legacy_Get (2.09s) - --- PASS: TestACL_Legacy_Get/wrong_id (1.05s) - --- PASS: TestACL_Legacy_Get/right_id (1.02s) -=== CONT TestACL_Legacy_UpdateUpsert -2021-10-31T23:13:24.352Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:24.377Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:24.379 [INFO] Agent2.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:73fea63e-949c-903f-d4e4-91edf023d288 Address:127.0.0.1:17083}]" -13:24.380 [INFO] Agent2.server.serf.wan: serf: EventMemberJoin: Node-73fea63e-949c-903f-d4e4-91edf023d288.dc1 127.0.0.1 -13:24.381 [INFO] Agent2.server.serf.lan: serf: EventMemberJoin: Node-73fea63e-949c-903f-d4e4-91edf023d288 127.0.0.1 -2021-10-31T23:13:24.381Z [INFO] agent.router: Initializing LAN area manager -13:24.381 [INFO] Agent2: Started DNS server: address=127.0.0.1:17078 network=udp -13:24.381 [INFO] Agent2.server.raft: entering follower state: follower="Node at 127.0.0.1:17083 [Follower]" leader= -13:24.381 [INFO] Agent2.server: Adding LAN server: server="Node-73fea63e-949c-903f-d4e4-91edf023d288 (Addr: tcp/127.0.0.1:17083) (DC: dc1)" -13:24.381 [INFO] Agent2.server: Handled event for server in area: event=member-join server=Node-73fea63e-949c-903f-d4e4-91edf023d288.dc1 area=wan -13:24.381 [INFO] Agent2: Started DNS server: address=127.0.0.1:17078 network=tcp -13:24.382 [INFO] Agent2: Started HTTP server: address=127.0.0.1:17079 network=tcp -13:24.382 [INFO] Agent2: started state syncer -13:24.382 [INFO] Agent2: Started gRPC server: address=127.0.0.1:16204 network=tcp -13:24.452 [WARN] Agent2.server.raft: heartbeat timeout reached, starting election: last-leader= -13:24.452 [INFO] Agent2.server.raft: entering candidate state: node="Node at 127.0.0.1:17083 [Candidate]" term=2 -13:24.452 [DEBUG] Agent2.server.raft: votes: needed=1 -13:24.452 [DEBUG] Agent2.server.raft: vote granted: from=73fea63e-949c-903f-d4e4-91edf023d288 term=2 tally=1 -13:24.452 [INFO] Agent2.server.raft: election won: tally=1 -13:24.452 [INFO] Agent2.server.raft: entering leader state: leader="Node at 127.0.0.1:17083 [Leader]" -13:24.452 [INFO] Agent2.server: cluster leadership acquired -13:24.452 [DEBUG] Agent2.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17083 -13:24.453 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:24.453 [INFO] Agent2.server: New leader elected: payload=Node-73fea63e-949c-903f-d4e4-91edf023d288 -13:24.476 [INFO] Agent2.server.connect: initialized primary datacenter CA with provider: provider=consul -13:24.476 [INFO] Agent2.leader: started routine: routine="federation state anti-entropy" -13:24.476 [INFO] Agent2.leader: started routine: routine="federation state pruning" -13:24.476 [INFO] Agent2.leader: started routine: routine="intermediate cert renew watch" -13:24.476 [INFO] Agent2.leader: started routine: routine="CA root pruning" -13:24.476 [DEBUG] Agent2.server: Skipping self join check for node since the cluster is too small: node=Node-73fea63e-949c-903f-d4e4-91edf023d288 -13:24.476 [INFO] Agent2.server: member joined, marking health alive: member=Node-73fea63e-949c-903f-d4e4-91edf023d288 -13:24.477 [INFO] Agent2.server: federation state anti-entropy synced -13:24.494 [DEBUG] Agent2: Skipping remote check since it is managed automatically: check=serfHealth -13:24.494 [INFO] Agent2: Synced node info -13:24.494 [DEBUG] Agent2: Node info in sync -13:24.540 [DEBUG] Agent2: Skipping remote check since it is managed automatically: check=serfHealth -13:24.540 [DEBUG] Agent2: Node info in sync -13:24.582 [DEBUG] Agent2.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:50452 -13:24.583 [INFO] Agent2.server.serf.lan: serf: EventMemberJoin: Node-670088bd-1a44-36f9-3924-00a8d726840f 127.0.0.1 -13:24.583 [INFO] Agent2.server: Adding LAN server: server="Node-670088bd-1a44-36f9-3924-00a8d726840f (Addr: tcp/127.0.0.1:17146) (DC: dc1)" -13:24.583 [INFO] Agent2.server: New leader elected: payload=Node-670088bd-1a44-36f9-3924-00a8d726840f -13:24.583 [ERROR] Agent2.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-670088bd-1a44-36f9-3924-00a8d726840f other=Node-73fea63e-949c-903f-d4e4-91edf023d288 -13:24.583 [INFO] Agent2.server: member joined, marking health alive: member=Node-670088bd-1a44-36f9-3924-00a8d726840f -13:24.583 [DEBUG] Agent2.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-670088bd-1a44-36f9-3924-00a8d726840f.dc1 127.0.0.1:17145 -13:24.584 [INFO] Agent2: Requesting shutdown -13:24.585 [INFO] Agent2.server: shutting down server -13:24.585 [DEBUG] Agent2.leader: stopping routine: routine="federation state anti-entropy" -13:24.585 [DEBUG] Agent2.leader: stopping routine: routine="federation state pruning" -13:24.585 [DEBUG] Agent2.leader: stopping routine: routine="intermediate cert renew watch" -13:24.585 [DEBUG] Agent2.leader: stopping routine: routine="CA root pruning" -13:24.585 [WARN] Agent2.server.serf.lan: serf: Shutdown without a Leave -13:24.585 [INFO] Agent2.server.serf.wan: serf: EventMemberJoin: Node-670088bd-1a44-36f9-3924-00a8d726840f.dc1 127.0.0.1 -13:24.585 [DEBUG] Agent2.server: Successfully performed flood-join for server at address: server=Node-670088bd-1a44-36f9-3924-00a8d726840f.dc1 address=127.0.0.1:17145 -13:24.585 [INFO] Agent2.server: Handled event for server in area: event=member-join server=Node-670088bd-1a44-36f9-3924-00a8d726840f.dc1 area=wan -13:24.585 [ERROR] Agent2.server: error performing anti-entropy sync of federation state: error="context canceled" -13:24.585 [DEBUG] Agent2.leader: stopped routine: routine="federation state anti-entropy" -13:24.585 [DEBUG] Agent2.leader: stopped routine: routine="federation state pruning" -13:24.585 [DEBUG] Agent2.leader: stopped routine: routine="intermediate cert renew watch" -13:24.585 [DEBUG] Agent2.leader: stopped routine: routine="CA root pruning" -13:24.585 [WARN] Agent2.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:24.585Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:24.585Z [INFO] agent.router.manager: shutting down -13:24.585 [INFO] Agent2: consul server down -13:24.585 [INFO] Agent2: shutdown complete -13:24.585 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:17078 network=tcp -13:24.585 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:17078 network=udp -13:24.585 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:17079 network=tcp -13:25.087 [INFO] Agent2: Waiting for endpoints to shut down -13:25.087 [INFO] Agent2: Endpoints down -2021-10-31T23:13:23.832Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.875Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:23.895 [INFO] Agent1.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:670088bd-1a44-36f9-3924-00a8d726840f Address:127.0.0.1:17146}]" -13:23.896 [INFO] Agent1.server.serf.wan: serf: EventMemberJoin: Node-670088bd-1a44-36f9-3924-00a8d726840f.dc1 127.0.0.1 -13:23.896 [INFO] Agent1.server.serf.lan: serf: EventMemberJoin: Node-670088bd-1a44-36f9-3924-00a8d726840f 127.0.0.1 -2021-10-31T23:13:23.896Z [INFO] agent.router: Initializing LAN area manager -13:23.897 [INFO] Agent1: Started DNS server: address=127.0.0.1:17141 network=udp -13:23.897 [INFO] Agent1.server.raft: entering follower state: follower="Node at 127.0.0.1:17146 [Follower]" leader= -13:23.898 [INFO] Agent1.server: Adding LAN server: server="Node-670088bd-1a44-36f9-3924-00a8d726840f (Addr: tcp/127.0.0.1:17146) (DC: dc1)" -13:23.898 [INFO] Agent1.server: Handled event for server in area: event=member-join server=Node-670088bd-1a44-36f9-3924-00a8d726840f.dc1 area=wan -13:23.898 [INFO] Agent1: Started DNS server: address=127.0.0.1:17141 network=tcp -13:23.898 [INFO] Agent1: Started HTTP server: address=127.0.0.1:17142 network=tcp -13:23.899 [INFO] Agent1: started state syncer -13:23.899 [INFO] Agent1: Started gRPC server: address=127.0.0.1:16225 network=tcp -13:23.949 [WARN] Agent1.server.raft: heartbeat timeout reached, starting election: last-leader= -13:23.949 [INFO] Agent1.server.raft: entering candidate state: node="Node at 127.0.0.1:17146 [Candidate]" term=2 -13:23.950 [DEBUG] Agent1.server.raft: votes: needed=1 -13:23.950 [DEBUG] Agent1.server.raft: vote granted: from=670088bd-1a44-36f9-3924-00a8d726840f term=2 tally=1 -13:23.950 [INFO] Agent1.server.raft: election won: tally=1 -13:23.950 [INFO] Agent1.server.raft: entering leader state: leader="Node at 127.0.0.1:17146 [Leader]" -13:23.950 [INFO] Agent1.server: cluster leadership acquired -13:23.950 [DEBUG] Agent1.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17146 -13:23.950 [INFO] Agent1.server: New leader elected: payload=Node-670088bd-1a44-36f9-3924-00a8d726840f -13:23.951 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:23.952 [INFO] Agent1.server.connect: initialized primary datacenter CA with provider: provider=consul -13:23.952 [INFO] Agent1.leader: started routine: routine="federation state anti-entropy" -13:23.952 [INFO] Agent1.leader: started routine: routine="federation state pruning" -13:23.952 [INFO] Agent1.leader: started routine: routine="intermediate cert renew watch" -13:23.952 [INFO] Agent1.leader: started routine: routine="CA root pruning" -13:23.952 [DEBUG] Agent1.server: Skipping self join check for node since the cluster is too small: node=Node-670088bd-1a44-36f9-3924-00a8d726840f -13:23.952 [INFO] Agent1.server: member joined, marking health alive: member=Node-670088bd-1a44-36f9-3924-00a8d726840f -13:23.953 [INFO] Agent1.server: federation state anti-entropy synced -13:24.101 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth -13:24.101 [INFO] Agent1: Synced node info -13:24.101 [DEBUG] Agent1: Node info in sync -13:24.569 [INFO] Agent1: (LAN) joining: lan_addresses=[127.0.0.1:17081] -13:24.582 [DEBUG] Agent1.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:17081 -13:24.584 [DEBUG] Agent1.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:57682 -13:24.584 [INFO] Agent1.server.serf.lan: serf: EventMemberJoin: Node-73fea63e-949c-903f-d4e4-91edf023d288 127.0.0.1 -13:24.584 [INFO] Agent1.server.serf.wan: serf: EventMemberJoin: Node-73fea63e-949c-903f-d4e4-91edf023d288.dc1 127.0.0.1 -13:24.584 [INFO] Agent1: (LAN) joined: number_of_nodes=1 -13:24.584 [DEBUG] Agent1: systemd notify failed: error="No socket" -13:24.584 [INFO] Agent1.server: Handled event for server in area: event=member-join server=Node-73fea63e-949c-903f-d4e4-91edf023d288.dc1 area=wan -13:24.585 [INFO] Agent1.server: Adding LAN server: server="Node-73fea63e-949c-903f-d4e4-91edf023d288 (Addr: tcp/127.0.0.1:17083) (DC: dc1)" -13:24.585 [ERROR] Agent1.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-73fea63e-949c-903f-d4e4-91edf023d288 other=Node-670088bd-1a44-36f9-3924-00a8d726840f -13:24.585 [INFO] Agent1.server: member joined, marking health alive: member=Node-73fea63e-949c-903f-d4e4-91edf023d288 -13:25.398 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-73fea63e-949c-903f-d4e4-91edf023d288 (timeout reached) -13:25.899 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-73fea63e-949c-903f-d4e4-91edf023d288 has failed, no acks received -13:25.983 [WARN] Agent1: error getting server health from server: server=Node-73fea63e-949c-903f-d4e4-91edf023d288 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:17083: connect: connection refused" -13:26.538 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth -13:26.538 [DEBUG] Agent1: Node info in sync -13:26.986 [WARN] Agent1: error getting server health from server: server=Node-73fea63e-949c-903f-d4e4-91edf023d288 error="context deadline exceeded" -13:27.421 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-73fea63e-949c-903f-d4e4-91edf023d288 (timeout reached) -13:27.970 [WARN] Agent1: error getting server health from server: server=Node-73fea63e-949c-903f-d4e4-91edf023d288 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:17083: connect: connection refused" -13:28.902 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-73fea63e-949c-903f-d4e4-91edf023d288 has failed, no acks received -13:28.974 [WARN] Agent1: error getting server health from server: server=Node-73fea63e-949c-903f-d4e4-91edf023d288 error="context deadline exceeded" -13:29.910 [INFO] Agent1.server.memberlist.lan: memberlist: Marking Node-73fea63e-949c-903f-d4e4-91edf023d288 as failed, suspect timeout reached (0 peer confirmations) -13:29.910 [INFO] Agent1.server.serf.lan: serf: EventMemberFailed: Node-73fea63e-949c-903f-d4e4-91edf023d288 127.0.0.1 -13:29.910 [INFO] Agent1.server: Removing LAN server: server="Node-73fea63e-949c-903f-d4e4-91edf023d288 (Addr: tcp/127.0.0.1:17083) (DC: dc1)" -13:29.910 [INFO] Agent1.server: member failed, marking health critical: member=Node-73fea63e-949c-903f-d4e4-91edf023d288 -13:29.935 [INFO] Agent1: Force leaving node: node=Node-73fea63e-949c-903f-d4e4-91edf023d288 -13:29.935 [INFO] Agent1.server.serf.lan: serf: EventMemberLeave (forced): Node-73fea63e-949c-903f-d4e4-91edf023d288 127.0.0.1 -13:29.935 [INFO] Agent1.server.serf.lan: serf: EventMemberReap (forced): Node-73fea63e-949c-903f-d4e4-91edf023d288 127.0.0.1 -13:29.935 [INFO] Agent1.server: Removing LAN server: server="Node-73fea63e-949c-903f-d4e4-91edf023d288 (Addr: tcp/127.0.0.1:17083) (DC: dc1)" -13:29.935 [INFO] Agent1.server: Removing LAN server: server="Node-73fea63e-949c-903f-d4e4-91edf023d288 (Addr: tcp/127.0.0.1:17083) (DC: dc1)" -13:29.936 [INFO] Agent1.server: deregistering member: member=Node-73fea63e-949c-903f-d4e4-91edf023d288 reason=left -13:30.409 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-73fea63e-949c-903f-d4e4-91edf023d288 (timeout reached) -13:31.902 [DEBUG] Agent1.server.memberlist.wan: memberlist: Failed ping: Node-73fea63e-949c-903f-d4e4-91edf023d288.dc1 (timeout reached) -13:32.909 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-73fea63e-949c-903f-d4e4-91edf023d288 has failed, no acks received -13:33.903 [INFO] Agent1.server.memberlist.wan: memberlist: Suspect Node-73fea63e-949c-903f-d4e4-91edf023d288.dc1 has failed, no acks received -13:37.936 [INFO] Agent1.server.serf.wan: serf: EventMemberReap (forced): Node-73fea63e-949c-903f-d4e4-91edf023d288.dc1 127.0.0.1 -13:37.936 [INFO] Agent1: Requesting shutdown -13:37.936 [INFO] Agent1.server: shutting down server -13:37.937 [DEBUG] Agent1.leader: stopping routine: routine="federation state anti-entropy" -13:37.937 [DEBUG] Agent1.leader: stopping routine: routine="federation state pruning" -13:37.937 [DEBUG] Agent1.leader: stopping routine: routine="intermediate cert renew watch" -13:37.937 [DEBUG] Agent1.leader: stopping routine: routine="CA root pruning" -13:37.937 [WARN] Agent1.server.serf.lan: serf: Shutdown without a Leave -13:37.937 [INFO] Agent1.server: Handled event for server in area: event=member-reap server=Node-73fea63e-949c-903f-d4e4-91edf023d288.dc1 area=wan -13:37.937 [ERROR] Agent1.server: error performing anti-entropy sync of federation state: error="context canceled" -13:37.937 [DEBUG] Agent1.leader: stopped routine: routine="federation state anti-entropy" -13:37.937 [DEBUG] Agent1.leader: stopped routine: routine="federation state pruning" -13:37.937 [DEBUG] Agent1.leader: stopped routine: routine="intermediate cert renew watch" -13:37.937 [DEBUG] Agent1.leader: stopped routine: routine="CA root pruning" -13:37.937 [WARN] Agent1.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:37.937Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:37.937Z [INFO] agent.router.manager: shutting down -13:37.937 [INFO] Agent1: consul server down -13:37.937 [INFO] Agent1: shutdown complete -13:37.937 [INFO] Agent1: Stopping server: protocol=DNS address=127.0.0.1:17141 network=tcp -13:37.937 [INFO] Agent1: Stopping server: protocol=DNS address=127.0.0.1:17141 network=udp -13:37.937 [INFO] Agent1: Stopping server: protocol=HTTP address=127.0.0.1:17142 network=tcp -13:38.437 [INFO] Agent1: Waiting for endpoints to shut down -13:38.437 [INFO] Agent1: Endpoints down ---- PASS: TestAgent_ForceLeavePrune (14.65s) -=== CONT TestOperator_Keyring_InvalidRelayFactor -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=255)_compressed=true#01 -2021-10-31T23:13:37.828Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:37.913Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:37.917 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6d7d8a3d-19ce-b221-3ebd-634ff0ecf1bd Address:127.0.0.1:16994}]" -13:37.917 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16994 [Follower]" leader= -13:37.930 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6d7d8a3d-19ce-b221-3ebd-634ff0ecf1bd.dc1 127.0.0.1 -13:37.938 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6d7d8a3d-19ce-b221-3ebd-634ff0ecf1bd 127.0.0.1 -2021-10-31T23:13:37.938Z [INFO] agent.router: Initializing LAN area manager -13:37.939 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6d7d8a3d-19ce-b221-3ebd-634ff0ecf1bd.dc1 area=wan -13:37.939 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16340 network=udp -13:37.939 [INFO] TestAgent.server: Adding LAN server: server="Node-6d7d8a3d-19ce-b221-3ebd-634ff0ecf1bd (Addr: tcp/127.0.0.1:16994) (DC: dc1)" -13:37.939 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16340 network=tcp -13:37.940 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16341 network=tcp -13:37.940 [INFO] TestAgent: started state syncer -13:37.940 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16995 network=tcp -13:37.989 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:37.989 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16994 [Candidate]" term=2 -13:37.990 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:37.990 [DEBUG] TestAgent.server.raft: vote granted: from=6d7d8a3d-19ce-b221-3ebd-634ff0ecf1bd term=2 tally=1 -13:37.990 [INFO] TestAgent.server.raft: election won: tally=1 -13:37.990 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16994 [Leader]" -13:37.990 [INFO] TestAgent.server: cluster leadership acquired -13:37.990 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16994 -13:37.991 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:37.993 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:37.993 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:37.993 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:37.993 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:37.993 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:37.993 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6d7d8a3d-19ce-b221-3ebd-634ff0ecf1bd -13:37.993 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6d7d8a3d-19ce-b221-3ebd-634ff0ecf1bd -13:37.998 [INFO] TestAgent.server: New leader elected: payload=Node-6d7d8a3d-19ce-b221-3ebd-634ff0ecf1bd -13:37.998 [INFO] TestAgent.server: federation state anti-entropy synced -13:38.034 [INFO] TestAgent: Requesting shutdown -13:38.034 [INFO] TestAgent.server: shutting down server -13:38.034 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:38.034 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:38.034 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:38.034 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:38.034 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:38.034 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:38.034 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:38.034 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:38.034 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:38.034 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:38.034 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:38.034 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:38.035Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:38.035Z [INFO] agent.router.manager: shutting down -13:38.035 [INFO] TestAgent: consul server down -13:38.035 [INFO] TestAgent: shutdown complete -13:38.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16340 network=tcp -13:38.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16340 network=udp -13:38.035 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16341 network=tcp -13:38.538 [INFO] TestAgent: Waiting for endpoints to shut down -13:38.538 [INFO] TestAgent: Endpoints down ---- PASS: TestUiIndex (0.78s) -=== CONT TestOperator_KeyringUse -[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:37.825Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:37.884Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:37.895 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a642968a-52a8-fc2f-cbcc-5365d4fa7012 Address:127.0.0.1:16317}]" -13:37.895 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a642968a-52a8-fc2f-cbcc-5365d4fa7012.dc1 127.0.0.1 -13:37.896 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a642968a-52a8-fc2f-cbcc-5365d4fa7012 127.0.0.1 -2021-10-31T23:13:37.896Z [INFO] agent.router: Initializing LAN area manager -13:37.896 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16317 [Follower]" leader= -13:37.896 [INFO] TestAgent.server: Adding LAN server: server="Node-a642968a-52a8-fc2f-cbcc-5365d4fa7012 (Addr: tcp/127.0.0.1:16317) (DC: dc1)" -13:37.896 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a642968a-52a8-fc2f-cbcc-5365d4fa7012.dc1 area=wan -13:37.897 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16312 network=tcp -13:37.897 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16312 network=udp -13:37.905 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16313 network=tcp -13:37.906 [INFO] TestAgent: started state syncer -13:37.906 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16339 network=tcp -13:37.957 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:37.957 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16317 [Candidate]" term=2 -13:37.957 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:37.957 [DEBUG] TestAgent.server.raft: vote granted: from=a642968a-52a8-fc2f-cbcc-5365d4fa7012 term=2 tally=1 -13:37.957 [INFO] TestAgent.server.raft: election won: tally=1 -13:37.957 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16317 [Leader]" -13:37.957 [INFO] TestAgent.server: cluster leadership acquired -13:37.958 [INFO] TestAgent.server: New leader elected: payload=Node-a642968a-52a8-fc2f-cbcc-5365d4fa7012 -13:37.958 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16317 -13:37.958 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:37.960 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:37.960 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:37.960 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:37.960 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:37.960 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:37.960 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a642968a-52a8-fc2f-cbcc-5365d4fa7012 -13:37.960 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a642968a-52a8-fc2f-cbcc-5365d4fa7012 -13:37.961 [INFO] TestAgent.server: federation state anti-entropy synced -13:38.106 [INFO] TestAgent: Requesting shutdown -13:38.107 [INFO] TestAgent.server: shutting down server -13:38.107 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:38.107 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:38.107 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:38.107 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:38.107 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:38.107 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:38.107 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:38.107 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:38.107 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:38.107 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:38.107 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:38.107 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:38.107Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:38.107Z [INFO] agent.router.manager: shutting down -13:38.107 [INFO] TestAgent: consul server down -13:38.107 [INFO] TestAgent: shutdown complete -13:38.107 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16312 network=tcp -13:38.107 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16312 network=udp -13:38.107 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16313 network=tcp -13:38.610 [INFO] TestAgent: Waiting for endpoints to shut down -13:38.610 [INFO] TestAgent: Endpoints down ---- PASS: TestKVSEndpoint_DELETE_CAS (0.85s) -=== CONT TestRemoteExecGetSpec_ACLDeny -2021-10-31T23:13:37.665Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:37.706Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:37.708 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4e61a81f-4907-2cf3-25c0-da0c04ef841f Address:127.0.0.1:16289}]" -13:37.708 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4e61a81f-4907-2cf3-25c0-da0c04ef841f.dc1 127.0.0.1 -13:37.709 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4e61a81f-4907-2cf3-25c0-da0c04ef841f 127.0.0.1 -2021-10-31T23:13:37.709Z [INFO] agent.router: Initializing LAN area manager -13:37.709 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16305 network=udp -13:37.709 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16289 [Follower]" leader= -13:37.710 [INFO] TestAgent.server: Adding LAN server: server="Node-4e61a81f-4907-2cf3-25c0-da0c04ef841f (Addr: tcp/127.0.0.1:16289) (DC: dc1)" -13:37.710 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16305 network=tcp -13:37.714 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16306 network=tcp -13:37.714 [INFO] TestAgent: started state syncer -13:37.710 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4e61a81f-4907-2cf3-25c0-da0c04ef841f.dc1 area=wan -13:37.715 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16325 network=tcp -13:37.790 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:37.790 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16289 [Candidate]" term=2 -13:37.790 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:37.790 [DEBUG] TestAgent.server.raft: vote granted: from=4e61a81f-4907-2cf3-25c0-da0c04ef841f term=2 tally=1 -13:37.790 [INFO] TestAgent.server.raft: election won: tally=1 -13:37.790 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16289 [Leader]" -13:37.790 [INFO] TestAgent.server: cluster leadership acquired -13:37.790 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16289 -13:37.792 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:37.801 [INFO] TestAgent.server: New leader elected: payload=Node-4e61a81f-4907-2cf3-25c0-da0c04ef841f -13:37.802 [INFO] TestAgent: Synced node info -13:37.802 [DEBUG] TestAgent: Node info in sync -13:37.807 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:37.807 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:37.807 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:37.807 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:37.807 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:37.807 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4e61a81f-4907-2cf3-25c0-da0c04ef841f -13:37.807 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4e61a81f-4907-2cf3-25c0-da0c04ef841f -13:37.808 [INFO] TestAgent.server: federation state anti-entropy synced -13:38.137 [INFO] TestAgent: Requesting shutdown -13:38.137 [INFO] TestAgent.server: shutting down server -13:38.137 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:38.137 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:38.137 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:38.137 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:38.137 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:38.137 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:38.137 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:38.137 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:38.137 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:38.137 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:38.137 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:38.137Z [INFO] agent.router.manager: shutting down -13:38.137 [INFO] TestAgent: consul server down -13:38.138 [INFO] TestAgent: shutdown complete -13:38.138 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16305 network=tcp -13:38.138 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16305 network=udp -13:38.138 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16306 network=tcp -2021-10-31T23:13:38.137Z [INFO] agent.router.manager: shutting down -13:38.640 [INFO] TestAgent: Waiting for endpoints to shut down -13:38.640 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionDestroy (1.01s) -=== CONT TestHandleRemoteExecFailed -2021-10-31T23:13:37.817Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:37.901Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:37.907 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:42bf3770-8174-6555-2f53-027e05d376cd Address:127.0.0.1:16338}]" -13:37.907 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-42bf3770-8174-6555-2f53-027e05d376cd.dc1 127.0.0.1 -13:37.908 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-42bf3770-8174-6555-2f53-027e05d376cd 127.0.0.1 -2021-10-31T23:13:37.908Z [INFO] agent.router: Initializing LAN area manager -13:37.908 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16347 network=udp -13:37.908 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16338 [Follower]" leader= -13:37.909 [INFO] TestAgent.server: Adding LAN server: server="Node-42bf3770-8174-6555-2f53-027e05d376cd (Addr: tcp/127.0.0.1:16338) (DC: dc1)" -13:37.909 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-42bf3770-8174-6555-2f53-027e05d376cd.dc1 area=wan -13:37.909 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16347 network=tcp -13:37.922 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16348 network=tcp -13:37.922 [INFO] TestAgent: started state syncer -13:37.922 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16297 network=tcp -13:37.952 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:37.952 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16338 [Candidate]" term=2 -13:37.952 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:37.952 [DEBUG] TestAgent.server.raft: vote granted: from=42bf3770-8174-6555-2f53-027e05d376cd term=2 tally=1 -13:37.952 [INFO] TestAgent.server.raft: election won: tally=1 -13:37.952 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16338 [Leader]" -13:37.953 [INFO] TestAgent.server: cluster leadership acquired -13:37.954 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16338 -13:37.954 [INFO] TestAgent.server: New leader elected: payload=Node-42bf3770-8174-6555-2f53-027e05d376cd -13:37.955 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:37.956 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:37.956 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:37.956 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:37.956 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:37.956 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:37.956 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-42bf3770-8174-6555-2f53-027e05d376cd -13:37.956 [INFO] TestAgent.server: member joined, marking health alive: member=Node-42bf3770-8174-6555-2f53-027e05d376cd -13:37.957 [INFO] TestAgent.server: federation state anti-entropy synced -13:38.055 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:38.056 [INFO] TestAgent: Synced node info -13:38.056 [DEBUG] TestAgent: Node info in sync -13:38.174 [INFO] TestAgent: Requesting shutdown -13:38.174 [INFO] TestAgent.server: shutting down server -13:38.174 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:38.174 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:38.174 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:38.174 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:38.174 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:38.174 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:38.174 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:38.174 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:38.174 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:38.174 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:38.175 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:38.175Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:38.175Z [INFO] agent.router.manager: shutting down -13:38.175 [INFO] TestAgent: consul server down -13:38.175 [INFO] TestAgent: shutdown complete -13:38.175 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16347 network=tcp -13:38.175 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16347 network=udp -13:38.175 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16348 network=tcp -13:38.677 [INFO] TestAgent: Waiting for endpoints to shut down -13:38.677 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionGet (1.61s) - --- PASS: TestSessionGet/#00 (0.71s) - --- PASS: TestSessionGet/#01 (0.90s) -=== CONT TestHandleRemoteExec -2021-10-31T23:13:38.105Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:38.159Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:38.174 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:337a884b-a367-8ea8-0c79-05c94b0ca620 Address:127.0.0.1:16401}]" -13:38.174 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16401 [Follower]" leader= -13:38.174 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-337a884b-a367-8ea8-0c79-05c94b0ca620.dc1 127.0.0.1 -13:38.175 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-337a884b-a367-8ea8-0c79-05c94b0ca620 127.0.0.1 -2021-10-31T23:13:38.175Z [INFO] agent.router: Initializing LAN area manager -13:38.175 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16417 network=udp -13:38.175 [INFO] TestAgent.server: Adding LAN server: server="Node-337a884b-a367-8ea8-0c79-05c94b0ca620 (Addr: tcp/127.0.0.1:16401) (DC: dc1)" -13:38.175 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-337a884b-a367-8ea8-0c79-05c94b0ca620.dc1 area=wan -13:38.175 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16417 network=tcp -13:38.176 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16418 network=tcp -13:38.176 [INFO] TestAgent: started state syncer -13:38.176 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16402 network=tcp -13:38.229 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:38.229 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16401 [Candidate]" term=2 -13:38.230 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:38.230 [DEBUG] TestAgent.server.raft: vote granted: from=337a884b-a367-8ea8-0c79-05c94b0ca620 term=2 tally=1 -13:38.230 [INFO] TestAgent.server.raft: election won: tally=1 -13:38.230 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16401 [Leader]" -13:38.230 [INFO] TestAgent.server: cluster leadership acquired -13:38.230 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16401 -13:38.231 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:38.232 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:38.232 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:38.232 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:38.232 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:38.232 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:38.232 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-337a884b-a367-8ea8-0c79-05c94b0ca620 -13:38.232 [INFO] TestAgent.server: member joined, marking health alive: member=Node-337a884b-a367-8ea8-0c79-05c94b0ca620 -13:38.233 [INFO] TestAgent.server: New leader elected: payload=Node-337a884b-a367-8ea8-0c79-05c94b0ca620 -13:38.233 [INFO] TestAgent.server: federation state anti-entropy synced -13:38.234 [INFO] TestAgent: Requesting shutdown -13:38.234 [INFO] TestAgent.server: shutting down server -13:38.234 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:38.234 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:38.234 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:38.234 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:38.234 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:38.234 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:38.234 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:38.234 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:38.234 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:38.234 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:38.234 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:38.234 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:38.234Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:38.234Z [INFO] agent.router.manager: shutting down -13:38.235 [INFO] TestAgent: consul server down -13:38.235 [INFO] TestAgent: shutdown complete -13:38.235 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16417 network=tcp -13:38.235 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16417 network=udp -13:38.235 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16418 network=tcp -13:38.737 [INFO] TestAgent: Waiting for endpoints to shut down -13:38.737 [INFO] TestAgent: Endpoints down ---- PASS: TestUiNodes (0.73s) -=== CONT TestRemoteExecWrites_ACLDeny -2021-10-31T23:13:37.828Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:37.924Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:37.942 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:82991695-0dc3-a68a-dc32-d069c122ac76 Address:127.0.0.1:16303}]" -13:37.943 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-82991695-0dc3-a68a-dc32-d069c122ac76.dc1 127.0.0.1 -13:37.943 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-82991695-0dc3-a68a-dc32-d069c122ac76 127.0.0.1 -2021-10-31T23:13:37.943Z [INFO] agent.router: Initializing LAN area manager -13:37.944 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16996 network=udp -13:37.944 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16303 [Follower]" leader= -13:37.944 [INFO] TestAgent.server: Adding LAN server: server="Node-82991695-0dc3-a68a-dc32-d069c122ac76 (Addr: tcp/127.0.0.1:16303) (DC: dc1)" -13:37.944 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-82991695-0dc3-a68a-dc32-d069c122ac76.dc1 area=wan -13:37.944 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16996 network=tcp -13:37.945 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16997 network=tcp -13:37.945 [INFO] TestAgent: started state syncer -13:37.945 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16346 network=tcp -13:38.005 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:38.013 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16303 [Candidate]" term=2 -13:38.014 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:38.014 [DEBUG] TestAgent.server.raft: vote granted: from=82991695-0dc3-a68a-dc32-d069c122ac76 term=2 tally=1 -13:38.014 [INFO] TestAgent.server.raft: election won: tally=1 -13:38.014 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16303 [Leader]" -13:38.014 [INFO] TestAgent.server: cluster leadership acquired -13:38.014 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16303 -13:38.014 [INFO] TestAgent.server: New leader elected: payload=Node-82991695-0dc3-a68a-dc32-d069c122ac76 -13:38.015 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:38.017 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:38.017 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:38.017 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:38.017 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:38.017 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:38.018 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-82991695-0dc3-a68a-dc32-d069c122ac76 -13:38.018 [INFO] TestAgent.server: member joined, marking health alive: member=Node-82991695-0dc3-a68a-dc32-d069c122ac76 -13:38.019 [INFO] TestAgent.server: federation state anti-entropy synced -13:38.030 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:38.035 [INFO] TestAgent: Synced node info -13:38.274 [INFO] TestAgent: Requesting shutdown -13:38.274 [INFO] TestAgent.server: shutting down server -13:38.274 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:38.274 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:38.274 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:38.274 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:38.274 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:38.274 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:38.274 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:38.274 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:38.274 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:38.274 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:38.274 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:38.275Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:38.275Z [INFO] agent.router.manager: shutting down -13:38.275 [INFO] TestAgent: consul server down -13:38.275 [INFO] TestAgent: shutdown complete -13:38.275 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16996 network=tcp -13:38.275 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16996 network=udp -13:38.275 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16997 network=tcp -13:38.777 [INFO] TestAgent: Waiting for endpoints to shut down -13:38.777 [INFO] TestAgent: Endpoints down ---- PASS: TestUiNodeInfo (1.01s) -=== CONT TestRemoteExecWrites_ACLAgentToken -2021-10-31T23:13:38.061Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:38.128Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:38.134 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:72dc8eb2-8f3f-be5a-571a-7776ccd7a891 Address:127.0.0.1:16394}]" -13:38.135 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-72dc8eb2-8f3f-be5a-571a-7776ccd7a891.dc1 127.0.0.1 -13:38.135 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16394 [Follower]" leader= -13:38.136 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-72dc8eb2-8f3f-be5a-571a-7776ccd7a891 127.0.0.1 -2021-10-31T23:13:38.136Z [INFO] agent.router: Initializing LAN area manager -13:38.136 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-72dc8eb2-8f3f-be5a-571a-7776ccd7a891.dc1 area=wan -13:38.136 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16396 network=udp -13:38.136 [INFO] TestAgent.server: Adding LAN server: server="Node-72dc8eb2-8f3f-be5a-571a-7776ccd7a891 (Addr: tcp/127.0.0.1:16394) (DC: dc1)" -13:38.136 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16396 network=tcp -13:38.137 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16397 network=tcp -13:38.137 [INFO] TestAgent: started state syncer -13:38.137 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16416 network=tcp -13:38.189 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:38.190 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16394 [Candidate]" term=2 -13:38.190 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:38.190 [DEBUG] TestAgent.server.raft: vote granted: from=72dc8eb2-8f3f-be5a-571a-7776ccd7a891 term=2 tally=1 -13:38.190 [INFO] TestAgent.server.raft: election won: tally=1 -13:38.190 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16394 [Leader]" -13:38.190 [INFO] TestAgent.server: cluster leadership acquired -13:38.190 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16394 -13:38.191 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:38.193 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:38.193 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:38.193 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:38.193 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:38.193 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:38.193 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-72dc8eb2-8f3f-be5a-571a-7776ccd7a891 -13:38.193 [INFO] TestAgent.server: member joined, marking health alive: member=Node-72dc8eb2-8f3f-be5a-571a-7776ccd7a891 -13:38.193 [INFO] TestAgent.server: New leader elected: payload=Node-72dc8eb2-8f3f-be5a-571a-7776ccd7a891 -13:38.194 [INFO] TestAgent.server: federation state anti-entropy synced -13:38.311 [INFO] TestAgent: Requesting shutdown -13:38.311 [INFO] TestAgent.server: shutting down server -13:38.311 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:38.311 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:38.311 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:38.311 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:38.311 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:38.311 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:38.311 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:38.311 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:38.311 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:38.311 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:38.311 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:38.311 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:38.312Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:38.312Z [INFO] agent.router.manager: shutting down -13:38.312 [INFO] TestAgent: consul server down -13:38.312 [INFO] TestAgent: shutdown complete -13:38.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16396 network=tcp -13:38.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16396 network=udp -13:38.312 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16397 network=tcp -13:38.813 [INFO] TestAgent: Waiting for endpoints to shut down -13:38.813 [INFO] TestAgent: Endpoints down ---- PASS: TestUiNodes_Filter (0.81s) -=== CONT TestRemoteExecWrites_ACLToken -[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:38.050Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:38.050Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:38.109Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:38.109Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:38.119 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9f20f42c-423b-84c7-3c96-f1e2c5503f0d Address:127.0.0.1:16359}]" -13:38.119 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9f20f42c-423b-84c7-3c96-f1e2c5503f0d.dc1 127.0.0.1 -13:38.120 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9f20f42c-423b-84c7-3c96-f1e2c5503f0d 127.0.0.1 -2021-10-31T23:13:38.120Z [INFO] agent.router: Initializing LAN area manager -13:38.120 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16298 network=udp -13:38.120 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16359 [Follower]" leader= -13:38.120 [INFO] TestAgent.server: Adding LAN server: server="Node-9f20f42c-423b-84c7-3c96-f1e2c5503f0d (Addr: tcp/127.0.0.1:16359) (DC: dc1)" -13:38.121 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9f20f42c-423b-84c7-3c96-f1e2c5503f0d.dc1 area=wan -13:38.121 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16298 network=tcp -13:38.121 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16299 network=tcp -13:38.130 [INFO] TestAgent: started state syncer -13:38.130 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16395 network=tcp -13:38.157 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:38.157 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16359 [Candidate]" term=2 -13:38.157 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:38.157 [DEBUG] TestAgent.server.raft: vote granted: from=9f20f42c-423b-84c7-3c96-f1e2c5503f0d term=2 tally=1 -13:38.157 [INFO] TestAgent.server.raft: election won: tally=1 -13:38.157 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16359 [Leader]" -13:38.157 [INFO] TestAgent.server: cluster leadership acquired -13:38.157 [INFO] TestAgent.server: New leader elected: payload=Node-9f20f42c-423b-84c7-3c96-f1e2c5503f0d -13:38.157 [INFO] TestAgent.server: initializing acls -13:38.158 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:38.158 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:38.158 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:38.159 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:38.159 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:38.159 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:38.159 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9f20f42c-423b-84c7-3c96-f1e2c5503f0d -13:38.159 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9f20f42c-423b-84c7-3c96-f1e2c5503f0d.dc1 -13:38.159 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:38.161 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:38.161 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:38.161 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:38.161 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:38.161 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:38.161 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9f20f42c-423b-84c7-3c96-f1e2c5503f0d -13:38.161 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9f20f42c-423b-84c7-3c96-f1e2c5503f0d -13:38.161 [INFO] TestAgent.server: Updating LAN server: server="Node-9f20f42c-423b-84c7-3c96-f1e2c5503f0d (Addr: tcp/127.0.0.1:16359) (DC: dc1)" -13:38.162 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9f20f42c-423b-84c7-3c96-f1e2c5503f0d.dc1 area=wan -13:38.174 [INFO] TestAgent.server: federation state anti-entropy synced -13:38.174 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9f20f42c-423b-84c7-3c96-f1e2c5503f0d -13:38.354 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:38.355 [INFO] TestAgent: Synced node info -13:38.545 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-9f20f42c-423b-84c7-3c96-f1e2c5503f0d -13:38.546 [INFO] TestAgent.server.fsm: snapshot created: duration=90.827µs -13:38.546 [INFO] TestAgent.server.raft: starting snapshot up to: index=15 -13:38.546 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot_Options_GET#02-TestAgent-agent030891140/raft/snapshots/2-15-1635722018546.tmp -13:38.547 [INFO] TestAgent.server.raft: snapshot complete up to: index=15 -13:38.549 [INFO] TestAgent: Requesting shutdown -13:38.549 [INFO] TestAgent.server: shutting down server -13:38.549 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:38.549 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:38.549 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:38.549 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:38.549 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:38.549 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:38.549 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:38.549 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:38.549 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:38.549 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:38.549 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:38.549 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:38.549 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:38.549 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:38.549 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:38.549Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:38.549Z [INFO] agent.router.manager: shutting down -13:38.550 [INFO] TestAgent: consul server down -13:38.550 [INFO] TestAgent: shutdown complete -13:38.550 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16298 network=tcp -13:38.550 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16298 network=udp -13:38.550 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16299 network=tcp -13:39.053 [INFO] TestAgent: Waiting for endpoints to shut down -13:39.053 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:37.487Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:37.524Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:37.532 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b702f422-f813-863c-f3b0-d85dbf73131a Address:127.0.0.1:16345}]" -13:37.532 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b702f422-f813-863c-f3b0-d85dbf73131a.dc1 127.0.0.1 -13:37.533 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b702f422-f813-863c-f3b0-d85dbf73131a 127.0.0.1 -2021-10-31T23:13:37.533Z [INFO] agent.router: Initializing LAN area manager -13:37.533 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16319 network=udp -13:37.533 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16345 [Follower]" leader= -13:37.534 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16319 network=tcp -13:37.534 [INFO] TestAgent.server: Adding LAN server: server="Node-b702f422-f813-863c-f3b0-d85dbf73131a (Addr: tcp/127.0.0.1:16345) (DC: dc1)" -13:37.534 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b702f422-f813-863c-f3b0-d85dbf73131a.dc1 area=wan -13:37.534 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16320 network=tcp -13:37.534 [INFO] TestAgent: started state syncer -13:37.534 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16304 network=tcp -13:37.584 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:37.584 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16345 [Candidate]" term=2 -13:37.584 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:37.584 [DEBUG] TestAgent.server.raft: vote granted: from=b702f422-f813-863c-f3b0-d85dbf73131a term=2 tally=1 -13:37.585 [INFO] TestAgent.server.raft: election won: tally=1 -13:37.585 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16345 [Leader]" -13:37.585 [INFO] TestAgent.server: cluster leadership acquired -13:37.585 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16345 -13:37.585 [INFO] TestAgent.server: New leader elected: payload=Node-b702f422-f813-863c-f3b0-d85dbf73131a -13:37.586 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:37.588 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:37.588 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:37.588 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:37.588 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:37.588 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:37.588 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b702f422-f813-863c-f3b0-d85dbf73131a -13:37.588 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b702f422-f813-863c-f3b0-d85dbf73131a -13:37.615 [INFO] TestAgent.server: federation state anti-entropy synced -13:37.814 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:37.814 [INFO] TestAgent: Synced node info -13:37.815 [DEBUG] TestAgent: Node info in sync -13:38.310 [DEBUG] TestAgent.server: Session TTL expired: session=2196fe52-5ccc-e8ba-d20b-7e1a02633bcc -13:38.564 [INFO] TestAgent: Requesting shutdown -13:38.564 [INFO] TestAgent.server: shutting down server -13:38.564 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:38.564 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:38.564 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:38.564 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:38.564 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:38.564 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:38.564 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:38.564 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:38.564 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:38.564 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:38.564 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:38.564Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:38.564Z [INFO] agent.router.manager: shutting down -13:38.564 [INFO] TestAgent: consul server down -13:38.564 [INFO] TestAgent: shutdown complete -13:38.564 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16319 network=tcp -13:38.565 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16319 network=udp -13:38.565 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16320 network=tcp -13:39.065 [INFO] TestAgent: Waiting for endpoints to shut down -13:39.065 [INFO] TestAgent: Endpoints down -=== RUN TestSnapshot_Options/PUT ---- PASS: TestSessionCustomTTL (1.62s) -=== CONT TestRemoteExecWrites -2021-10-31T23:13:38.403Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:38.462Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:38.464 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c6d27ddb-5444-7873-4cd7-2f68dd0ce690 Address:127.0.0.1:16380}]" -13:38.464 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16380 [Follower]" leader= -13:38.489 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c6d27ddb-5444-7873-4cd7-2f68dd0ce690.dc1 127.0.0.1 -13:38.490 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c6d27ddb-5444-7873-4cd7-2f68dd0ce690 127.0.0.1 -2021-10-31T23:13:38.490Z [INFO] agent.router: Initializing LAN area manager -13:38.490 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16403 network=udp -13:38.492 [INFO] TestAgent.server: Adding LAN server: server="Node-c6d27ddb-5444-7873-4cd7-2f68dd0ce690 (Addr: tcp/127.0.0.1:16380) (DC: dc1)" -13:38.492 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c6d27ddb-5444-7873-4cd7-2f68dd0ce690.dc1 area=wan -13:38.493 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16403 network=tcp -13:38.494 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16404 network=tcp -13:38.494 [INFO] TestAgent: started state syncer -13:38.495 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16388 network=tcp -13:38.525 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:38.526 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16380 [Candidate]" term=2 -13:38.526 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:38.526 [DEBUG] TestAgent.server.raft: vote granted: from=c6d27ddb-5444-7873-4cd7-2f68dd0ce690 term=2 tally=1 -13:38.526 [INFO] TestAgent.server.raft: election won: tally=1 -13:38.526 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16380 [Leader]" -13:38.526 [INFO] TestAgent.server: cluster leadership acquired -13:38.526 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16380 -13:38.526 [INFO] TestAgent.server: New leader elected: payload=Node-c6d27ddb-5444-7873-4cd7-2f68dd0ce690 -13:38.528 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:38.529 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:38.529 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:38.529 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:38.529 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:38.529 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:38.529 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c6d27ddb-5444-7873-4cd7-2f68dd0ce690 -13:38.537 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c6d27ddb-5444-7873-4cd7-2f68dd0ce690 -13:38.530 [INFO] TestAgent.server: federation state anti-entropy synced -13:38.550 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:38.550 [INFO] TestAgent: Synced node info -13:38.569 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:38.570 [INFO] TestAgent: Requesting shutdown -13:38.570 [INFO] TestAgent.server: shutting down server -13:38.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:38.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:38.570 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:38.570 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:38.570 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:38.570 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:38.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:38.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:38.570 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:38.570 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:38.570 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:38.570Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:38.570Z [INFO] agent.router.manager: shutting down -13:38.571 [INFO] TestAgent: consul server down -13:38.571 [INFO] TestAgent: shutdown complete -13:38.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16403 network=tcp -13:38.571 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16403 network=udp -13:38.571 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16404 network=tcp -13:39.071 [INFO] TestAgent: Waiting for endpoints to shut down -13:39.071 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Execute/#06 -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:38.424Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:38.424Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:38.458Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:38.458Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:38.460 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:735d68a3-e078-7f6a-9308-cd439333dc60 Address:127.0.0.1:16436}]" -13:38.461 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-735d68a3-e078-7f6a-9308-cd439333dc60.dc1 127.0.0.1 -13:38.461 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-735d68a3-e078-7f6a-9308-cd439333dc60 127.0.0.1 -2021-10-31T23:13:38.461Z [INFO] agent.router: Initializing LAN area manager -13:38.461 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16436 [Follower]" leader= -13:38.462 [INFO] TestAgent.server: Adding LAN server: server="Node-735d68a3-e078-7f6a-9308-cd439333dc60 (Addr: tcp/127.0.0.1:16436) (DC: dc1)" -13:38.462 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-735d68a3-e078-7f6a-9308-cd439333dc60.dc1 area=wan -13:38.464 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16389 network=udp -13:38.464 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16389 network=tcp -13:38.465 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16390 network=tcp -13:38.465 [INFO] TestAgent: started state syncer -13:38.465 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16451 network=tcp -13:38.509 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:38.510 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16436 [Candidate]" term=2 -13:38.510 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:38.510 [DEBUG] TestAgent.server.raft: vote granted: from=735d68a3-e078-7f6a-9308-cd439333dc60 term=2 tally=1 -13:38.510 [INFO] TestAgent.server.raft: election won: tally=1 -13:38.510 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16436 [Leader]" -13:38.510 [INFO] TestAgent.server: cluster leadership acquired -13:38.510 [INFO] TestAgent.server: initializing acls -13:38.511 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:38.511 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:38.511 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:38.512 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:38.512 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:38.512 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:38.512 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-735d68a3-e078-7f6a-9308-cd439333dc60 -13:38.512 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-735d68a3-e078-7f6a-9308-cd439333dc60.dc1 -13:38.513 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-735d68a3-e078-7f6a-9308-cd439333dc60.dc1 area=wan -13:38.513 [INFO] TestAgent.server: New leader elected: payload=Node-735d68a3-e078-7f6a-9308-cd439333dc60 -13:38.513 [INFO] TestAgent.server: Updating LAN server: server="Node-735d68a3-e078-7f6a-9308-cd439333dc60 (Addr: tcp/127.0.0.1:16436) (DC: dc1)" -13:38.513 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:38.528 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:38.528 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:38.528 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:38.528 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:38.528 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:38.528 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-735d68a3-e078-7f6a-9308-cd439333dc60 -13:38.528 [INFO] TestAgent.server: member joined, marking health alive: member=Node-735d68a3-e078-7f6a-9308-cd439333dc60 -13:38.529 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-735d68a3-e078-7f6a-9308-cd439333dc60 -13:38.550 [INFO] TestAgent.server: federation state anti-entropy synced -13:38.590 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:38.591 [INFO] TestAgent: Synced node info -13:38.606 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-735d68a3-e078-7f6a-9308-cd439333dc60 -13:38.610 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-735d68a3-e078-7f6a-9308-cd439333dc60 -13:38.610 [INFO] TestAgent: Requesting shutdown -13:38.610 [INFO] TestAgent.server: shutting down server -13:38.610 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:38.610 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:38.610 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:38.610 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:38.610 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:38.610 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:38.610 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:38.611 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:38.611 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:38.611 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:38.611 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:38.611 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:38.611 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:38.611 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:38.611 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:38.611Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:38.611Z [INFO] agent.router.manager: shutting down -13:38.611 [INFO] TestAgent: consul server down -13:38.611 [INFO] TestAgent: shutdown complete -13:38.611 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16389 network=tcp -13:38.611 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16389 network=udp -13:38.611 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16390 network=tcp -13:39.120 [INFO] TestAgent: Waiting for endpoints to shut down -13:39.120 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Legacy_UpdateUpsert (0.73s) -=== CONT TestACL_Legacy_Update -2021-10-31T23:13:37.772Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:37.832Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:37.902 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2f493f27-a4ec-33df-4eb0-17adf6ffc3f2 Address:127.0.0.1:16331}]" -13:37.903 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2f493f27-a4ec-33df-4eb0-17adf6ffc3f2.dc1 127.0.0.1 -13:37.903 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2f493f27-a4ec-33df-4eb0-17adf6ffc3f2 127.0.0.1 -2021-10-31T23:13:37.903Z [INFO] agent.router: Initializing LAN area manager -13:37.904 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16179 network=udp -13:37.905 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16331 [Follower]" leader= -13:37.906 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2f493f27-a4ec-33df-4eb0-17adf6ffc3f2.dc1 area=wan -13:37.906 [INFO] TestAgent.server: Adding LAN server: server="Node-2f493f27-a4ec-33df-4eb0-17adf6ffc3f2 (Addr: tcp/127.0.0.1:16331) (DC: dc1)" -13:37.906 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16179 network=tcp -13:37.907 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16180 network=tcp -13:37.907 [INFO] TestAgent: started state syncer -13:37.907 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16311 network=tcp -13:37.949 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:37.949 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16331 [Candidate]" term=2 -13:37.949 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:37.949 [DEBUG] TestAgent.server.raft: vote granted: from=2f493f27-a4ec-33df-4eb0-17adf6ffc3f2 term=2 tally=1 -13:37.949 [INFO] TestAgent.server.raft: election won: tally=1 -13:37.949 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16331 [Leader]" -13:37.949 [INFO] TestAgent.server: cluster leadership acquired -13:37.949 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16331 -13:37.949 [INFO] TestAgent.server: New leader elected: payload=Node-2f493f27-a4ec-33df-4eb0-17adf6ffc3f2 -13:37.950 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:37.953 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:37.953 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:37.953 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:37.953 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:37.953 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:37.953 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2f493f27-a4ec-33df-4eb0-17adf6ffc3f2 -13:37.953 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2f493f27-a4ec-33df-4eb0-17adf6ffc3f2 -13:37.962 [INFO] TestAgent.server: federation state anti-entropy synced -13:38.107 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:38.108 [INFO] TestAgent: Synced node info -13:38.636 [INFO] TestAgent: Requesting shutdown -13:38.636 [INFO] TestAgent.server: shutting down server -13:38.636 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:38.636 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:38.636 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:38.636 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:38.636 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:38.636 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:38.636 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:38.636 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:38.636 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:38.636 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:38.636 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:38.637Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:38.637Z [INFO] agent.router.manager: shutting down -13:38.637 [INFO] TestAgent: consul server down -13:38.637 [INFO] TestAgent: shutdown complete -13:38.637 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16179 network=tcp -13:38.637 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16179 network=udp -13:38.637 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16180 network=tcp -13:39.137 [INFO] TestAgent: Waiting for endpoints to shut down -13:39.137 [INFO] TestAgent: Endpoints down -=== RUN TestTxnEndpoint_Bad_Size_Item/allowed -2021-10-31T23:13:38.494Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:38.560Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:38.571 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2c64841e-c034-2cba-ce9f-b9c06f0d6ece Address:127.0.0.1:16450}]" -13:38.571 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16450 [Follower]" leader= -13:38.571 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2c64841e-c034-2cba-ce9f-b9c06f0d6ece.dc1 127.0.0.1 -13:38.572 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2c64841e-c034-2cba-ce9f-b9c06f0d6ece 127.0.0.1 -2021-10-31T23:13:38.572Z [INFO] agent.router: Initializing LAN area manager -13:38.572 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16452 network=udp -13:38.573 [INFO] TestAgent.server: Adding LAN server: server="Node-2c64841e-c034-2cba-ce9f-b9c06f0d6ece (Addr: tcp/127.0.0.1:16450) (DC: dc1)" -13:38.573 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2c64841e-c034-2cba-ce9f-b9c06f0d6ece.dc1 area=wan -13:38.573 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16452 network=tcp -13:38.583 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16453 network=tcp -13:38.583 [INFO] TestAgent: started state syncer -13:38.583 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16430 network=tcp -13:38.640 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:38.640 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16450 [Candidate]" term=2 -13:38.641 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:38.641 [DEBUG] TestAgent.server.raft: vote granted: from=2c64841e-c034-2cba-ce9f-b9c06f0d6ece term=2 tally=1 -13:38.641 [INFO] TestAgent.server.raft: election won: tally=1 -13:38.641 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16450 [Leader]" -13:38.641 [INFO] TestAgent.server: cluster leadership acquired -13:38.641 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16450 -13:38.641 [INFO] TestAgent.server: New leader elected: payload=Node-2c64841e-c034-2cba-ce9f-b9c06f0d6ece -13:38.646 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:38.648 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:38.648 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:38.648 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:38.648 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:38.648 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:38.648 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2c64841e-c034-2cba-ce9f-b9c06f0d6ece -13:38.648 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2c64841e-c034-2cba-ce9f-b9c06f0d6ece -13:38.648 [INFO] TestAgent.server: federation state anti-entropy synced -13:38.762 [INFO] TestAgent: Requesting shutdown -13:38.762 [INFO] TestAgent.server: shutting down server -13:38.762 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:38.762 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:38.762 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:38.762 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:38.762 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:38.762 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:38.762 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:38.762 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:38.762 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:38.762 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:38.762 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:38.762 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:38.762Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:38.762Z [INFO] agent.router.manager: shutting down -13:38.762 [INFO] TestAgent: consul server down -13:38.762 [INFO] TestAgent: shutdown complete -13:38.762 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16452 network=tcp -13:38.762 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16452 network=udp -13:38.762 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16453 network=tcp -13:39.263 [INFO] TestAgent: Waiting for endpoints to shut down -13:39.264 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_Keyring_InvalidRelayFactor (0.83s) -=== CONT TestTxnEndpoint_KV_Actions -=== RUN TestTxnEndpoint_KV_Actions/#00 -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:38.636Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:38.636Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:38.663Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:38.663Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:38.665 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7c85d815-6700-c472-0edc-0b00d3cc2815 Address:127.0.0.1:16415}]" -13:38.665 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16415 [Follower]" leader= -13:38.678 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7c85d815-6700-c472-0edc-0b00d3cc2815.dc1 127.0.0.1 -13:38.678 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7c85d815-6700-c472-0edc-0b00d3cc2815 127.0.0.1 -2021-10-31T23:13:38.678Z [INFO] agent.router: Initializing LAN area manager -13:38.679 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16333 network=udp -13:38.679 [INFO] TestAgent.server: Adding LAN server: server="Node-7c85d815-6700-c472-0edc-0b00d3cc2815 (Addr: tcp/127.0.0.1:16415) (DC: dc1)" -13:38.679 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7c85d815-6700-c472-0edc-0b00d3cc2815.dc1 area=wan -13:38.679 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16333 network=tcp -13:38.679 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16334 network=tcp -13:38.679 [INFO] TestAgent: started state syncer -13:38.679 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16360 network=tcp -13:38.720 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:38.720 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16415 [Candidate]" term=2 -13:38.721 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:38.721 [DEBUG] TestAgent.server.raft: vote granted: from=7c85d815-6700-c472-0edc-0b00d3cc2815 term=2 tally=1 -13:38.721 [INFO] TestAgent.server.raft: election won: tally=1 -13:38.721 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16415 [Leader]" -13:38.721 [INFO] TestAgent.server: cluster leadership acquired -13:38.722 [INFO] TestAgent.server: initializing acls -13:38.722 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:38.722 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:38.722 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:38.723 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:38.723 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:38.723 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:38.723 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7c85d815-6700-c472-0edc-0b00d3cc2815 -13:38.723 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7c85d815-6700-c472-0edc-0b00d3cc2815.dc1 -13:38.724 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:38.724 [INFO] TestAgent.server: New leader elected: payload=Node-7c85d815-6700-c472-0edc-0b00d3cc2815 -13:38.724 [INFO] TestAgent.server: Updating LAN server: server="Node-7c85d815-6700-c472-0edc-0b00d3cc2815 (Addr: tcp/127.0.0.1:16415) (DC: dc1)" -13:38.724 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7c85d815-6700-c472-0edc-0b00d3cc2815.dc1 area=wan -13:38.725 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:38.725 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:38.730 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:38.730 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:38.730 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:38.730 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7c85d815-6700-c472-0edc-0b00d3cc2815 -13:38.730 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7c85d815-6700-c472-0edc-0b00d3cc2815 -13:38.730 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7c85d815-6700-c472-0edc-0b00d3cc2815 -13:38.898 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7c85d815-6700-c472-0edc-0b00d3cc2815 -13:38.898 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-7c85d815-6700-c472-0edc-0b00d3cc2815 -13:38.900 [ERROR] TestAgent: failed to get remote exec job: error="Permission denied" -13:38.900 [INFO] TestAgent: Requesting shutdown -13:38.900 [INFO] TestAgent.server: shutting down server -13:38.900 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:38.900 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:38.900 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:38.900 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:38.900 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:38.900 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:38.900 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:38.900 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" -13:38.900 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:38.900 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:38.900 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:38.900 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:38.900 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:38.900 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:38.900 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:38.900 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:38.900Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:38.900Z [INFO] agent.router.manager: shutting down -13:38.901 [INFO] TestAgent: consul server down -13:38.901 [INFO] TestAgent: shutdown complete -13:38.901 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16333 network=tcp -13:38.901 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16333 network=udp -13:38.901 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16334 network=tcp -13:39.401 [INFO] TestAgent: Waiting for endpoints to shut down -13:39.401 [INFO] TestAgent: Endpoints down ---- PASS: TestRemoteExecGetSpec_ACLDeny (0.79s) -=== CONT TestTxnEndpoint_UpdateCheck -2021-10-31T23:13:38.620Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:38.690Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:38.692 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5a7ac6f3-be45-1aad-b44f-97ef27888755 Address:127.0.0.1:16310}]" -13:38.692 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5a7ac6f3-be45-1aad-b44f-97ef27888755.dc1 127.0.0.1 -13:38.693 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5a7ac6f3-be45-1aad-b44f-97ef27888755 127.0.0.1 -2021-10-31T23:13:38.693Z [INFO] agent.router: Initializing LAN area manager -13:38.694 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16310 [Follower]" leader= -13:38.694 [INFO] TestAgent.server: Adding LAN server: server="Node-5a7ac6f3-be45-1aad-b44f-97ef27888755 (Addr: tcp/127.0.0.1:16310) (DC: dc1)" -13:38.694 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5a7ac6f3-be45-1aad-b44f-97ef27888755.dc1 area=wan -13:38.694 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16431 network=tcp -13:38.710 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16431 network=udp -13:38.710 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16432 network=tcp -13:38.710 [INFO] TestAgent: started state syncer -13:38.710 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16332 network=tcp -13:38.753 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:38.753 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16310 [Candidate]" term=2 -13:38.754 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:38.754 [DEBUG] TestAgent.server.raft: vote granted: from=5a7ac6f3-be45-1aad-b44f-97ef27888755 term=2 tally=1 -13:38.754 [INFO] TestAgent.server.raft: election won: tally=1 -13:38.754 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16310 [Leader]" -13:38.754 [INFO] TestAgent.server: cluster leadership acquired -13:38.754 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16310 -13:38.755 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:38.755 [INFO] TestAgent.server: New leader elected: payload=Node-5a7ac6f3-be45-1aad-b44f-97ef27888755 -13:38.756 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:38.756 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:38.756 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:38.756 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:38.756 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:38.756 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a7ac6f3-be45-1aad-b44f-97ef27888755 -13:38.756 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5a7ac6f3-be45-1aad-b44f-97ef27888755 -13:38.757 [INFO] TestAgent.server: federation state anti-entropy synced -13:38.870 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:38.871 [INFO] TestAgent: Synced node info -13:38.926 [INFO] TestAgent.server.serf.wan: serf: Received install-key query -13:38.938 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-5a7ac6f3-be45-1aad-b44f-97ef27888755.dc1 -13:38.938 [INFO] TestAgent.server.serf.lan: serf: Received install-key query -13:38.991 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-5a7ac6f3-be45-1aad-b44f-97ef27888755 -13:38.991 [INFO] TestAgent.server.serf.wan: serf: Received use-key query -13:38.992 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-5a7ac6f3-be45-1aad-b44f-97ef27888755.dc1 -13:38.992 [INFO] TestAgent.server.serf.lan: serf: Received use-key query -13:38.992 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-5a7ac6f3-be45-1aad-b44f-97ef27888755 -13:38.993 [INFO] TestAgent.server.serf.wan: serf: Received remove-key query -13:38.993 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-5a7ac6f3-be45-1aad-b44f-97ef27888755.dc1 -13:38.993 [INFO] TestAgent.server.serf.lan: serf: Received remove-key query -13:38.993 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-5a7ac6f3-be45-1aad-b44f-97ef27888755 -13:38.993 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -13:38.993 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-5a7ac6f3-be45-1aad-b44f-97ef27888755.dc1 -13:38.993 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -13:38.994 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-5a7ac6f3-be45-1aad-b44f-97ef27888755 -13:38.994 [INFO] TestAgent: Requesting shutdown -13:38.994 [INFO] TestAgent.server: shutting down server -13:38.994 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:38.994 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:38.994 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:38.994 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:38.994 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:38.994 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:38.994 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:38.994 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:38.994 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:38.994 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:38.994 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:38.994Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:38.994Z [INFO] agent.router.manager: shutting down -13:38.994 [INFO] TestAgent: consul server down -13:38.994 [INFO] TestAgent: shutdown complete -13:38.994 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16431 network=tcp -13:38.994 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16431 network=udp -13:38.994 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16432 network=tcp -13:39.518 [INFO] TestAgent: Waiting for endpoints to shut down -13:39.518 [INFO] TestAgent: Endpoints down ---- PASS: TestOperator_KeyringUse (0.98s) -=== CONT TestPreparedQuery_Integration -2021-10-31T23:13:38.718Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:38.773Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:38.787 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4f81fe42-aed1-7381-0b45-7551d9cef63a Address:127.0.0.1:16464}]" -13:38.787 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4f81fe42-aed1-7381-0b45-7551d9cef63a.dc1 127.0.0.1 -13:38.788 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16464 [Follower]" leader= -13:38.788 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4f81fe42-aed1-7381-0b45-7551d9cef63a 127.0.0.1 -2021-10-31T23:13:38.788Z [INFO] agent.router: Initializing LAN area manager -13:38.788 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16354 network=udp -13:38.788 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4f81fe42-aed1-7381-0b45-7551d9cef63a.dc1 area=wan -13:38.788 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16354 network=tcp -13:38.788 [INFO] TestAgent.server: Adding LAN server: server="Node-4f81fe42-aed1-7381-0b45-7551d9cef63a (Addr: tcp/127.0.0.1:16464) (DC: dc1)" -13:38.789 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16355 network=tcp -13:38.789 [INFO] TestAgent: started state syncer -13:38.789 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16409 network=tcp -13:38.862 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:38.862 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16464 [Candidate]" term=2 -13:38.862 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:38.862 [DEBUG] TestAgent.server.raft: vote granted: from=4f81fe42-aed1-7381-0b45-7551d9cef63a term=2 tally=1 -13:38.862 [INFO] TestAgent.server.raft: election won: tally=1 -13:38.862 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16464 [Leader]" -13:38.864 [INFO] TestAgent.server: cluster leadership acquired -13:38.864 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16464 -13:38.865 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:38.870 [INFO] TestAgent.server: New leader elected: payload=Node-4f81fe42-aed1-7381-0b45-7551d9cef63a -13:38.875 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:38.875 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:38.875 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:38.875 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:38.875 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:38.875 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4f81fe42-aed1-7381-0b45-7551d9cef63a -13:38.875 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4f81fe42-aed1-7381-0b45-7551d9cef63a -13:38.877 [INFO] TestAgent.server: federation state anti-entropy synced -13:38.995 [DEBUG] TestAgent: received remote exec event: id=3e24085c-35b3-7e37-3c01-b4db490fca73 -13:38.995 [INFO] TestAgent: remote exec script: script=uptime -13:39.023 [INFO] TestAgent: Requesting shutdown -13:39.023 [INFO] TestAgent.server: shutting down server -13:39.023 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:39.023 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:39.023 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:39.023 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:39.023 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:39.023 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:39.023 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:39.023 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:39.023 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:39.023 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:39.023 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:39.023 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:39.023Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:39.023Z [INFO] agent.router.manager: shutting down -13:39.023 [INFO] TestAgent: consul server down -13:39.023 [INFO] TestAgent: shutdown complete -13:39.023 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16354 network=tcp -13:39.023 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16354 network=udp -13:39.023 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16355 network=tcp -13:39.525 [INFO] TestAgent: Waiting for endpoints to shut down -13:39.525 [INFO] TestAgent: Endpoints down ---- PASS: TestHandleRemoteExec (0.85s) -=== CONT TestRemoteExecGetSpec_ACLAgentToken -2021-10-31T23:13:38.687Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:38.734Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:38.736 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6186ea3e-c168-539f-987e-6d9d094b5d06 Address:127.0.0.1:16352}]" -13:38.736 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6186ea3e-c168-539f-987e-6d9d094b5d06.dc1 127.0.0.1 -13:38.737 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16352 [Follower]" leader= -13:38.737 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6186ea3e-c168-539f-987e-6d9d094b5d06 127.0.0.1 -2021-10-31T23:13:38.737Z [INFO] agent.router: Initializing LAN area manager -13:38.737 [INFO] TestAgent.server: Adding LAN server: server="Node-6186ea3e-c168-539f-987e-6d9d094b5d06 (Addr: tcp/127.0.0.1:16352) (DC: dc1)" -13:38.738 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16361 network=udp -13:38.738 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6186ea3e-c168-539f-987e-6d9d094b5d06.dc1 area=wan -13:38.738 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16361 network=tcp -13:38.738 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16362 network=tcp -13:38.738 [INFO] TestAgent: started state syncer -13:38.739 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16353 network=tcp -13:38.777 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:38.778 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16352 [Candidate]" term=2 -13:38.778 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:38.778 [DEBUG] TestAgent.server.raft: vote granted: from=6186ea3e-c168-539f-987e-6d9d094b5d06 term=2 tally=1 -13:38.778 [INFO] TestAgent.server.raft: election won: tally=1 -13:38.778 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16352 [Leader]" -13:38.778 [INFO] TestAgent.server: cluster leadership acquired -13:38.778 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16352 -13:38.778 [INFO] TestAgent.server: New leader elected: payload=Node-6186ea3e-c168-539f-987e-6d9d094b5d06 -13:38.779 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:38.780 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:38.780 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:38.780 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:38.780 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:38.780 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:38.780 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6186ea3e-c168-539f-987e-6d9d094b5d06 -13:38.780 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6186ea3e-c168-539f-987e-6d9d094b5d06 -13:38.781 [INFO] TestAgent.server: federation state anti-entropy synced -13:39.042 [DEBUG] TestAgent: received remote exec event: id=23e3d740-92a3-b054-998a-992e8282fc65 -13:39.043 [INFO] TestAgent: remote exec script: script="echo failing;exit 2" -13:39.071 [INFO] TestAgent: Requesting shutdown -13:39.071 [INFO] TestAgent.server: shutting down server -13:39.071 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:39.071 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:39.071 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:39.071 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:39.071 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:39.071 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:39.071 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:39.071 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:39.071 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:39.071 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:39.071 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:39.071 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:39.071Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:39.071Z [INFO] agent.router.manager: shutting down -13:39.071 [INFO] TestAgent: consul server down -13:39.071 [INFO] TestAgent: shutdown complete -13:39.071 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16361 network=tcp -13:39.071 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16361 network=udp -13:39.071 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16362 network=tcp -13:39.572 [INFO] TestAgent: Waiting for endpoints to shut down -13:39.572 [INFO] TestAgent: Endpoints down ---- PASS: TestHandleRemoteExecFailed (0.93s) -=== CONT TestRemoteExecGetSpec_ACLToken +2022-12-04T05:49:41.566Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:41.576Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:41.577 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6b1ec41a-a21d-cfe4-85e1-6c833a835f14 Address:127.0.0.1:23737}]" +49:41.577 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23737 [Follower]" leader= +49:41.578 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:41.578 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:41.578Z [INFO] agent.router: Initializing LAN area manager +49:41.578 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23737) (DC: dc1)" +49:41.578 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:41.578 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23613 network=udp +49:41.578 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23613 network=tcp +49:41.579 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23614 network=tcp +49:41.579 [INFO] TestAgent: started state syncer +49:41.579 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23689 network=tcp +49:41.634 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:41.634 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23737 [Candidate]" term=2 +49:41.634 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:41.634 [DEBUG] TestAgent.server.raft: vote granted: from=6b1ec41a-a21d-cfe4-85e1-6c833a835f14 term=2 tally=1 +49:41.634 [INFO] TestAgent.server.raft: election won: tally=1 +49:41.634 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23737 [Leader]" +49:41.634 [INFO] TestAgent.server: cluster leadership acquired +49:41.634 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23737 +49:41.634 [INFO] TestAgent.server: New leader elected: payload=test-node +49:41.634 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:41.635 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:41.635 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:41.636 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:41.636 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:41.636 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:41.636 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:41.636 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:41.636 [INFO] TestAgent.server: federation state anti-entropy synced +49:41.803 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=979.895µs client=127.0.0.1:33263 client_network=udp +49:41.803 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=775.131µs client=127.0.0.1:59141 client_network=udp +49:41.804 [DEBUG] TestAgent.dns: request served from client: name=923bfe0c-569c-d537-59fb-1835a66843a4.query.consul. type=ANY class=IN latency=913.95µs client=127.0.0.1:39920 client_network=udp +49:41.805 [INFO] TestAgent: Requesting shutdown +49:41.805 [INFO] TestAgent.server: shutting down server +49:41.805 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:41.805 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:41.805 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:41.805 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:41.805 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:41.805 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:41.805 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:41.805 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:41.805 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:41.805 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:41.805 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:41.805 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:41.805Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:41.805Z [INFO] agent.router.manager: shutting down +49:41.805 [INFO] TestAgent: consul server down +49:41.805 [INFO] TestAgent: shutdown complete +49:41.805 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23613 network=tcp +49:41.805 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23613 network=udp +49:41.805 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23614 network=tcp +49:42.305 [INFO] TestAgent: Waiting for endpoints to shut down +49:42.305 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{2_2_2_2_2_2_2_2_2_2_2} +2022-12-04T05:49:41.618Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:41.628Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:41.629 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ac57e53f-25b2-24d6-4fbb-e953c148e54f Address:127.0.0.1:23695}]" +49:41.629 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23695 [Follower]" leader= +49:41.630 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:41.630 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:41.630Z [INFO] agent.router: Initializing LAN area manager +49:41.630 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23695) (DC: dc1)" +49:41.630 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:41.630 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23690 network=udp +49:41.630 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23690 network=tcp +49:41.631 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23691 network=tcp +49:41.631 [INFO] TestAgent: started state syncer +49:41.631 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23486 network=tcp +49:41.672 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:41.672 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23695 [Candidate]" term=2 +49:41.672 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:41.672 [DEBUG] TestAgent.server.raft: vote granted: from=ac57e53f-25b2-24d6-4fbb-e953c148e54f term=2 tally=1 +49:41.672 [INFO] TestAgent.server.raft: election won: tally=1 +49:41.672 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23695 [Leader]" +49:41.673 [INFO] TestAgent.server: cluster leadership acquired +49:41.673 [INFO] TestAgent.server: New leader elected: payload=test-node +49:41.673 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23695 +49:41.673 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:41.674 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:41.674 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:41.674 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:41.674 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:41.674 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:41.674 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:41.674 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:41.675 [INFO] TestAgent.server: federation state anti-entropy synced +49:41.730 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:41.731 [INFO] TestAgent: Synced node info +49:41.798 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:41.798 [DEBUG] TestAgent: Node info in sync +49:41.798 [DEBUG] TestAgent: Node info in sync +49:41.952 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.14945ms client=127.0.0.1:34499 client_network=udp +49:41.954 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.009838ms client=127.0.0.1:42925 client_network=udp +49:41.955 [DEBUG] TestAgent.dns: request served from client: name=47b381f2-31e9-c672-ae96-8374d3bc4029.query.consul. type=AAAA class=IN latency=868.303µs client=127.0.0.1:53486 client_network=udp +49:41.955 [INFO] TestAgent: Requesting shutdown +49:41.955 [INFO] TestAgent.server: shutting down server +49:41.955 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:41.955 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:41.955 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:41.955 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:41.955 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:41.955 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:41.955 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:41.955 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:41.955 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:41.955 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:41.955 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:41.955Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:41.955Z [INFO] agent.router.manager: shutting down +49:41.956 [INFO] TestAgent: consul server down +49:41.956 [INFO] TestAgent: shutdown complete +49:41.956 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23690 network=tcp +49:41.956 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23690 network=udp +49:41.956 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23691 network=tcp +49:42.456 [INFO] TestAgent: Waiting for endpoints to shut down +49:42.456 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{2_2_2_2_2_2_2_2_2_2_2} +2022-12-04T05:49:41.787Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:41.797Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:41.798 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a65482cf-d409-13ee-0021-24421c0eee01 Address:127.0.0.1:23485}]" +49:41.798 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23485 [Follower]" leader= +49:41.799 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:41.799 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:41.799Z [INFO] agent.router: Initializing LAN area manager +49:41.799 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23485) (DC: dc1)" +49:41.799 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23732 network=udp +49:41.799 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:41.799 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23732 network=tcp +49:41.800 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23733 network=tcp +49:41.800 [INFO] TestAgent: started state syncer +49:41.800 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23668 network=tcp +49:41.850 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:41.851 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23485 [Candidate]" term=2 +49:41.851 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:41.851 [DEBUG] TestAgent.server.raft: vote granted: from=a65482cf-d409-13ee-0021-24421c0eee01 term=2 tally=1 +49:41.851 [INFO] TestAgent.server.raft: election won: tally=1 +49:41.851 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23485 [Leader]" +49:41.851 [INFO] TestAgent.server: cluster leadership acquired +49:41.851 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23485 +49:41.852 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:41.852 [INFO] TestAgent.server: New leader elected: payload=test-node +49:41.855 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:41.855 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:41.855 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:41.855 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:41.855 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:41.855 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:41.855 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:41.855 [INFO] TestAgent.server: federation state anti-entropy synced +49:41.957 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=879.146µs client=127.0.0.1:36024 client_network=udp +49:41.958 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=662.104µs client=127.0.0.1:60864 client_network=udp +49:41.959 [DEBUG] TestAgent.dns: request served from client: name=740c19bf-7163-d9cf-cbcf-9ec37f231adf.query.consul. type=A class=IN latency=723.065µs client=127.0.0.1:55336 client_network=udp +49:41.959 [INFO] TestAgent: Requesting shutdown +49:41.959 [INFO] TestAgent.server: shutting down server +49:41.959 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:41.959 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:41.959 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:41.959 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:41.959 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:41.959 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:41.959 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:41.959 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:41.959 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:41.959 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:41.959 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:41.959 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:41.959Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:41.959Z [INFO] agent.router.manager: shutting down +49:41.959 [INFO] TestAgent: consul server down +49:41.959 [INFO] TestAgent: shutdown complete +49:41.959 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23732 network=tcp +49:41.959 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23732 network=udp +49:41.959 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23733 network=tcp +49:42.460 [INFO] TestAgent: Waiting for endpoints to shut down +49:42.460 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{2_2_2_2_2_2_2_2_2_2_2} +2022-12-04T05:49:41.867Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:41.877Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:41.879 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b127ca33-6b12-cca5-b2c4-8c9728961be7 Address:127.0.0.1:23688}]" +49:41.879 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23688 [Follower]" leader= +49:41.880 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:41.880 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:41.880Z [INFO] agent.router: Initializing LAN area manager +49:41.880 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23718 network=udp +49:41.880 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23688) (DC: dc1)" +49:41.880 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:41.881 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23718 network=tcp +49:41.881 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23719 network=tcp +49:41.881 [INFO] TestAgent: started state syncer +49:41.881 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23793 network=tcp +49:41.920 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:41.920 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23688 [Candidate]" term=2 +49:41.920 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:41.920 [DEBUG] TestAgent.server.raft: vote granted: from=b127ca33-6b12-cca5-b2c4-8c9728961be7 term=2 tally=1 +49:41.920 [INFO] TestAgent.server.raft: election won: tally=1 +49:41.920 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23688 [Leader]" +49:41.920 [INFO] TestAgent.server: cluster leadership acquired +49:41.920 [INFO] TestAgent.server: New leader elected: payload=test-node +49:41.921 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23688 +49:41.921 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:41.923 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:41.923 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:41.923 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:41.923 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:41.923 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:41.923 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:41.923 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:41.923 [INFO] TestAgent.server: federation state anti-entropy synced +49:42.034 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=942.578µs client=127.0.0.1:36285 client_network=udp +49:42.035 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=754.663µs client=127.0.0.1:58206 client_network=udp +49:42.036 [DEBUG] TestAgent.dns: request served from client: name=98886b94-e1d2-be71-69fa-12ec0799e354.query.consul. type=AAAA class=IN latency=883.846µs client=127.0.0.1:55758 client_network=udp +49:42.036 [INFO] TestAgent: Requesting shutdown +49:42.036 [INFO] TestAgent.server: shutting down server +49:42.036 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:42.036 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:42.036 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:42.036 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:42.037 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:42.037 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:42.037 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:42.037 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:42.037 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:42.037 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:42.037 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:42.037 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:42.037Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.037Z [INFO] agent.router.manager: shutting down +49:42.037 [INFO] TestAgent: consul server down +49:42.037 [INFO] TestAgent: shutdown complete +49:42.037 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23718 network=tcp +49:42.037 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23718 network=udp +49:42.037 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23719 network=tcp +49:42.540 [INFO] TestAgent: Waiting for endpoints to shut down +49:42.540 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{6_6_6_6_6_6_6_5_6_6_-5} [WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:38.788Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:38.788Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:38.827Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:38.827Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:38.834 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bd6d2456-4bd3-0846-cb86-2d0156d01d37 Address:127.0.0.1:16429}]" -13:38.835 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bd6d2456-4bd3-0846-cb86-2d0156d01d37.dc1 127.0.0.1 -13:38.835 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bd6d2456-4bd3-0846-cb86-2d0156d01d37 127.0.0.1 -2021-10-31T23:13:38.835Z [INFO] agent.router: Initializing LAN area manager -13:38.836 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16410 network=udp -13:38.836 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16429 [Follower]" leader= -13:38.836 [INFO] TestAgent.server: Adding LAN server: server="Node-bd6d2456-4bd3-0846-cb86-2d0156d01d37 (Addr: tcp/127.0.0.1:16429) (DC: dc1)" -13:38.836 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bd6d2456-4bd3-0846-cb86-2d0156d01d37.dc1 area=wan -13:38.836 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16410 network=tcp -13:38.837 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16411 network=tcp -13:38.837 [INFO] TestAgent: started state syncer -13:38.837 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16374 network=tcp -13:38.886 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:38.893 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:38.893 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16429 [Candidate]" term=2 -13:38.894 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:38.894 [DEBUG] TestAgent.server.raft: vote granted: from=bd6d2456-4bd3-0846-cb86-2d0156d01d37 term=2 tally=1 -13:38.894 [INFO] TestAgent.server.raft: election won: tally=1 -13:38.894 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16429 [Leader]" -13:38.894 [INFO] TestAgent.server: cluster leadership acquired -13:38.894 [INFO] TestAgent.server: initializing acls -13:38.895 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:38.895 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:38.895 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:38.896 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:38.896 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:38.896 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:38.896 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-bd6d2456-4bd3-0846-cb86-2d0156d01d37 -13:38.896 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-bd6d2456-4bd3-0846-cb86-2d0156d01d37.dc1 -13:38.897 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:38.897 [INFO] TestAgent.server: New leader elected: payload=Node-bd6d2456-4bd3-0846-cb86-2d0156d01d37 -13:38.898 [INFO] TestAgent.server: Updating LAN server: server="Node-bd6d2456-4bd3-0846-cb86-2d0156d01d37 (Addr: tcp/127.0.0.1:16429) (DC: dc1)" -13:38.898 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-bd6d2456-4bd3-0846-cb86-2d0156d01d37.dc1 area=wan -13:38.916 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:38.916 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:38.916 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:38.916 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:38.916 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:38.916 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bd6d2456-4bd3-0846-cb86-2d0156d01d37 -13:38.916 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bd6d2456-4bd3-0846-cb86-2d0156d01d37 -13:38.917 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bd6d2456-4bd3-0846-cb86-2d0156d01d37 -13:38.917 [INFO] TestAgent.server: federation state anti-entropy synced -13:38.956 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -13:38.991 [WARN] TestAgent: Node info update blocked by ACLs: node=bd6d2456-4bd3-0846-cb86-2d0156d01d37 accessorID=00000000-0000-0000-0000-000000000002 -13:39.239 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bd6d2456-4bd3-0846-cb86-2d0156d01d37 -13:39.239 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bd6d2456-4bd3-0846-cb86-2d0156d01d37 -13:39.240 [ERROR] TestAgent: failed to ack remote exec job: error="Permission denied" -13:39.240 [ERROR] TestAgent: failed to write output for remote exec job: error="Permission denied" -13:39.240 [ERROR] TestAgent: failed to write output for remote exec job: error="Permission denied" -13:39.240 [INFO] TestAgent: Requesting shutdown -13:39.240 [INFO] TestAgent.server: shutting down server -13:39.240 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:39.240 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:39.240 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:39.240 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:39.240 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:39.240 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:39.240 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:39.241 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:39.241 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:39.241 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:39.241 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:39.241 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:39.241 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:39.241 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:39.241 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:39.242Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:39.242Z [INFO] agent.router.manager: shutting down -13:39.242 [INFO] TestAgent: consul server down -13:39.242 [INFO] TestAgent: shutdown complete -13:39.242 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16410 network=tcp -13:39.242 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16410 network=udp -13:39.242 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16411 network=tcp -13:39.759 [INFO] TestAgent: Waiting for endpoints to shut down -13:39.759 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:38.869Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:38.869Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:38.938Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:38.938Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:38.941 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ab40355c-4418-10f7-8ce5-d7ccb7ce1e95 Address:127.0.0.1:16373}]" -13:38.946 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16373 [Follower]" leader= -13:38.990 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ab40355c-4418-10f7-8ce5-d7ccb7ce1e95.dc1 127.0.0.1 -13:38.991 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ab40355c-4418-10f7-8ce5-d7ccb7ce1e95 127.0.0.1 -2021-10-31T23:13:38.991Z [INFO] agent.router: Initializing LAN area manager -13:38.991 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16445 network=udp -13:38.992 [INFO] TestAgent.server: Adding LAN server: server="Node-ab40355c-4418-10f7-8ce5-d7ccb7ce1e95 (Addr: tcp/127.0.0.1:16373) (DC: dc1)" -13:38.992 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ab40355c-4418-10f7-8ce5-d7ccb7ce1e95.dc1 area=wan -13:38.992 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16445 network=tcp -13:38.993 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16446 network=tcp -13:38.993 [INFO] TestAgent: started state syncer -13:38.993 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16423 network=tcp -13:38.995 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:38.995 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16373 [Candidate]" term=2 -13:38.995 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:38.995 [DEBUG] TestAgent.server.raft: vote granted: from=ab40355c-4418-10f7-8ce5-d7ccb7ce1e95 term=2 tally=1 -13:38.995 [INFO] TestAgent.server.raft: election won: tally=1 -13:38.995 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16373 [Leader]" -13:38.995 [INFO] TestAgent.server: cluster leadership acquired -13:38.995 [INFO] TestAgent.server: initializing acls -13:38.996 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:38.996 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:38.996 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:38.996 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:38.996 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:38.996 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:38.996 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ab40355c-4418-10f7-8ce5-d7ccb7ce1e95 -13:38.997 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ab40355c-4418-10f7-8ce5-d7ccb7ce1e95.dc1 -13:38.997 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:38.997 [INFO] TestAgent.server: New leader elected: payload=Node-ab40355c-4418-10f7-8ce5-d7ccb7ce1e95 -13:38.998 [INFO] TestAgent.server: Updating LAN server: server="Node-ab40355c-4418-10f7-8ce5-d7ccb7ce1e95 (Addr: tcp/127.0.0.1:16373) (DC: dc1)" -13:38.998 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ab40355c-4418-10f7-8ce5-d7ccb7ce1e95.dc1 area=wan -13:39.027 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:39.027 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:39.027 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:39.027 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:39.027 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:39.027 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ab40355c-4418-10f7-8ce5-d7ccb7ce1e95 -13:39.027 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ab40355c-4418-10f7-8ce5-d7ccb7ce1e95 -13:39.028 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ab40355c-4418-10f7-8ce5-d7ccb7ce1e95 -13:39.028 [INFO] TestAgent.server: federation state anti-entropy synced -13:39.167 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:39.167 [INFO] TestAgent: Synced node info -13:39.240 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ab40355c-4418-10f7-8ce5-d7ccb7ce1e95 -13:39.240 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-ab40355c-4418-10f7-8ce5-d7ccb7ce1e95 -13:39.242 [INFO] TestAgent: Requesting shutdown -13:39.242 [INFO] TestAgent.server: shutting down server -13:39.242 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:39.242 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:39.242 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:39.242 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:39.242 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:39.242 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:39.242 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:39.242 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:39.242 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:39.242 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:39.242 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:39.242 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:39.242 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:39.242 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:39.242 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -2021-10-31T23:13:39.242Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:39.243Z [INFO] agent.router.manager: shutting down -13:39.243 [INFO] TestAgent: consul server down -13:39.243 [INFO] TestAgent: shutdown complete -13:39.243 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16445 network=tcp -13:39.243 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16445 network=udp -13:39.243 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16446 network=tcp -13:39.759 [INFO] TestAgent: Waiting for endpoints to shut down -13:39.759 [INFO] TestAgent: Endpoints down ---- PASS: TestRemoteExecWrites_ACLDeny (1.01s) ---- PASS: TestRemoteExecWrites_ACLToken (0.95s) -=== CONT TestRemoteExecGetSpec -=== CONT TestPreparedQuery_Delete -2021-10-31T23:13:38.797Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:38.797Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:38.875Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:38.875Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:38.880 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bae29851-c665-decb-2af4-b9d4863b02b8 Address:127.0.0.1:16443}]" -13:38.881 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bae29851-c665-decb-2af4-b9d4863b02b8.dc1 127.0.0.1 -13:38.882 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16443 [Follower]" leader= -13:38.887 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bae29851-c665-decb-2af4-b9d4863b02b8 127.0.0.1 -2021-10-31T23:13:38.887Z [INFO] agent.router: Initializing LAN area manager -13:38.887 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16375 network=udp -13:38.887 [INFO] TestAgent.server: Adding LAN server: server="Node-bae29851-c665-decb-2af4-b9d4863b02b8 (Addr: tcp/127.0.0.1:16443) (DC: dc1)" -13:38.887 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bae29851-c665-decb-2af4-b9d4863b02b8.dc1 area=wan -13:38.887 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16375 network=tcp -13:38.889 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16376 network=tcp -13:38.889 [INFO] TestAgent: started state syncer -13:38.889 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16444 network=tcp -13:38.941 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:38.942 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:38.942 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16443 [Candidate]" term=2 -13:38.942 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:38.942 [DEBUG] TestAgent.server.raft: vote granted: from=bae29851-c665-decb-2af4-b9d4863b02b8 term=2 tally=1 -13:38.942 [INFO] TestAgent.server.raft: election won: tally=1 -13:38.942 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16443 [Leader]" -13:38.942 [INFO] TestAgent.server: cluster leadership acquired -13:38.953 [INFO] TestAgent.server: New leader elected: payload=Node-bae29851-c665-decb-2af4-b9d4863b02b8 -13:39.000 [INFO] TestAgent.server: initializing acls -13:39.001 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:39.001 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:39.001 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:39.002 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:39.002 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:39.002 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:39.002 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-bae29851-c665-decb-2af4-b9d4863b02b8 -13:39.002 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-bae29851-c665-decb-2af4-b9d4863b02b8.dc1 -13:39.003 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:39.004 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:39.004 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:39.004 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:39.004 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:39.004 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:39.004 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bae29851-c665-decb-2af4-b9d4863b02b8 -13:39.004 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bae29851-c665-decb-2af4-b9d4863b02b8 -13:39.005 [INFO] TestAgent.server: Updating LAN server: server="Node-bae29851-c665-decb-2af4-b9d4863b02b8 (Addr: tcp/127.0.0.1:16443) (DC: dc1)" -13:39.005 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bae29851-c665-decb-2af4-b9d4863b02b8 -13:39.005 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-bae29851-c665-decb-2af4-b9d4863b02b8.dc1 area=wan -13:39.006 [INFO] TestAgent.server: federation state anti-entropy synced -13:39.258 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:39.259 [INFO] TestAgent: Synced node info -13:39.259 [DEBUG] TestAgent: Node info in sync -13:39.278 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bae29851-c665-decb-2af4-b9d4863b02b8 -13:39.278 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-bae29851-c665-decb-2af4-b9d4863b02b8 -13:39.280 [INFO] TestAgent: Requesting shutdown -13:39.280 [INFO] TestAgent.server: shutting down server -13:39.280 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:39.280 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:39.280 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:39.280 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:39.280 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:39.280 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:39.280 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:39.280 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:39.280 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:39.280 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:39.280 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:39.280 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:39.280 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:39.280 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:39.280 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:39.280Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:39.280Z [INFO] agent.router.manager: shutting down -13:39.280 [INFO] TestAgent: consul server down -13:39.280 [INFO] TestAgent: shutdown complete -13:39.280 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16375 network=tcp -13:39.280 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16375 network=udp -13:39.280 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16376 network=tcp -13:39.781 [INFO] TestAgent: Waiting for endpoints to shut down -13:39.782 [INFO] TestAgent: Endpoints down ---- PASS: TestRemoteExecWrites_ACLAgentToken (1.00s) -=== CONT TestPreparedQuery_parseLimit ---- PASS: TestPreparedQuery_parseLimit (0.00s) -=== CONT TestTxnEndpoint_Bad_Size_Ops +2022-12-04T05:49:41.956Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:41.967Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:41.969 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2f04a209-1c2c-7a82-0d1d-3ca06973a920 Address:127.0.0.1:23478}]" +49:41.969 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23478 [Follower]" leader= +49:41.969 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:41.969 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:41.969Z [INFO] agent.router: Initializing LAN area manager +49:41.970 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23478) (DC: dc1)" +49:41.970 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:41.970 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23760 network=udp +49:41.970 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23760 network=tcp +49:41.970 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23761 network=tcp +49:41.970 [INFO] TestAgent: started state syncer +49:41.970 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23472 network=tcp +49:42.010 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:42.010 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23478 [Candidate]" term=2 +49:42.010 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:42.010 [DEBUG] TestAgent.server.raft: vote granted: from=2f04a209-1c2c-7a82-0d1d-3ca06973a920 term=2 tally=1 +49:42.010 [INFO] TestAgent.server.raft: election won: tally=1 +49:42.010 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23478 [Leader]" +49:42.010 [INFO] TestAgent.server: cluster leadership acquired +49:42.010 [INFO] TestAgent.server: New leader elected: payload=test-node +49:42.010 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23478 +49:42.011 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:42.013 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:42.013 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:42.013 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:42.013 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:42.013 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:42.013 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:42.013 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:42.013 [INFO] TestAgent.server: federation state anti-entropy synced +49:42.067 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.05045ms client=127.0.0.1:49921 client_network=udp +49:42.068 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=865.711µs client=127.0.0.1:60016 client_network=udp +49:42.069 [DEBUG] TestAgent.dns: request served from client: name=b3790091-c5d0-2a60-8940-5ae2cb28ea56.query.consul. type=ANY class=IN latency=1.01511ms client=127.0.0.1:39959 client_network=udp +49:42.069 [INFO] TestAgent: Requesting shutdown +49:42.069 [INFO] TestAgent.server: shutting down server +49:42.069 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:42.069 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:42.069 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:42.069 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:42.069 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:42.069 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:42.070 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:42.070 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:42.070 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:42.070 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:42.070 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:42.069 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +2022-12-04T05:49:42.070Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.070Z [INFO] agent.router.manager: shutting down +49:42.070 [INFO] TestAgent: consul server down +49:42.070 [INFO] TestAgent: shutdown complete +49:42.070 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23760 network=tcp +49:42.070 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23760 network=udp +49:42.070 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23761 network=tcp +49:42.572 [INFO] TestAgent: Waiting for endpoints to shut down +49:42.572 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{1_1_1_1_1_1_1_1_1_1_1} +2022-12-04T05:49:41.711Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:41.711Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:41.721Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:41.721Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:41.722 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cdbe90da-df0a-b6c2-834a-1146c2902ef9 Address:127.0.0.1:23730}]" +49:41.722 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23730 [Follower]" leader= +49:41.723 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cdbe90da-df0a-b6c2-834a-1146c2902ef9.dc1 127.0.0.1 +49:41.723 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cdbe90da-df0a-b6c2-834a-1146c2902ef9 127.0.0.1 +2022-12-04T05:49:41.723Z [INFO] agent.router: Initializing LAN area manager +49:41.723 [INFO] TestAgent.server: Adding LAN server: server="Node-cdbe90da-df0a-b6c2-834a-1146c2902ef9 (Addr: tcp/127.0.0.1:23730) (DC: dc1)" +49:41.723 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cdbe90da-df0a-b6c2-834a-1146c2902ef9.dc1 area=wan +49:41.723 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23487 network=udp +49:41.723 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23487 network=tcp +49:41.724 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23488 network=tcp +49:41.724 [INFO] TestAgent: started state syncer +49:41.724 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23731 network=tcp +49:41.760 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:41.760 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23730 [Candidate]" term=2 +49:41.760 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:41.760 [DEBUG] TestAgent.server.raft: vote granted: from=cdbe90da-df0a-b6c2-834a-1146c2902ef9 term=2 tally=1 +49:41.760 [INFO] TestAgent.server.raft: election won: tally=1 +49:41.760 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23730 [Leader]" +49:41.760 [INFO] TestAgent.server: cluster leadership acquired +49:41.760 [INFO] TestAgent.server: New leader elected: payload=Node-cdbe90da-df0a-b6c2-834a-1146c2902ef9 +49:41.760 [INFO] TestAgent.server: initializing acls +49:41.760 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:41.760 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:41.761 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:41.761 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:41.761 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:41.761 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:41.761 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-cdbe90da-df0a-b6c2-834a-1146c2902ef9 +49:41.761 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-cdbe90da-df0a-b6c2-834a-1146c2902ef9.dc1 +49:41.761 [INFO] TestAgent.server: Updating LAN server: server="Node-cdbe90da-df0a-b6c2-834a-1146c2902ef9 (Addr: tcp/127.0.0.1:23730) (DC: dc1)" +49:41.761 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-cdbe90da-df0a-b6c2-834a-1146c2902ef9.dc1 area=wan +49:41.762 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:41.763 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:41.763 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:41.763 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:41.763 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:41.763 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:41.763 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cdbe90da-df0a-b6c2-834a-1146c2902ef9 +49:41.763 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cdbe90da-df0a-b6c2-834a-1146c2902ef9 +49:41.764 [INFO] TestAgent.server: federation state anti-entropy synced +49:41.764 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cdbe90da-df0a-b6c2-834a-1146c2902ef9 +49:41.886 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:41.887 [INFO] TestAgent: Synced node info +49:42.080 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-cdbe90da-df0a-b6c2-834a-1146c2902ef9 +49:42.080 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-cdbe90da-df0a-b6c2-834a-1146c2902ef9 +49:42.081 [INFO] TestAgent: Requesting shutdown +49:42.081 [INFO] TestAgent.server: shutting down server +49:42.081 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:42.081 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:42.081 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:42.081 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:42.081 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:42.081 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:42.081 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:42.081 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:42.081 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:42.081 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:42.081 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:42.081 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:42.081 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:42.081 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:42.081 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:42.081Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.081Z [INFO] agent.router.manager: shutting down +49:42.081 [INFO] TestAgent: consul server down +49:42.081 [INFO] TestAgent: shutdown complete +49:42.081 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23487 network=tcp +49:42.081 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23487 network=udp +49:42.081 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23488 network=tcp +49:42.581 [INFO] TestAgent: Waiting for endpoints to shut down +49:42.581 [INFO] TestAgent: Endpoints down +--- PASS: TestACL_Legacy_Get (1.62s) + --- PASS: TestACL_Legacy_Get/wrong_id (0.74s) + --- PASS: TestACL_Legacy_Get/right_id (0.88s) +=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{1_1_1_1_1_1_1_1_1_1_1} +2022-12-04T05:49:41.844Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:41.865Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:41.866 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:81dd5a66-b9c5-3ed3-4101-c34c782fd4d0 Address:127.0.0.1:23702}]" +49:41.867 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:41.867 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:41.867Z [INFO] agent.router: Initializing LAN area manager +49:41.868 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23669 network=udp +49:41.868 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23702 [Follower]" leader= +49:41.868 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:41.868 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23669 network=tcp +49:41.868 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23702) (DC: dc1)" +49:41.868 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23670 network=tcp +49:41.868 [INFO] TestAgent: started state syncer +49:41.868 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23717 network=tcp +49:41.930 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:41.930 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23702 [Candidate]" term=2 +49:41.930 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:41.930 [DEBUG] TestAgent.server.raft: vote granted: from=81dd5a66-b9c5-3ed3-4101-c34c782fd4d0 term=2 tally=1 +49:41.930 [INFO] TestAgent.server.raft: election won: tally=1 +49:41.930 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23702 [Leader]" +49:41.930 [INFO] TestAgent.server: cluster leadership acquired +49:41.930 [INFO] TestAgent.server: New leader elected: payload=test-node +49:41.930 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23702 +49:41.931 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:41.932 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:41.932 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:41.932 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:41.932 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:41.932 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:41.932 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:41.932 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:41.932 [INFO] TestAgent.server: federation state anti-entropy synced +49:42.086 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.011464ms client=127.0.0.1:42328 client_network=udp +49:42.087 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=784.756µs client=127.0.0.1:42930 client_network=udp +49:42.088 [DEBUG] TestAgent.dns: request served from client: name=9e3a9e1d-30ef-351c-db73-d5fea1ff0874.query.consul. type=ANY class=IN latency=916.599µs client=127.0.0.1:39107 client_network=udp +49:42.088 [INFO] TestAgent: Requesting shutdown +49:42.088 [INFO] TestAgent.server: shutting down server +49:42.088 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:42.088 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:42.088 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:42.088 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:42.088 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:42.088 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:42.088 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:42.088 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:42.088 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:42.088 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:42.088 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:42.088 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:42.089Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.089Z [INFO] agent.router.manager: shutting down +49:42.089 [INFO] TestAgent: consul server down +49:42.089 [INFO] TestAgent: shutdown complete +49:42.089 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23669 network=tcp +49:42.089 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23669 network=udp +49:42.089 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23670 network=tcp +49:42.589 [INFO] TestAgent: Waiting for endpoints to shut down +49:42.589 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{1_1_1_1_1_1_1_1_1_1_1} [WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:39.133Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:39.239Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:39.240 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ed2ea55b-3a87-a959-ddf6-4eeb114b71b6 Address:127.0.0.1:16212}]" -13:39.240 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ed2ea55b-3a87-a959-ddf6-4eeb114b71b6.dc1 127.0.0.1 -13:39.241 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ed2ea55b-3a87-a959-ddf6-4eeb114b71b6 127.0.0.1 -2021-10-31T23:13:39.241Z [INFO] agent.router: Initializing LAN area manager -13:39.242 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16212 [Follower]" leader= -13:39.242 [INFO] TestAgent.server: Adding LAN server: server="Node-ed2ea55b-3a87-a959-ddf6-4eeb114b71b6 (Addr: tcp/127.0.0.1:16212) (DC: dc1)" -13:39.242 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ed2ea55b-3a87-a959-ddf6-4eeb114b71b6.dc1 area=wan -13:39.243 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16466 network=udp -13:39.243 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16466 network=tcp -13:39.244 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16467 network=tcp -13:39.244 [INFO] TestAgent: started state syncer -13:39.244 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16185 network=tcp -13:39.277 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:39.277 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16212 [Candidate]" term=2 -13:39.278 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:39.278 [DEBUG] TestAgent.server.raft: vote granted: from=ed2ea55b-3a87-a959-ddf6-4eeb114b71b6 term=2 tally=1 -13:39.278 [INFO] TestAgent.server.raft: election won: tally=1 -13:39.278 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16212 [Leader]" -13:39.278 [INFO] TestAgent.server: cluster leadership acquired -13:39.278 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16212 -13:39.279 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:39.279 [INFO] TestAgent.server: New leader elected: payload=Node-ed2ea55b-3a87-a959-ddf6-4eeb114b71b6 -13:39.281 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:39.281 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:39.281 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:39.281 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:39.281 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:39.281 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ed2ea55b-3a87-a959-ddf6-4eeb114b71b6 -13:39.281 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ed2ea55b-3a87-a959-ddf6-4eeb114b71b6 -13:39.283 [INFO] TestAgent.server: federation state anti-entropy synced -13:39.351 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:39.352 [INFO] TestAgent: Requesting shutdown -13:39.352 [INFO] TestAgent.server: shutting down server -13:39.352 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:39.352 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:39.352 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:39.352 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:39.352 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:39.352 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:39.352 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:39.352 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:39.352 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:39.352 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:39.352 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:39.352 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:39.352Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:39.352Z [INFO] agent.router.manager: shutting down -13:39.352 [INFO] TestAgent: consul server down -13:39.352 [INFO] TestAgent: shutdown complete -13:39.352 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16466 network=tcp -13:39.352 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16466 network=udp -13:39.352 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16467 network=tcp -13:39.856 [INFO] TestAgent: Waiting for endpoints to shut down -13:39.869 [INFO] TestAgent: Endpoints down -=== RUN TestPreparedQuery_Execute/#07 -2021-10-31T23:13:39.123Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:39.239Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:39.241 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:91c9782c-ee77-ef03-5930-adc66919b81b Address:127.0.0.1:16485}]" -13:39.241 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16485 [Follower]" leader= -13:39.250 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-91c9782c-ee77-ef03-5930-adc66919b81b.dc1 127.0.0.1 -13:39.251 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-91c9782c-ee77-ef03-5930-adc66919b81b 127.0.0.1 -2021-10-31T23:13:39.251Z [INFO] agent.router: Initializing LAN area manager -13:39.251 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16284 network=udp -13:39.251 [INFO] TestAgent.server: Adding LAN server: server="Node-91c9782c-ee77-ef03-5930-adc66919b81b (Addr: tcp/127.0.0.1:16485) (DC: dc1)" -13:39.251 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-91c9782c-ee77-ef03-5930-adc66919b81b.dc1 area=wan -13:39.251 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16284 network=tcp -13:39.283 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16285 network=tcp -13:39.283 [INFO] TestAgent: started state syncer -13:39.283 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16465 network=tcp -13:39.297 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:39.297 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16485 [Candidate]" term=2 -13:39.298 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:39.298 [DEBUG] TestAgent.server.raft: vote granted: from=91c9782c-ee77-ef03-5930-adc66919b81b term=2 tally=1 -13:39.298 [INFO] TestAgent.server.raft: election won: tally=1 -13:39.298 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16485 [Leader]" -13:39.298 [INFO] TestAgent.server: cluster leadership acquired -13:39.298 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16485 -13:39.299 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:39.301 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:39.301 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:39.301 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:39.301 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:39.301 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:39.301 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-91c9782c-ee77-ef03-5930-adc66919b81b -13:39.301 [INFO] TestAgent.server: member joined, marking health alive: member=Node-91c9782c-ee77-ef03-5930-adc66919b81b -13:39.301 [INFO] TestAgent.server: New leader elected: payload=Node-91c9782c-ee77-ef03-5930-adc66919b81b -13:39.322 [INFO] TestAgent.server: federation state anti-entropy synced -13:39.446 [INFO] TestAgent: Requesting shutdown -13:39.446 [INFO] TestAgent.server: shutting down server -13:39.446 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:39.446 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:39.446 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:39.446 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:39.446 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:39.446 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:39.446 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:39.446 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:39.446 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:39.446 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:39.446 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:39.446 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:39.446Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:39.446Z [INFO] agent.router.manager: shutting down -13:39.446 [INFO] TestAgent: consul server down -13:39.446 [INFO] TestAgent: shutdown complete -13:39.446 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16284 network=tcp -13:39.446 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16284 network=udp -13:39.446 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16285 network=tcp -13:39.947 [INFO] TestAgent: Waiting for endpoints to shut down -13:39.947 [INFO] TestAgent: Endpoints down ---- PASS: TestRemoteExecWrites (0.88s) -=== CONT TestPreparedQuery_Update -2021-10-31T23:13:39.175Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:39.175Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:39.268Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:39.268Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:39.277 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5313ae22-1682-08f8-54c8-cb820479eb0a Address:127.0.0.1:16478}]" -13:39.277 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5313ae22-1682-08f8-54c8-cb820479eb0a.dc1 127.0.0.1 -13:39.277 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16478 [Follower]" leader= -13:39.278 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5313ae22-1682-08f8-54c8-cb820479eb0a 127.0.0.1 -2021-10-31T23:13:39.278Z [INFO] agent.router: Initializing LAN area manager -13:39.278 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16186 network=udp -13:39.278 [INFO] TestAgent.server: Adding LAN server: server="Node-5313ae22-1682-08f8-54c8-cb820479eb0a (Addr: tcp/127.0.0.1:16478) (DC: dc1)" -13:39.278 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5313ae22-1682-08f8-54c8-cb820479eb0a.dc1 area=wan -13:39.279 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16186 network=tcp -13:39.279 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16187 network=tcp -13:39.279 [INFO] TestAgent: started state syncer -13:39.279 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16381 network=tcp -13:39.329 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:39.330 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:39.330 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16478 [Candidate]" term=2 -13:39.330 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:39.330 [DEBUG] TestAgent.server.raft: vote granted: from=5313ae22-1682-08f8-54c8-cb820479eb0a term=2 tally=1 -13:39.330 [INFO] TestAgent.server.raft: election won: tally=1 -13:39.330 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16478 [Leader]" -13:39.331 [INFO] TestAgent.server: cluster leadership acquired -13:39.331 [INFO] TestAgent.server: initializing acls -13:39.331 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:39.331 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:39.332 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:39.332 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:39.332 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:39.332 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:39.333 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5313ae22-1682-08f8-54c8-cb820479eb0a -13:39.333 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5313ae22-1682-08f8-54c8-cb820479eb0a.dc1 -13:39.334 [INFO] TestAgent.server: New leader elected: payload=Node-5313ae22-1682-08f8-54c8-cb820479eb0a -13:39.334 [INFO] TestAgent.server: Updating LAN server: server="Node-5313ae22-1682-08f8-54c8-cb820479eb0a (Addr: tcp/127.0.0.1:16478) (DC: dc1)" -13:39.334 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5313ae22-1682-08f8-54c8-cb820479eb0a.dc1 area=wan -13:39.346 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:39.348 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:39.348 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:39.348 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:39.348 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:39.348 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:39.348 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5313ae22-1682-08f8-54c8-cb820479eb0a -13:39.348 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5313ae22-1682-08f8-54c8-cb820479eb0a -13:39.350 [INFO] TestAgent.server: federation state anti-entropy synced -13:39.356 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5313ae22-1682-08f8-54c8-cb820479eb0a -13:39.493 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5313ae22-1682-08f8-54c8-cb820479eb0a -13:39.494 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-5313ae22-1682-08f8-54c8-cb820479eb0a -13:39.495 [INFO] TestAgent: Requesting shutdown -13:39.495 [INFO] TestAgent.server: shutting down server -13:39.495 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:39.495 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:39.495 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:39.495 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:39.495 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:39.495 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:39.495 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:39.495 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:39.512 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:39.512 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:39.512 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:39.512 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:39.512 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:39.512 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:39.512Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:39.512Z [INFO] agent.router.manager: shutting down -13:39.512 [INFO] TestAgent: consul server down -13:39.512 [INFO] TestAgent: shutdown complete -13:39.512 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16186 network=tcp -13:39.512 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16186 network=udp -13:39.512 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16187 network=tcp -13:39.513 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:39.513 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:40.013 [INFO] TestAgent: Waiting for endpoints to shut down -13:40.013 [INFO] TestAgent: Endpoints down ---- PASS: TestACL_Legacy_Update (0.89s) -=== CONT TestSessionCreate_Delete -2021-10-31T23:13:39.084Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:39.084Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:39.231Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:39.231Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:39.233 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a21eeb91-accf-a489-f9aa-8721285eaadd Address:127.0.0.1:16324}]" -13:39.233 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16324 [Follower]" leader= -13:39.242 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a21eeb91-accf-a489-f9aa-8721285eaadd.dc1 127.0.0.1 -13:39.242 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a21eeb91-accf-a489-f9aa-8721285eaadd 127.0.0.1 -2021-10-31T23:13:39.242Z [INFO] agent.router: Initializing LAN area manager -13:39.243 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a21eeb91-accf-a489-f9aa-8721285eaadd.dc1 area=wan -13:39.243 [INFO] TestAgent.server: Adding LAN server: server="Node-a21eeb91-accf-a489-f9aa-8721285eaadd (Addr: tcp/127.0.0.1:16324) (DC: dc1)" -13:39.243 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16424 network=udp -13:39.243 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16424 network=tcp -13:39.243 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16425 network=tcp -13:39.244 [INFO] TestAgent: started state syncer -13:39.244 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16283 network=tcp -13:39.293 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:39.302 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:39.302 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16324 [Candidate]" term=2 -13:39.302 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:39.302 [DEBUG] TestAgent.server.raft: vote granted: from=a21eeb91-accf-a489-f9aa-8721285eaadd term=2 tally=1 -13:39.302 [INFO] TestAgent.server.raft: election won: tally=1 -13:39.302 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16324 [Leader]" -13:39.302 [INFO] TestAgent.server: cluster leadership acquired -13:39.302 [INFO] TestAgent.server: New leader elected: payload=Node-a21eeb91-accf-a489-f9aa-8721285eaadd -13:39.302 [INFO] TestAgent.server: initializing acls -13:39.302 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:39.302 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:39.303 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:39.303 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:39.303 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:39.303 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:39.303 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a21eeb91-accf-a489-f9aa-8721285eaadd -13:39.303 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a21eeb91-accf-a489-f9aa-8721285eaadd.dc1 -13:39.304 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:39.322 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a21eeb91-accf-a489-f9aa-8721285eaadd.dc1 area=wan -13:39.322 [INFO] TestAgent.server: Updating LAN server: server="Node-a21eeb91-accf-a489-f9aa-8721285eaadd (Addr: tcp/127.0.0.1:16324) (DC: dc1)" -13:39.328 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:39.328 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:39.328 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:39.328 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:39.328 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:39.328 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a21eeb91-accf-a489-f9aa-8721285eaadd -13:39.328 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a21eeb91-accf-a489-f9aa-8721285eaadd -13:39.329 [INFO] TestAgent.server: federation state anti-entropy synced -13:39.329 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a21eeb91-accf-a489-f9aa-8721285eaadd -13:39.398 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:39.399 [INFO] TestAgent: Synced node info -13:39.550 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-a21eeb91-accf-a489-f9aa-8721285eaadd -13:39.550 [INFO] TestAgent: Requesting shutdown -13:39.550 [INFO] TestAgent.server: shutting down server -13:39.550 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:39.550 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:39.550 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:39.550 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:39.550 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:39.550 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:39.550 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:39.550 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:39.550 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:39.550 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:39.550 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:39.550 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:39.550 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:39.550 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:39.550 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:39.551Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:39.551Z [INFO] agent.router.manager: shutting down -13:39.551 [INFO] TestAgent: consul server down -13:39.551 [INFO] TestAgent: shutdown complete -13:39.551 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16424 network=tcp -13:39.551 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16424 network=udp -13:39.551 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16425 network=tcp -13:40.053 [INFO] TestAgent: Waiting for endpoints to shut down -13:40.053 [INFO] TestAgent: Endpoints down -=== RUN TestSnapshot_Options/PUT#01 -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:39.441Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:39.487Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:39.489 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d5f1c5a8-4c0b-1d62-5b6f-1933da4748ab Address:127.0.0.1:16457}]" -13:39.489 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d5f1c5a8-4c0b-1d62-5b6f-1933da4748ab.dc1 127.0.0.1 -13:39.489 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16457 [Follower]" leader= -13:39.513 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d5f1c5a8-4c0b-1d62-5b6f-1933da4748ab 127.0.0.1 -2021-10-31T23:13:39.513Z [INFO] agent.router: Initializing LAN area manager -13:39.513 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16480 network=udp -13:39.513 [INFO] TestAgent.server: Adding LAN server: server="Node-d5f1c5a8-4c0b-1d62-5b6f-1933da4748ab (Addr: tcp/127.0.0.1:16457) (DC: dc1)" -13:39.513 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d5f1c5a8-4c0b-1d62-5b6f-1933da4748ab.dc1 area=wan -13:39.515 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16480 network=tcp -13:39.516 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16481 network=tcp -13:39.516 [INFO] TestAgent: started state syncer -13:39.517 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16458 network=tcp -13:39.529 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:39.529 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16457 [Candidate]" term=2 -13:39.530 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:39.530 [DEBUG] TestAgent.server.raft: vote granted: from=d5f1c5a8-4c0b-1d62-5b6f-1933da4748ab term=2 tally=1 -13:39.530 [INFO] TestAgent.server.raft: election won: tally=1 -13:39.530 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16457 [Leader]" -13:39.530 [INFO] TestAgent.server: cluster leadership acquired -13:39.530 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16457 -13:39.531 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:39.532 [INFO] TestAgent.server: New leader elected: payload=Node-d5f1c5a8-4c0b-1d62-5b6f-1933da4748ab -13:39.534 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:39.534 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:39.534 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:39.534 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:39.534 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:39.535 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d5f1c5a8-4c0b-1d62-5b6f-1933da4748ab -13:39.535 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d5f1c5a8-4c0b-1d62-5b6f-1933da4748ab -13:39.536 [INFO] TestAgent.server: federation state anti-entropy synced -13:39.691 [INFO] TestAgent: Requesting shutdown -13:39.691 [INFO] TestAgent.server: shutting down server -13:39.691 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:39.691 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:39.691 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:39.691 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:39.691 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:39.691 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:39.691 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:39.691 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:39.691 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:39.691 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:39.691 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:39.691 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:39.691Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:39.691Z [INFO] agent.router.manager: shutting down -13:39.691 [INFO] TestAgent: consul server down -13:39.691 [INFO] TestAgent: shutdown complete -13:39.691 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16480 network=tcp -13:39.691 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16480 network=udp -13:39.691 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16481 network=tcp -13:40.193 [INFO] TestAgent: Waiting for endpoints to shut down -13:40.193 [INFO] TestAgent: Endpoints down ---- PASS: TestTxnEndpoint_UpdateCheck (0.79s) -=== CONT TestSessionCreate_DefaultCheck -2021-10-31T23:13:39.323Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:39.364Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:39.367 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b72326de-489c-dc5b-c0a6-8c3c0c6323f2 Address:127.0.0.1:16366}]" -13:39.368 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b72326de-489c-dc5b-c0a6-8c3c0c6323f2.dc1 127.0.0.1 -13:39.369 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b72326de-489c-dc5b-c0a6-8c3c0c6323f2 127.0.0.1 -2021-10-31T23:13:39.369Z [INFO] agent.router: Initializing LAN area manager -13:39.369 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16366 [Follower]" leader= -13:39.370 [INFO] TestAgent.server: Adding LAN server: server="Node-b72326de-489c-dc5b-c0a6-8c3c0c6323f2 (Addr: tcp/127.0.0.1:16366) (DC: dc1)" -13:39.370 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b72326de-489c-dc5b-c0a6-8c3c0c6323f2.dc1 area=wan -13:39.370 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16438 network=tcp -13:39.374 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16438 network=udp -13:39.376 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16439 network=tcp -13:39.376 [INFO] TestAgent: started state syncer -13:39.376 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16479 network=tcp -13:39.411 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:39.411 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16366 [Candidate]" term=2 -13:39.411 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:39.411 [DEBUG] TestAgent.server.raft: vote granted: from=b72326de-489c-dc5b-c0a6-8c3c0c6323f2 term=2 tally=1 -13:39.411 [INFO] TestAgent.server.raft: election won: tally=1 -13:39.411 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16366 [Leader]" -13:39.411 [INFO] TestAgent.server: cluster leadership acquired -13:39.434 [INFO] TestAgent.server: New leader elected: payload=Node-b72326de-489c-dc5b-c0a6-8c3c0c6323f2 -13:39.434 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16366 -13:39.435 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:39.436 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:39.436 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:39.436 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:39.436 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:39.436 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:39.436 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b72326de-489c-dc5b-c0a6-8c3c0c6323f2 -13:39.436 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b72326de-489c-dc5b-c0a6-8c3c0c6323f2 -13:39.437 [INFO] TestAgent.server: federation state anti-entropy synced -13:39.577 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:39.578 [INFO] TestAgent: Synced node info -13:39.687 [INFO] TestAgent: Requesting shutdown -13:39.687 [INFO] TestAgent.server: shutting down server -13:39.687 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:39.687 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:39.687 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:39.687 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:39.687 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:39.687 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:39.687 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:39.687 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:39.687 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:39.687 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:39.687 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:39.687Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:39.687Z [INFO] agent.router.manager: shutting down -13:39.688 [INFO] TestAgent: consul server down -13:39.688 [INFO] TestAgent: shutdown complete -13:39.688 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16438 network=tcp -13:39.688 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16438 network=udp -13:39.688 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16439 network=tcp -13:40.189 [INFO] TestAgent: Waiting for endpoints to shut down -13:40.189 [INFO] TestAgent: Endpoints down -=== RUN TestTxnEndpoint_KV_Actions/#01 +2022-12-04T05:49:41.878Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:41.888Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:41.889 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9dccf470-bbb3-0546-bcde-6b44c6ce4f82 Address:127.0.0.1:23716}]" +49:41.890 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23716 [Follower]" leader= +49:41.890 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:41.890 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:41.890Z [INFO] agent.router: Initializing LAN area manager +49:41.890 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:41.890 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23716) (DC: dc1)" +49:41.891 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23794 network=tcp +49:41.891 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23794 network=udp +49:41.891 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23795 network=tcp +49:41.891 [INFO] TestAgent: started state syncer +49:41.891 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23759 network=tcp +49:41.950 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:41.950 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23716 [Candidate]" term=2 +49:41.950 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:41.950 [DEBUG] TestAgent.server.raft: vote granted: from=9dccf470-bbb3-0546-bcde-6b44c6ce4f82 term=2 tally=1 +49:41.950 [INFO] TestAgent.server.raft: election won: tally=1 +49:41.950 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23716 [Leader]" +49:41.950 [INFO] TestAgent.server: cluster leadership acquired +49:41.950 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23716 +49:41.951 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:41.952 [INFO] TestAgent.server: New leader elected: payload=test-node +49:41.952 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:41.952 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:41.952 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:41.952 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:41.952 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:41.952 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:41.952 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:41.952 [INFO] TestAgent.server: federation state anti-entropy synced +49:41.973 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:41.974 [INFO] TestAgent: Synced node info +49:42.336 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.077423ms client=127.0.0.1:58881 client_network=udp +49:42.337 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=826.696µs client=127.0.0.1:43384 client_network=udp +49:42.338 [DEBUG] TestAgent.dns: request served from client: name=37449a1d-7f84-33b1-e251-3cf003d67fdb.query.consul. type=A class=IN latency=693.436µs client=127.0.0.1:44303 client_network=udp +49:42.338 [INFO] TestAgent: Requesting shutdown +49:42.338 [INFO] TestAgent.server: shutting down server +49:42.338 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:42.338 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:42.338 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:42.338 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:42.338 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:42.338 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:42.338 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:42.339 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:42.339 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:42.339 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:42.339 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:42.339Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.339Z [INFO] agent.router.manager: shutting down +49:42.339 [INFO] TestAgent: consul server down +49:42.339 [INFO] TestAgent: shutdown complete +49:42.339 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23794 network=tcp +49:42.339 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23794 network=udp +49:42.339 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23795 network=tcp +49:42.839 [INFO] TestAgent: Waiting for endpoints to shut down +49:42.839 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{0_0_0_0_0_0_0_0_0_0_0} +2022-12-04T05:49:42.131Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:42.143Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:42.145 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1dc48146-daba-febf-3a2f-94693f209c25 Address:127.0.0.1:23771}]" +49:42.145 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23771 [Follower]" leader= +49:42.145 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:42.146 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:42.146Z [INFO] agent.router: Initializing LAN area manager +49:42.146 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:42.146 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23771) (DC: dc1)" +49:42.146 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23473 network=udp +49:42.146 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23473 network=tcp +49:42.146 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23474 network=tcp +49:42.146 [INFO] TestAgent: started state syncer +49:42.146 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23738 network=tcp +49:42.211 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:42.211 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23771 [Candidate]" term=2 +49:42.211 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:42.211 [DEBUG] TestAgent.server.raft: vote granted: from=1dc48146-daba-febf-3a2f-94693f209c25 term=2 tally=1 +49:42.211 [INFO] TestAgent.server.raft: election won: tally=1 +49:42.211 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23771 [Leader]" +49:42.211 [INFO] TestAgent.server: cluster leadership acquired +49:42.211 [INFO] TestAgent.server: New leader elected: payload=test-node +49:42.211 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23771 +49:42.212 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:42.213 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:42.213 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:42.213 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:42.213 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:42.213 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:42.213 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:42.213 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:42.214 [INFO] TestAgent.server: federation state anti-entropy synced +49:42.433 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:42.434 [INFO] TestAgent: Synced node info +49:42.434 [DEBUG] TestAgent: Node info in sync +49:42.489 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.001183ms client=127.0.0.1:40990 client_network=udp +49:42.490 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=685.342µs client=127.0.0.1:45707 client_network=udp +49:42.494 [DEBUG] TestAgent.dns: request served from client: name=9e4eb405-dc2f-2846-5aa9-bf6d5e3f9191.query.consul. type=AAAA class=IN latency=1.01605ms client=127.0.0.1:60379 client_network=udp +49:42.494 [INFO] TestAgent: Requesting shutdown +49:42.494 [INFO] TestAgent.server: shutting down server +49:42.494 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:42.494 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:42.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:42.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:42.494 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:42.494 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:42.494 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:42.494 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:42.494 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:42.494 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:42.494 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +2022-12-04T05:49:42.494Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.494Z [INFO] agent.router.manager: shutting down +49:42.494 [INFO] TestAgent: consul server down +49:42.494 [INFO] TestAgent: shutdown complete +49:42.494 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23473 network=tcp +49:42.494 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23473 network=udp +49:42.495 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23474 network=tcp +49:42.996 [INFO] TestAgent: Waiting for endpoints to shut down +49:42.996 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{0_0_0_0_0_0_0_0_0_0_0} [WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=255)_compressed=false#01 -2021-10-31T23:13:39.686Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:39.686Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:39.720Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:39.720Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:39.759 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:eb4a9b09-93b6-128a-4c1c-b6e44b6b3c27 Address:127.0.0.1:16387}]" -13:39.759 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16387 [Follower]" leader= -13:39.759 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-eb4a9b09-93b6-128a-4c1c-b6e44b6b3c27.dc1 127.0.0.1 -13:39.760 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-eb4a9b09-93b6-128a-4c1c-b6e44b6b3c27 127.0.0.1 -2021-10-31T23:13:39.760Z [INFO] agent.router: Initializing LAN area manager -13:39.761 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16368 network=udp -13:39.761 [INFO] TestAgent.server: Adding LAN server: server="Node-eb4a9b09-93b6-128a-4c1c-b6e44b6b3c27 (Addr: tcp/127.0.0.1:16387) (DC: dc1)" -13:39.761 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-eb4a9b09-93b6-128a-4c1c-b6e44b6b3c27.dc1 area=wan -13:39.761 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16368 network=tcp -13:39.770 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16369 network=tcp -13:39.770 [INFO] TestAgent: started state syncer -13:39.771 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16493 network=tcp -13:39.800 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:39.800 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16387 [Candidate]" term=2 -13:39.800 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:39.800 [DEBUG] TestAgent.server.raft: vote granted: from=eb4a9b09-93b6-128a-4c1c-b6e44b6b3c27 term=2 tally=1 -13:39.800 [INFO] TestAgent.server.raft: election won: tally=1 -13:39.800 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16387 [Leader]" -13:39.800 [INFO] TestAgent.server: cluster leadership acquired -13:39.800 [INFO] TestAgent.server: initializing acls -13:39.801 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:39.801 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:39.801 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:39.801 [INFO] TestAgent.server: New leader elected: payload=Node-eb4a9b09-93b6-128a-4c1c-b6e44b6b3c27 -13:39.802 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:39.802 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:39.802 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:39.802 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-eb4a9b09-93b6-128a-4c1c-b6e44b6b3c27 -13:39.802 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-eb4a9b09-93b6-128a-4c1c-b6e44b6b3c27.dc1 -13:39.803 [INFO] TestAgent.server: Updating LAN server: server="Node-eb4a9b09-93b6-128a-4c1c-b6e44b6b3c27 (Addr: tcp/127.0.0.1:16387) (DC: dc1)" -13:39.803 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-eb4a9b09-93b6-128a-4c1c-b6e44b6b3c27.dc1 area=wan -13:39.804 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:39.811 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:39.811 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:39.811 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:39.811 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:39.811 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:39.812 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eb4a9b09-93b6-128a-4c1c-b6e44b6b3c27 -13:39.812 [INFO] TestAgent.server: member joined, marking health alive: member=Node-eb4a9b09-93b6-128a-4c1c-b6e44b6b3c27 -13:39.813 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eb4a9b09-93b6-128a-4c1c-b6e44b6b3c27 -13:39.830 [INFO] TestAgent.server: federation state anti-entropy synced -13:39.958 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-eb4a9b09-93b6-128a-4c1c-b6e44b6b3c27 -13:39.958 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-eb4a9b09-93b6-128a-4c1c-b6e44b6b3c27 -13:39.959 [INFO] TestAgent: Requesting shutdown -13:39.959 [INFO] TestAgent.server: shutting down server -13:39.959 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:39.959 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:39.959 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:39.959 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:39.959 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:39.959 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:39.959 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:39.959 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:39.959 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:39.959 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:39.959 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:39.959 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:39.959 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:39.959 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:39.959 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:39.959 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:39.959Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:39.959Z [INFO] agent.router.manager: shutting down -13:39.960 [INFO] TestAgent: consul server down -13:39.960 [INFO] TestAgent: shutdown complete -13:39.960 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16368 network=tcp -13:39.960 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16368 network=udp -13:39.960 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16369 network=tcp -13:40.461 [INFO] TestAgent: Waiting for endpoints to shut down -13:40.461 [INFO] TestAgent: Endpoints down ---- PASS: TestRemoteExecGetSpec_ACLToken (0.89s) -=== CONT TestSessionCreate_NodeChecks -2021-10-31T23:13:28.781Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:28.803Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:28.805 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c522b85e-4ec5-2150-3c3a-2a74c0978bbb Address:127.0.0.1:16532}]" -13:28.805 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16532 [Follower]" leader= -13:28.816 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c522b85e-4ec5-2150-3c3a-2a74c0978bbb.dc1 127.0.0.1 -13:28.816 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c522b85e-4ec5-2150-3c3a-2a74c0978bbb 127.0.0.1 -2021-10-31T23:13:28.816Z [INFO] agent.router: Initializing LAN area manager -13:28.817 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16534 network=udp -13:28.817 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c522b85e-4ec5-2150-3c3a-2a74c0978bbb.dc1 area=wan -13:28.817 [INFO] TestAgent.server: Adding LAN server: server="Node-c522b85e-4ec5-2150-3c3a-2a74c0978bbb (Addr: tcp/127.0.0.1:16532) (DC: dc1)" -13:28.817 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16534 network=tcp -13:28.817 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16528 network=tcp -13:28.818 [INFO] TestAgent: started state syncer -13:28.818 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16603 network=tcp -13:28.882 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:28.882 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16532 [Candidate]" term=2 -13:28.883 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:28.883 [DEBUG] TestAgent.server.raft: vote granted: from=c522b85e-4ec5-2150-3c3a-2a74c0978bbb term=2 tally=1 -13:28.883 [INFO] TestAgent.server.raft: election won: tally=1 -13:28.883 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16532 [Leader]" -13:28.883 [INFO] TestAgent.server: cluster leadership acquired -13:28.883 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16532 -13:28.884 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:28.884 [INFO] TestAgent.server: New leader elected: payload=Node-c522b85e-4ec5-2150-3c3a-2a74c0978bbb -13:28.901 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:28.901 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:28.901 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:28.901 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:28.901 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:28.901 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c522b85e-4ec5-2150-3c3a-2a74c0978bbb -13:28.901 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c522b85e-4ec5-2150-3c3a-2a74c0978bbb -13:28.919 [INFO] TestAgent.server: federation state anti-entropy synced -13:29.014 [INFO] TestAgent.server: server starting leave -13:29.014 [INFO] TestAgent.server.serf.wan: serf: EventMemberLeave: Node-c522b85e-4ec5-2150-3c3a-2a74c0978bbb.dc1 127.0.0.1 -13:29.014 [INFO] TestAgent.server: Handled event for server in area: event=member-leave server=Node-c522b85e-4ec5-2150-3c3a-2a74c0978bbb.dc1 area=wan -2021-10-31T23:13:29.014Z [INFO] agent.router.manager: shutting down -13:29.158 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:29.158 [INFO] TestAgent: Synced node info -13:29.610 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:29.610 [DEBUG] TestAgent: Node info in sync -13:29.610 [DEBUG] TestAgent: Node info in sync -13:32.018 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-c522b85e-4ec5-2150-3c3a-2a74c0978bbb 127.0.0.1 -13:32.018 [INFO] TestAgent.server: Removing LAN server: server="Node-c522b85e-4ec5-2150-3c3a-2a74c0978bbb (Addr: tcp/127.0.0.1:16532) (DC: dc1)" -13:32.018 [WARN] TestAgent.server: deregistering self should be done by follower: name=Node-c522b85e-4ec5-2150-3c3a-2a74c0978bbb -13:32.884 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -13:34.886 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -13:35.021 [INFO] TestAgent.server: Waiting to drain RPC traffic: drain_time=5s -13:36.889 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -13:38.886 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -13:38.886 [ERROR] TestAgent.server.autopilot: Error promoting servers: error="error getting server raft protocol versions: No servers found" -13:40.022 [INFO] TestAgent: Requesting shutdown -13:40.022 [INFO] TestAgent.server: shutting down server -13:40.022 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:40.022 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:40.022 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:40.022 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:40.022 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:40.022 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:40.023 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:40.023 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:40.023 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2021-10-31T23:13:40.023Z [INFO] agent.router.manager: shutting down -13:40.023 [INFO] TestAgent: consul server down -13:40.023 [INFO] TestAgent: shutdown complete -13:40.023 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16534 network=tcp -13:40.023 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16534 network=udp -13:40.023 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16528 network=tcp -13:40.525 [INFO] TestAgent: Waiting for endpoints to shut down -13:40.525 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_StartStop (11.84s) -=== CONT TestSessionCreate -2021-10-31T23:13:39.594Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:39.594Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:39.717Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:39.717Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:39.727 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:91e009f3-bd06-7f6f-bce6-484bd168691f Address:127.0.0.1:16408}]" -13:39.728 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-91e009f3-bd06-7f6f-bce6-484bd168691f.dc1 127.0.0.1 -13:39.728 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-91e009f3-bd06-7f6f-bce6-484bd168691f 127.0.0.1 -2021-10-31T23:13:39.728Z [INFO] agent.router: Initializing LAN area manager -13:39.729 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16473 network=udp -13:39.729 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16408 [Follower]" leader= -13:39.729 [INFO] TestAgent.server: Adding LAN server: server="Node-91e009f3-bd06-7f6f-bce6-484bd168691f (Addr: tcp/127.0.0.1:16408) (DC: dc1)" -13:39.729 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-91e009f3-bd06-7f6f-bce6-484bd168691f.dc1 area=wan -13:39.729 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16473 network=tcp -13:39.758 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16474 network=tcp -13:39.758 [INFO] TestAgent: started state syncer -13:39.759 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16367 network=tcp -13:39.786 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:39.802 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:39.802 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16408 [Candidate]" term=2 -13:39.802 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:39.802 [DEBUG] TestAgent.server.raft: vote granted: from=91e009f3-bd06-7f6f-bce6-484bd168691f term=2 tally=1 -13:39.802 [INFO] TestAgent.server.raft: election won: tally=1 -13:39.802 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16408 [Leader]" -13:39.802 [INFO] TestAgent.server: cluster leadership acquired -13:39.803 [INFO] TestAgent.server: initializing acls -13:39.803 [INFO] TestAgent.server: New leader elected: payload=Node-91e009f3-bd06-7f6f-bce6-484bd168691f -13:39.803 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:39.803 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:39.804 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:39.805 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:39.805 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:39.805 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:39.822 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-91e009f3-bd06-7f6f-bce6-484bd168691f -13:39.822 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-91e009f3-bd06-7f6f-bce6-484bd168691f.dc1 -13:39.822 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-91e009f3-bd06-7f6f-bce6-484bd168691f.dc1 area=wan -13:39.823 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:39.825 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:39.825 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:39.825 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:39.825 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:39.825 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:39.825 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-91e009f3-bd06-7f6f-bce6-484bd168691f -13:39.825 [INFO] TestAgent.server: member joined, marking health alive: member=Node-91e009f3-bd06-7f6f-bce6-484bd168691f -13:39.826 [INFO] TestAgent.server: federation state anti-entropy synced -13:39.826 [INFO] TestAgent.server: Updating LAN server: server="Node-91e009f3-bd06-7f6f-bce6-484bd168691f (Addr: tcp/127.0.0.1:16408) (DC: dc1)" -13:39.841 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-91e009f3-bd06-7f6f-bce6-484bd168691f -13:40.050 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-91e009f3-bd06-7f6f-bce6-484bd168691f -13:40.050 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-91e009f3-bd06-7f6f-bce6-484bd168691f -13:40.052 [INFO] TestAgent: Requesting shutdown -13:40.052 [INFO] TestAgent.server: shutting down server -13:40.052 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:40.052 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:40.052 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:40.052 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:40.052 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:40.052 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:40.052 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:40.052 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:40.052 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:40.052 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:40.052 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:40.052 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:40.052 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:40.052 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:40.052 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:40.052 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:40.052Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:40.052Z [INFO] agent.router.manager: shutting down -13:40.052 [INFO] TestAgent: consul server down -13:40.052 [INFO] TestAgent: shutdown complete -13:40.052 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16473 network=tcp -13:40.052 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16473 network=udp -13:40.053 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16474 network=tcp -13:40.553 [INFO] TestAgent: Waiting for endpoints to shut down -13:40.553 [INFO] TestAgent: Endpoints down ---- PASS: TestRemoteExecGetSpec_ACLAgentToken (1.03s) -=== CONT TestDNS_NodeLookup_PeriodName -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:40.023Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:40.087Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:40.102 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2f4e56d5-1260-b45e-ffe1-200d655ccafc Address:127.0.0.1:16604}]" -13:40.103 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16604 [Follower]" leader= -13:40.103 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2f4e56d5-1260-b45e-ffe1-200d655ccafc.dc1 127.0.0.1 -13:40.104 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2f4e56d5-1260-b45e-ffe1-200d655ccafc 127.0.0.1 -2021-10-31T23:13:40.104Z [INFO] agent.router: Initializing LAN area manager -13:40.145 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:40.145 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16604 [Candidate]" term=2 -13:40.145 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=2f4e56d5-1260-b45e-ffe1-200d655ccafc fallback=127.0.0.1:16604 error="Could not find address for server id 2f4e56d5-1260-b45e-ffe1-200d655ccafc" -13:40.145 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:40.145 [DEBUG] TestAgent.server.raft: vote granted: from=2f4e56d5-1260-b45e-ffe1-200d655ccafc term=2 tally=1 -13:40.145 [INFO] TestAgent.server.raft: election won: tally=1 -13:40.145 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16604 [Leader]" -13:40.158 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2f4e56d5-1260-b45e-ffe1-200d655ccafc.dc1 area=wan -13:40.158 [INFO] TestAgent.server: Adding LAN server: server="Node-2f4e56d5-1260-b45e-ffe1-200d655ccafc (Addr: tcp/127.0.0.1:16604) (DC: dc1)" -13:40.158 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16543 network=udp -13:40.158 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16543 network=tcp -13:40.158 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16544 network=tcp -13:40.158 [INFO] TestAgent: started state syncer -13:40.159 [INFO] TestAgent: Requesting shutdown -13:40.159 [INFO] TestAgent.server: shutting down server -13:40.159 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:40.159 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16584 network=tcp -13:40.159 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -13:40.159 [INFO] TestAgent: Synced node info -13:40.159 [DEBUG] TestAgent: Node info in sync -13:40.166 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:40.166Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:40.166Z [INFO] agent.router.manager: shutting down -13:40.166 [INFO] TestAgent: consul server down -13:40.166 [INFO] TestAgent: shutdown complete -13:40.166 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16543 network=tcp -13:40.166 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16543 network=udp -13:40.166 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16544 network=tcp -13:40.166 [INFO] TestAgent.server: cluster leadership acquired -13:40.166 [DEBUG] TestAgent.server: shutting down leader loop -13:40.166 [ERROR] TestAgent.server: failed to wait for barrier: error="raft is already shutdown" -13:40.166 [INFO] TestAgent.server: cluster leadership lost -13:40.667 [INFO] TestAgent: Waiting for endpoints to shut down -13:40.667 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_Execute (6.65s) - --- PASS: TestPreparedQuery_Execute/#00 (1.06s) - --- PASS: TestPreparedQuery_Execute/#01 (0.74s) - --- PASS: TestPreparedQuery_Execute/#02 (1.02s) - --- PASS: TestPreparedQuery_Execute/#03 (0.81s) - --- PASS: TestPreparedQuery_Execute/#04 (0.71s) - --- PASS: TestPreparedQuery_Execute/#05 (0.71s) - --- PASS: TestPreparedQuery_Execute/#06 (0.85s) - --- PASS: TestPreparedQuery_Execute/#07 (0.74s) -=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{0_0_0_0_0_0_0_0_0_0_0} -2021-10-31T23:13:39.657Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:39.721Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:39.723 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f57cc999-4053-ff61-1dd7-45ad93a5b36b Address:127.0.0.1:16492}]" -13:39.724 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f57cc999-4053-ff61-1dd7-45ad93a5b36b.dc1 127.0.0.1 -13:39.724 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f57cc999-4053-ff61-1dd7-45ad93a5b36b 127.0.0.1 -2021-10-31T23:13:39.724Z [INFO] agent.router: Initializing LAN area manager -13:39.724 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16459 network=udp -13:39.724 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16492 [Follower]" leader= -13:39.725 [INFO] TestAgent.server: Adding LAN server: server="Node-f57cc999-4053-ff61-1dd7-45ad93a5b36b (Addr: tcp/127.0.0.1:16492) (DC: dc1)" -13:39.725 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f57cc999-4053-ff61-1dd7-45ad93a5b36b.dc1 area=wan -13:39.725 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16459 network=tcp -13:39.725 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16460 network=tcp -13:39.758 [INFO] TestAgent: started state syncer -13:39.758 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16472 network=tcp -13:39.782 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:39.782 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16492 [Candidate]" term=2 -13:39.782 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:39.782 [DEBUG] TestAgent.server.raft: vote granted: from=f57cc999-4053-ff61-1dd7-45ad93a5b36b term=2 tally=1 -13:39.782 [INFO] TestAgent.server.raft: election won: tally=1 -13:39.782 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16492 [Leader]" -13:39.782 [INFO] TestAgent.server: cluster leadership acquired -13:39.783 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16492 -13:39.784 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:39.785 [INFO] TestAgent.server: New leader elected: payload=Node-f57cc999-4053-ff61-1dd7-45ad93a5b36b -13:39.801 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:39.801 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:39.801 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:39.801 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:39.801 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:39.801 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f57cc999-4053-ff61-1dd7-45ad93a5b36b -13:39.801 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f57cc999-4053-ff61-1dd7-45ad93a5b36b -13:39.802 [INFO] TestAgent.server: federation state anti-entropy synced -13:39.882 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:39.882 [INFO] TestAgent: Synced node info -13:39.882 [DEBUG] TestAgent: Node info in sync -13:40.169 [INFO] TestAgent: Requesting shutdown -13:40.169 [INFO] TestAgent.server: shutting down server -13:40.169 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:40.169 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:40.169 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:40.169 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:40.169 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:40.170 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:40.170 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:40.170 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:40.170 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:40.170 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:40.170 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:40.170Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:40.170Z [INFO] agent.router.manager: shutting down -13:40.170 [INFO] TestAgent: consul server down -13:40.170 [INFO] TestAgent: shutdown complete -13:40.170 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16459 network=tcp -13:40.170 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16459 network=udp -13:40.170 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16460 network=tcp -13:40.674 [INFO] TestAgent: Waiting for endpoints to shut down -13:40.674 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_Integration (1.16s) -=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{6_6_6_6_6_6_6_5_6_6_-5} -2021-10-31T23:13:39.238Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:39.268Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:39.271 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:41d2f4d6-25ad-f7fa-227d-a8b789bb5ad1 Address:127.0.0.1:16422}]" -13:39.271 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16422 [Follower]" leader= -13:39.271 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-41d2f4d6-25ad-f7fa-227d-a8b789bb5ad1.dc1 127.0.0.1 -13:39.272 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-41d2f4d6-25ad-f7fa-227d-a8b789bb5ad1 127.0.0.1 -2021-10-31T23:13:39.272Z [INFO] agent.router: Initializing LAN area manager -13:39.272 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16382 network=udp -13:39.272 [INFO] TestAgent.server: Adding LAN server: server="Node-41d2f4d6-25ad-f7fa-227d-a8b789bb5ad1 (Addr: tcp/127.0.0.1:16422) (DC: dc1)" -13:39.272 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-41d2f4d6-25ad-f7fa-227d-a8b789bb5ad1.dc1 area=wan -13:39.272 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16382 network=tcp -13:39.273 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16383 network=tcp -13:39.273 [INFO] TestAgent: started state syncer -13:39.273 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16437 network=tcp -13:39.326 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:39.326 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16422 [Candidate]" term=2 -13:39.326 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:39.326 [DEBUG] TestAgent.server.raft: vote granted: from=41d2f4d6-25ad-f7fa-227d-a8b789bb5ad1 term=2 tally=1 -13:39.326 [INFO] TestAgent.server.raft: election won: tally=1 -13:39.326 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16422 [Leader]" -13:39.327 [INFO] TestAgent.server: cluster leadership acquired -13:39.327 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16422 -13:39.328 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:39.329 [INFO] TestAgent.server: New leader elected: payload=Node-41d2f4d6-25ad-f7fa-227d-a8b789bb5ad1 -13:39.343 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:39.343 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:39.343 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:39.343 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:39.343 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:39.343 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-41d2f4d6-25ad-f7fa-227d-a8b789bb5ad1 -13:39.343 [INFO] TestAgent.server: member joined, marking health alive: member=Node-41d2f4d6-25ad-f7fa-227d-a8b789bb5ad1 -13:39.344 [INFO] TestAgent.server: federation state anti-entropy synced -13:39.376 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:39.377 [INFO] TestAgent: Synced node info -13:39.377 [DEBUG] TestAgent: Node info in sync -13:39.922 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:39.922 [DEBUG] TestAgent: Node info in sync -13:40.178 [WARN] TestAgent.server.rpc: Attempting to apply large raft entry: size_in_bytes=1573029 -13:40.200 [INFO] TestAgent: Requesting shutdown -13:40.200 [INFO] TestAgent.server: shutting down server -13:40.200 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:40.200 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:40.200 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:40.200 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:40.200 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:40.200 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:40.200 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:40.200 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:40.200 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:40.200 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:40.200 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:40.201Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:40.201Z [INFO] agent.router.manager: shutting down -13:40.201 [INFO] TestAgent: consul server down -13:40.201 [INFO] TestAgent: shutdown complete -13:40.201 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16382 network=tcp -13:40.201 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16382 network=udp -13:40.201 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16383 network=tcp -13:40.705 [INFO] TestAgent: Waiting for endpoints to shut down -13:40.705 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:39.926Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:40.009Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:40.020 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:25ab33df-2bd8-8dfb-07ee-db42cf21111f Address:127.0.0.1:16513}]" -13:40.021 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-25ab33df-2bd8-8dfb-07ee-db42cf21111f.dc1 127.0.0.1 -13:40.021 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-25ab33df-2bd8-8dfb-07ee-db42cf21111f 127.0.0.1 -2021-10-31T23:13:40.021Z [INFO] agent.router: Initializing LAN area manager -13:40.022 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16513 [Follower]" leader= -13:40.034 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16515 network=udp -13:40.034 [INFO] TestAgent.server: Adding LAN server: server="Node-25ab33df-2bd8-8dfb-07ee-db42cf21111f (Addr: tcp/127.0.0.1:16513) (DC: dc1)" -13:40.034 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-25ab33df-2bd8-8dfb-07ee-db42cf21111f.dc1 area=wan -13:40.034 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16515 network=tcp -13:40.035 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16516 network=tcp -13:40.035 [INFO] TestAgent: started state syncer -13:40.035 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16542 network=tcp -13:40.077 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:40.077 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16513 [Candidate]" term=2 -13:40.086 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:40.086 [DEBUG] TestAgent.server.raft: vote granted: from=25ab33df-2bd8-8dfb-07ee-db42cf21111f term=2 tally=1 -13:40.086 [INFO] TestAgent.server.raft: election won: tally=1 -13:40.086 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16513 [Leader]" -13:40.086 [INFO] TestAgent.server: cluster leadership acquired -13:40.086 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16513 -13:40.087 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:40.089 [INFO] TestAgent.server: New leader elected: payload=Node-25ab33df-2bd8-8dfb-07ee-db42cf21111f -13:40.089 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:40.089 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:40.089 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:40.089 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:40.089 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:40.090 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-25ab33df-2bd8-8dfb-07ee-db42cf21111f -13:40.090 [INFO] TestAgent.server: member joined, marking health alive: member=Node-25ab33df-2bd8-8dfb-07ee-db42cf21111f -13:40.091 [INFO] TestAgent.server: federation state anti-entropy synced -13:40.200 [INFO] TestAgent: Requesting shutdown -13:40.201 [INFO] TestAgent.server: shutting down server -13:40.201 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:40.201 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:40.201 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:40.201 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:40.201 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:40.201 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:40.201 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:40.201 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:40.201 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:40.201 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:40.201 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:40.201 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:40.201Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:40.201Z [INFO] agent.router.manager: shutting down -13:40.201 [INFO] TestAgent: consul server down -13:40.201 [INFO] TestAgent: shutdown complete -13:40.201 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16515 network=tcp -13:40.201 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16515 network=udp -13:40.201 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16516 network=tcp -13:40.705 [INFO] TestAgent: Waiting for endpoints to shut down -13:40.705 [INFO] TestAgent: Endpoints down ---- PASS: TestTxnEndpoint_Bad_Size_Ops (0.92s) +2022-12-04T05:49:34.957Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.969Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.970 [INFO] dc2.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:01c73dc0-0d73-207e-9af5-da93cf1e2158 Address:127.0.0.1:24070}]" +49:34.970 [INFO] dc2.server.raft: entering follower state: follower="Node at 127.0.0.1:24070 [Follower]" leader= +49:34.971 [INFO] dc2.server.serf.wan: serf: EventMemberJoin: Node-01c73dc0-0d73-207e-9af5-da93cf1e2158.dc2 127.0.0.1 +49:34.971 [INFO] dc2.server.serf.lan: serf: EventMemberJoin: Node-01c73dc0-0d73-207e-9af5-da93cf1e2158 127.0.0.1 +2022-12-04T05:49:34.971Z [INFO] agent.router: Initializing LAN area manager +49:34.971 [INFO] dc2: Started DNS server: address=127.0.0.1:24083 network=udp +49:34.971 [INFO] dc2.server: Handled event for server in area: event=member-join server=Node-01c73dc0-0d73-207e-9af5-da93cf1e2158.dc2 area=wan +49:34.972 [INFO] dc2.server: Adding LAN server: server="Node-01c73dc0-0d73-207e-9af5-da93cf1e2158 (Addr: tcp/127.0.0.1:24070) (DC: dc2)" +49:34.972 [INFO] dc2: Started DNS server: address=127.0.0.1:24083 network=tcp +49:34.972 [INFO] dc2: Started HTTP server: address=127.0.0.1:24066 network=tcp +49:34.972 [INFO] dc2: started state syncer +49:34.972 [INFO] dc2: Started gRPC server: address=127.0.0.1:24057 network=tcp +49:35.015 [WARN] dc2.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.015 [INFO] dc2.server.raft: entering candidate state: node="Node at 127.0.0.1:24070 [Candidate]" term=2 +49:35.015 [DEBUG] dc2.server.raft: votes: needed=1 +49:35.015 [DEBUG] dc2.server.raft: vote granted: from=01c73dc0-0d73-207e-9af5-da93cf1e2158 term=2 tally=1 +49:35.015 [INFO] dc2.server.raft: election won: tally=1 +49:35.015 [INFO] dc2.server.raft: entering leader state: leader="Node at 127.0.0.1:24070 [Leader]" +49:35.016 [INFO] dc2.server: cluster leadership acquired +49:35.016 [INFO] dc2.server: New leader elected: payload=Node-01c73dc0-0d73-207e-9af5-da93cf1e2158 +49:35.016 [DEBUG] dc2.server: Cannot upgrade to new ACLs, servers in acl datacenter are not yet upgraded: ACLDatacenter=dc1 mode=1 found=false +49:35.016 [WARN] dc2.server.connect: primary datacenter is configured but unreachable - deferring initialization of the secondary datacenter CA +49:35.016 [INFO] dc2.leader: started routine: routine="config entry replication" +49:35.016 [INFO] dc2.leader: started routine: routine="federation state replication" +49:35.016 [INFO] dc2.leader: started routine: routine="federation state anti-entropy" +49:35.016 [INFO] dc2.leader: started routine: routine="secondary CA roots watch" +49:35.016 [INFO] dc2.leader: started routine: routine="intention replication" +49:35.016 [INFO] dc2.leader: started routine: routine="intermediate cert renew watch" +49:35.016 [INFO] dc2.leader: started routine: routine="CA root pruning" +49:35.016 [DEBUG] dc2.server: federation states are not enabled in the primary dc +49:35.016 [WARN] dc2.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConfigEntry.ListAll +49:35.016 [DEBUG] dc2.server.connect: starting Connect intention replication from primary datacenter: primary=dc1 +49:35.016 [DEBUG] dc2.server.connect: starting Connect CA root replication from primary datacenter: primary=dc1 +49:35.017 [WARN] dc2.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=ConnectCA.Roots +49:35.017 [ERROR] dc2.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: No path to datacenter" +49:35.016 [DEBUG] dc2.server: federation states are not enabled in the primary dc +49:35.017 [DEBUG] dc2.server: federation states are not enabled in the primary dc +49:35.017 [DEBUG] dc2.server: federation states are not enabled in the primary dc +49:35.017 [DEBUG] dc2.server: federation states are not enabled in the primary dc +49:35.016 [DEBUG] dc2.server: federation states are not enabled in the primary dc +49:35.016 [DEBUG] dc2.server: Skipping self join check for node since the cluster is too small: node=Node-01c73dc0-0d73-207e-9af5-da93cf1e2158 +49:35.017 [INFO] dc2.server: member joined, marking health alive: member=Node-01c73dc0-0d73-207e-9af5-da93cf1e2158 +49:35.016 [WARN] dc2.server.rpc: RPC request for DC is currently failing as no path was found: datacenter=dc1 method=Intention.List +49:35.017 [ERROR] dc2.server.connect: error replicating intentions: routine="intention replication" error="No path to datacenter" +49:35.152 [INFO] dc2: (WAN) joining: wan_addresses=[127.0.0.1:23957] +49:35.165 [DEBUG] dc2.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:23957 +49:35.167 [INFO] dc2.server.serf.wan: serf: EventMemberJoin: Node-2bdfedc6-3b23-0d8f-e63d-2e0436f16273.dc1 127.0.0.1 +49:35.167 [INFO] dc2: (WAN) joined: number_of_nodes=1 +49:35.168 [INFO] dc2.server: Handled event for server in area: event=member-join server=Node-2bdfedc6-3b23-0d8f-e63d-2e0436f16273.dc1 area=wan +49:35.221 [DEBUG] dc2: Skipping remote check since it is managed automatically: check=serfHealth +49:35.222 [INFO] dc2: Synced node info +49:35.664 [DEBUG] dc2.server.serf.wan: serf: messageJoinType: Node-01c73dc0-0d73-207e-9af5-da93cf1e2158.dc2 +49:36.098 [DEBUG] dc2.server: finished fetching remote objects: amount=1 +49:36.098 [DEBUG] dc2.server: diffing replication state: local_amount=0 remote_amount=1 +49:36.098 [DEBUG] dc2.server: diffed replication state: deletions=0 updates=1 +49:36.098 [DEBUG] dc2.server: performing updates: updates=1 +49:36.099 [DEBUG] dc2.server: finished updates +49:36.099 [DEBUG] dc2.replication.federation_state: replication completed through remote index: index=11 +49:36.099 [DEBUG] dc2.server.replication.config_entry: finished fetching config entries: amount=0 +49:36.099 [DEBUG] dc2.server.replication.config_entry: Config Entry replication: local=0 remote=0 +49:36.099 [DEBUG] dc2.server.replication.config_entry: Config Entry replication: deletions=0 updates=0 +49:36.099 [DEBUG] dc2.server.replication.config_entry: replication completed through remote index: index=1 +49:36.169 [DEBUG] dc2.server.serf.wan: serf: messageJoinType: Node-01c73dc0-0d73-207e-9af5-da93cf1e2158.dc2 +49:36.665 [DEBUG] dc2.server.serf.wan: serf: messageJoinType: Node-01c73dc0-0d73-207e-9af5-da93cf1e2158.dc2 +49:37.022 [DEBUG] connect.ca.consul: consul CA provider configured: id=ad:4a:c6:ab:ef:63:c9:60:1a:51:7f:19:62:e3:e9:d9:0e:76:55:10:6e:74:24:69:28:a1:6c:b8:b9:8f:fd:89 is_primary=false +49:37.027 [INFO] dc2.server.connect: received new intermediate certificate from primary datacenter +49:37.028 [INFO] dc2.server.connect: updated root certificates from primary datacenter +49:37.039 [DEBUG] dc2: Node info in sync +49:37.041 [INFO] dc2: Synced service: service=foo +49:37.041 [DEBUG] dc2: Check in sync: check=service:foo +49:37.164 [DEBUG] dc2.server.serf.wan: serf: messageJoinType: Node-01c73dc0-0d73-207e-9af5-da93cf1e2158.dc2 +49:37.736 [DEBUG] dc2: Skipping remote check since it is managed automatically: check=serfHealth +49:37.736 [DEBUG] dc2: Node info in sync +49:37.736 [DEBUG] dc2: Service in sync: service=foo +49:37.736 [DEBUG] dc2: Check in sync: check=service:foo +49:37.736 [DEBUG] dc2: Node info in sync +49:37.736 [DEBUG] dc2: Service in sync: service=foo +49:37.736 [DEBUG] dc2: Check in sync: check=service:foo +49:40.025 [DEBUG] dc2.server: finished fetching remote objects: amount=2 +49:40.025 [DEBUG] dc2.server: diffing replication state: local_amount=1 remote_amount=2 +49:40.025 [DEBUG] dc2.server: diffed replication state: deletions=0 updates=1 +49:40.025 [DEBUG] dc2.server: performing updates: updates=1 +49:40.025 [INFO] dc2.server: federation state anti-entropy synced +49:40.025 [DEBUG] dc2.server: finished updates +49:40.025 [DEBUG] dc2.replication.federation_state: replication completed through remote index: index=19 +49:42.053 [INFO] dc2.server.connect: received new intermediate certificate from primary datacenter +49:42.054 [INFO] dc2.server.connect: updated root certificates from primary datacenter +49:42.077 [INFO] dc2: Requesting shutdown +49:42.077 [INFO] dc2.server: shutting down server +49:42.077 [DEBUG] dc2.leader: stopping routine: routine="intermediate cert renew watch" +49:42.077 [DEBUG] dc2.leader: stopping routine: routine="CA root pruning" +49:42.077 [DEBUG] dc2.leader: stopping routine: routine="config entry replication" +49:42.077 [DEBUG] dc2.leader: stopping routine: routine="federation state replication" +49:42.077 [DEBUG] dc2.leader: stopping routine: routine="federation state anti-entropy" +49:42.077 [DEBUG] dc2.leader: stopping routine: routine="secondary CA roots watch" +49:42.077 [DEBUG] dc2.leader: stopping routine: routine="intention replication" +49:42.077 [WARN] dc2.server.serf.lan: serf: Shutdown without a Leave +49:42.077 [DEBUG] dc2.leader: stopped routine: routine="intermediate cert renew watch" +49:42.077 [DEBUG] dc2.leader: stopped routine: routine="CA root pruning" +49:42.077 [DEBUG] dc2.leader: stopped routine: routine="federation state anti-entropy" +49:42.078 [WARN] dc2.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:42.078Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.078Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.078Z [INFO] agent.router.manager: shutting down +49:42.078 [INFO] dc2: consul server down +49:42.078 [INFO] dc2: shutdown complete +49:42.078 [INFO] dc2: Stopping server: protocol=DNS address=127.0.0.1:24083 network=tcp +49:42.078 [INFO] dc2: Stopping server: protocol=DNS address=127.0.0.1:24083 network=udp +49:42.078 [INFO] dc2: Stopping server: protocol=HTTP address=127.0.0.1:24066 network=tcp +49:42.078 [ERROR] dc2.server.rpc: RPC failed to server in DC: server=127.0.0.1:23958 datacenter=dc1 method=ConfigEntry.ListAll error="rpc error making call: EOF" +49:42.078 [INFO] dc2.server.replication.config_entry: stopped replication +49:42.078 [DEBUG] dc2.leader: stopped routine: routine="config entry replication" +49:42.078 [ERROR] dc2.server.rpc: RPC failed to server in DC: server=127.0.0.1:23958 datacenter=dc1 method=Intention.List error="rpc error making call: EOF" +49:42.078 [ERROR] dc2.server.connect: error replicating intentions: routine="intention replication" error="rpc error making call: EOF" +49:42.078 [DEBUG] dc2.leader: stopped routine: routine="intention replication" +49:42.078 [ERROR] dc2.server.rpc: RPC failed to server in DC: server=127.0.0.1:23958 datacenter=dc1 method=ConnectCA.Roots error="rpc error making call: EOF" +49:42.078 [ERROR] dc2.server.connect: CA root replication failed, will retry: routine="secondary CA roots watch" error="Error retrieving the primary datacenter's roots: rpc error making call: EOF" +49:42.078 [DEBUG] dc2.leader: stopped routine: routine="secondary CA roots watch" +49:42.078 [ERROR] dc2.server.rpc: RPC failed to server in DC: server=127.0.0.1:23958 datacenter=dc1 method=FederationState.List error="rpc error making call: EOF" +49:42.078 [WARN] dc2.replication.federation_state: replication error (will retry if still leader): error="failed to retrieve federation states: rpc error making call: EOF" +49:42.078 [INFO] dc2.replication.federation_state: stopped replication +49:42.078 [DEBUG] dc2.leader: stopped routine: routine="federation state replication" +49:42.580 [INFO] dc2: Waiting for endpoints to shut down +49:42.580 [INFO] dc2: Endpoints down +2022-12-04T05:49:34.647Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.661Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.663 [INFO] dc1.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2bdfedc6-3b23-0d8f-e63d-2e0436f16273 Address:127.0.0.1:23958}]" +49:34.663 [INFO] dc1.server.raft: entering follower state: follower="Node at 127.0.0.1:23958 [Follower]" leader= +49:34.663 [INFO] dc1.server.serf.wan: serf: EventMemberJoin: Node-2bdfedc6-3b23-0d8f-e63d-2e0436f16273.dc1 127.0.0.1 +49:34.664 [INFO] dc1.server.serf.lan: serf: EventMemberJoin: Node-2bdfedc6-3b23-0d8f-e63d-2e0436f16273 127.0.0.1 +2022-12-04T05:49:34.664Z [INFO] agent.router: Initializing LAN area manager +49:34.664 [INFO] dc1.server: Adding LAN server: server="Node-2bdfedc6-3b23-0d8f-e63d-2e0436f16273 (Addr: tcp/127.0.0.1:23958) (DC: dc1)" +49:34.664 [INFO] dc1.server: Handled event for server in area: event=member-join server=Node-2bdfedc6-3b23-0d8f-e63d-2e0436f16273.dc1 area=wan +49:34.664 [INFO] dc1: Started DNS server: address=127.0.0.1:24030 network=udp +49:34.664 [INFO] dc1: Started DNS server: address=127.0.0.1:24030 network=tcp +49:34.665 [INFO] dc1: Started HTTP server: address=127.0.0.1:23954 network=tcp +49:34.665 [INFO] dc1: started state syncer +49:34.665 [INFO] dc1: Started gRPC server: address=127.0.0.1:24050 network=tcp +49:34.704 [WARN] dc1.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.704 [INFO] dc1.server.raft: entering candidate state: node="Node at 127.0.0.1:23958 [Candidate]" term=2 +49:34.704 [DEBUG] dc1.server.raft: votes: needed=1 +49:34.704 [DEBUG] dc1.server.raft: vote granted: from=2bdfedc6-3b23-0d8f-e63d-2e0436f16273 term=2 tally=1 +49:34.704 [INFO] dc1.server.raft: election won: tally=1 +49:34.704 [INFO] dc1.server.raft: entering leader state: leader="Node at 127.0.0.1:23958 [Leader]" +49:34.704 [INFO] dc1.server: cluster leadership acquired +49:34.704 [INFO] dc1.server: New leader elected: payload=Node-2bdfedc6-3b23-0d8f-e63d-2e0436f16273 +49:34.704 [DEBUG] dc1.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23958 +49:34.705 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.706 [INFO] dc1.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.706 [INFO] dc1.leader: started routine: routine="federation state anti-entropy" +49:34.706 [INFO] dc1.leader: started routine: routine="federation state pruning" +49:34.706 [INFO] dc1.leader: started routine: routine="intermediate cert renew watch" +49:34.706 [INFO] dc1.leader: started routine: routine="CA root pruning" +49:34.706 [DEBUG] dc1.server: Skipping self join check for node since the cluster is too small: node=Node-2bdfedc6-3b23-0d8f-e63d-2e0436f16273 +49:34.707 [INFO] dc1.server: member joined, marking health alive: member=Node-2bdfedc6-3b23-0d8f-e63d-2e0436f16273 +49:34.707 [INFO] dc1.server: federation state anti-entropy synced +49:34.962 [DEBUG] dc1: Skipping remote check since it is managed automatically: check=serfHealth +49:34.962 [INFO] dc1: Synced node info +49:35.165 [DEBUG] dc1.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:49146 +49:35.166 [INFO] dc1.server.serf.wan: serf: EventMemberJoin: Node-01c73dc0-0d73-207e-9af5-da93cf1e2158.dc2 127.0.0.1 +49:35.166 [INFO] dc1.server: Handled event for server in area: event=member-join server=Node-01c73dc0-0d73-207e-9af5-da93cf1e2158.dc2 area=wan +49:35.169 [DEBUG] connect.ca.consul: consul CA provider configured: id=ae:37:b1:f1:20:40:bb:2f:2b:49:52:47:3b:fd:fd:e4:3e:ce:b8:47:50:e2:5a:30:66:bd:9f:06:08:b3:8e:6b is_primary=true +49:35.172 [INFO] dc1.server.connect: CA rotated to new root under provider: provider=consul +49:35.473 [DEBUG] dc1.server.serf.wan: serf: messageJoinType: Node-01c73dc0-0d73-207e-9af5-da93cf1e2158.dc2 +49:35.972 [DEBUG] dc1.server.serf.wan: serf: messageJoinType: Node-01c73dc0-0d73-207e-9af5-da93cf1e2158.dc2 +49:36.471 [DEBUG] dc1.server.serf.wan: serf: messageJoinType: Node-01c73dc0-0d73-207e-9af5-da93cf1e2158.dc2 +49:36.974 [DEBUG] dc1.server.serf.wan: serf: messageJoinType: Node-01c73dc0-0d73-207e-9af5-da93cf1e2158.dc2 +49:37.017 [DEBUG] dc1: Skipping remote check since it is managed automatically: check=serfHealth +49:37.017 [DEBUG] dc1: Node info in sync +49:37.017 [DEBUG] dc1: Node info in sync +49:42.049 [DEBUG] connect.ca.consul: consul CA provider configured: id=4b:e2:46:c8:61:40:06:59:e5:5a:67:4f:8d:b6:3f:07:2d:b6:b9:d3:1e:df:c8:69:cd:b4:78:91:e8:31:f7:1c is_primary=true +49:42.050 [INFO] dc1.server.connect: CA rotated to new root under provider: provider=consul +49:42.580 [INFO] dc1: Requesting shutdown +49:42.580 [INFO] dc1.server: shutting down server +49:42.580 [DEBUG] dc1.leader: stopping routine: routine="federation state anti-entropy" +49:42.580 [DEBUG] dc1.leader: stopping routine: routine="federation state pruning" +49:42.580 [DEBUG] dc1.leader: stopping routine: routine="intermediate cert renew watch" +49:42.580 [DEBUG] dc1.leader: stopping routine: routine="CA root pruning" +49:42.580 [WARN] dc1.server.serf.lan: serf: Shutdown without a Leave +49:42.580 [DEBUG] dc1.leader: stopped routine: routine="intermediate cert renew watch" +49:42.580 [DEBUG] dc1.leader: stopped routine: routine="CA root pruning" +49:42.580 [WARN] dc1.server.serf.wan: serf: Shutdown without a Leave +49:42.580 [ERROR] dc1.server: error performing anti-entropy sync of federation state: error="context canceled" +49:42.580 [DEBUG] dc1.leader: stopped routine: routine="federation state anti-entropy" +49:42.580 [DEBUG] dc1.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:49:42.580Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.580Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.580Z [INFO] agent.router.manager: shutting down +49:42.580 [INFO] dc1: consul server down +49:42.580 [INFO] dc1: shutdown complete +49:42.580 [INFO] dc1: Stopping server: protocol=DNS address=127.0.0.1:24030 network=tcp +49:42.580 [INFO] dc1: Stopping server: protocol=DNS address=127.0.0.1:24030 network=udp +49:42.580 [INFO] dc1: Stopping server: protocol=HTTP address=127.0.0.1:23954 network=tcp +49:43.081 [INFO] dc1: Waiting for endpoints to shut down +49:43.081 [INFO] dc1: Endpoints down +--- PASS: TestAgentConnectCALeafCert_secondaryDC_good (8.45s) +=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{9_9_8_8_6_8_8_5_8_8_-5} +2022-12-04T05:49:42.272Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:42.292Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:42.294 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7d87ebf4-4b11-27f2-f32d-d0b9a665b878 Address:127.0.0.1:23778}]" +49:42.296 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:42.296 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23778 [Follower]" leader= +49:42.296 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:42.296Z [INFO] agent.router: Initializing LAN area manager +49:42.296 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:42.296 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23778) (DC: dc1)" +49:42.297 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23739 network=udp +49:42.297 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23739 network=tcp +49:42.304 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23740 network=tcp +49:42.304 [INFO] TestAgent: started state syncer +49:42.304 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23779 network=tcp +49:42.356 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:42.356 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23778 [Candidate]" term=2 +49:42.356 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:42.356 [DEBUG] TestAgent.server.raft: vote granted: from=7d87ebf4-4b11-27f2-f32d-d0b9a665b878 term=2 tally=1 +49:42.356 [INFO] TestAgent.server.raft: election won: tally=1 +49:42.356 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23778 [Leader]" +49:42.356 [INFO] TestAgent.server: cluster leadership acquired +49:42.356 [INFO] TestAgent.server: New leader elected: payload=test-node +49:42.356 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23778 +49:42.357 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:42.358 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:42.358 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:42.358 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:42.358 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:42.358 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:42.358 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:42.358 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:42.359 [INFO] TestAgent.server: federation state anti-entropy synced +49:42.577 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=911.554µs client=127.0.0.1:51230 client_network=udp +49:42.581 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.151075ms client=127.0.0.1:45139 client_network=udp +49:42.582 [DEBUG] TestAgent.dns: request served from client: name=1d099dfe-0b8e-2f10-a822-6c6da2647d1b.query.consul. type=A class=IN latency=778.765µs client=127.0.0.1:48907 client_network=udp +49:42.582 [INFO] TestAgent: Requesting shutdown +49:42.582 [INFO] TestAgent.server: shutting down server +49:42.582 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:42.582 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:42.582 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:42.582 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:42.582 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:42.582 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:42.582 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:42.582 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:42.582 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:42.582 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:42.583 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:42.583 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:42.583Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.583Z [INFO] agent.router.manager: shutting down +49:42.583 [INFO] TestAgent: consul server down +49:42.583 [INFO] TestAgent: shutdown complete +49:42.583 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23739 network=tcp +49:42.583 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23739 network=udp +49:42.583 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23740 network=tcp +49:43.084 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.084 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} ---- PASS: TestTxnEndpoint_Bad_Size_Item (4.99s) - --- PASS: TestTxnEndpoint_Bad_Size_Item/exceeds_default_limits (0.97s) - --- PASS: TestTxnEndpoint_Bad_Size_Item/exceeds_configured_max_txn_len (1.01s) - --- PASS: TestTxnEndpoint_Bad_Size_Item/exceeds_default_max_kv_value_size (1.44s) - --- PASS: TestTxnEndpoint_Bad_Size_Item/allowed (1.57s) +2022-12-04T05:49:42.467Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:42.478Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:42.479 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5c6e6b06-f805-c7f8-c805-fcae3806eb69 Address:127.0.0.1:23709}]" +49:42.480 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23709 [Follower]" leader= +49:42.480 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:42.480 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:42.480Z [INFO] agent.router: Initializing LAN area manager +49:42.481 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23709) (DC: dc1)" +49:42.481 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:42.481 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23822 network=udp +49:42.481 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23822 network=tcp +49:42.481 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23823 network=tcp +49:42.481 [INFO] TestAgent: started state syncer +49:42.481 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23696 network=tcp +49:42.548 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:42.548 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23709 [Candidate]" term=2 +49:42.548 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:42.548 [DEBUG] TestAgent.server.raft: vote granted: from=5c6e6b06-f805-c7f8-c805-fcae3806eb69 term=2 tally=1 +49:42.548 [INFO] TestAgent.server.raft: election won: tally=1 +49:42.548 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23709 [Leader]" +49:42.548 [INFO] TestAgent.server: cluster leadership acquired +49:42.548 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23709 +49:42.549 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:42.550 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:42.550 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:42.550 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:42.550 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:42.550 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:42.550 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:42.550 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:42.550 [INFO] TestAgent.server: New leader elected: payload=test-node +49:42.550 [INFO] TestAgent.server: federation state anti-entropy synced +49:42.588 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.044673ms client=127.0.0.1:36984 client_network=udp +49:42.589 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.244315ms client=127.0.0.1:34994 client_network=udp +49:42.590 [DEBUG] TestAgent.dns: request served from client: name=71aca29a-b41c-b186-ed3c-3eee4aac8482.query.consul. type=AAAA class=IN latency=1.011043ms client=127.0.0.1:46775 client_network=udp +49:42.590 [INFO] TestAgent: Requesting shutdown +49:42.590 [INFO] TestAgent.server: shutting down server +49:42.590 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:42.590 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:42.590 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:42.590 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:42.590 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:42.590 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:42.590 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:42.590 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:42.590 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:42.590 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:42.590 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:42.590 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:42.591Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.591Z [INFO] agent.router.manager: shutting down +49:42.591 [INFO] TestAgent: consul server down +49:42.591 [INFO] TestAgent: shutdown complete +49:42.591 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23822 network=tcp +49:42.591 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23822 network=udp +49:42.591 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23823 network=tcp +49:43.091 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.091 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} -2021-10-31T23:13:39.922Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:39.944Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:39.950 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:46887ab9-15b0-7270-0241-555bc537ba33 Address:127.0.0.1:16548}]" -13:39.951 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-46887ab9-15b0-7270-0241-555bc537ba33.dc1 127.0.0.1 -13:39.951 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-46887ab9-15b0-7270-0241-555bc537ba33 127.0.0.1 -2021-10-31T23:13:39.951Z [INFO] agent.router: Initializing LAN area manager -13:39.952 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16487 network=udp -13:39.952 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16548 [Follower]" leader= -13:39.952 [INFO] TestAgent.server: Adding LAN server: server="Node-46887ab9-15b0-7270-0241-555bc537ba33 (Addr: tcp/127.0.0.1:16548) (DC: dc1)" -13:39.952 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-46887ab9-15b0-7270-0241-555bc537ba33.dc1 area=wan -13:39.952 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16487 network=tcp -13:39.953 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16488 network=tcp -13:39.953 [INFO] TestAgent: started state syncer -13:39.953 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16514 network=tcp -13:40.006 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:40.006 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16548 [Candidate]" term=2 -13:40.006 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:40.006 [DEBUG] TestAgent.server.raft: vote granted: from=46887ab9-15b0-7270-0241-555bc537ba33 term=2 tally=1 -13:40.006 [INFO] TestAgent.server.raft: election won: tally=1 -13:40.006 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16548 [Leader]" -13:40.006 [INFO] TestAgent.server: cluster leadership acquired -13:40.007 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16548 -13:40.008 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:40.008 [INFO] TestAgent.server: New leader elected: payload=Node-46887ab9-15b0-7270-0241-555bc537ba33 -13:40.010 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:40.010 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:40.010 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:40.010 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:40.010 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:40.010 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-46887ab9-15b0-7270-0241-555bc537ba33 -13:40.010 [INFO] TestAgent.server: member joined, marking health alive: member=Node-46887ab9-15b0-7270-0241-555bc537ba33 -13:40.011 [INFO] TestAgent.server: federation state anti-entropy synced -13:40.243 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:40.243 [INFO] TestAgent: Synced node info -13:40.277 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:40.298 [INFO] TestAgent: Requesting shutdown -13:40.314 [INFO] TestAgent.server: shutting down server -13:40.314 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:40.314 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:40.314 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:40.314 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:40.314 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:40.314 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:40.314 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:40.314 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:40.314 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:40.314 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:40.314 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:40.315Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:40.315Z [INFO] agent.router.manager: shutting down -13:40.315 [INFO] TestAgent: consul server down -13:40.315 [INFO] TestAgent: shutdown complete -13:40.315 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16487 network=tcp -13:40.315 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16487 network=udp -13:40.315 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16488 network=tcp -13:40.838 [INFO] TestAgent: Waiting for endpoints to shut down -13:40.838 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_Delete (1.08s) +2022-12-04T05:49:42.321Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:42.334Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:42.335 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dfc0c0ef-d74e-c2b3-a428-4ae21bd89491 Address:127.0.0.1:23792}]" +49:42.335 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23792 [Follower]" leader= +49:42.336 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:42.336 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:42.336Z [INFO] agent.router: Initializing LAN area manager +49:42.336 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23780 network=udp +49:42.336 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:42.336 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23792) (DC: dc1)" +49:42.336 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23780 network=tcp +49:42.337 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23781 network=tcp +49:42.337 [INFO] TestAgent: started state syncer +49:42.337 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23821 network=tcp +49:42.382 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:42.382 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23792 [Candidate]" term=2 +49:42.382 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:42.382 [DEBUG] TestAgent.server.raft: vote granted: from=dfc0c0ef-d74e-c2b3-a428-4ae21bd89491 term=2 tally=1 +49:42.382 [INFO] TestAgent.server.raft: election won: tally=1 +49:42.382 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23792 [Leader]" +49:42.382 [INFO] TestAgent.server: cluster leadership acquired +49:42.382 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23792 +49:42.382 [INFO] TestAgent.server: New leader elected: payload=test-node +49:42.383 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:42.384 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:42.384 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:42.384 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:42.384 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:42.384 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:42.384 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:42.384 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:42.385 [INFO] TestAgent.server: federation state anti-entropy synced +49:42.460 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:42.461 [INFO] TestAgent: Synced node info +49:42.612 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:42.612 [DEBUG] TestAgent: Node info in sync +49:42.612 [DEBUG] TestAgent: Node info in sync +49:42.749 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.54889ms client=127.0.0.1:45577 client_network=udp +49:42.751 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.107943ms client=127.0.0.1:56606 client_network=udp +49:42.752 [DEBUG] TestAgent.dns: request served from client: name=0d99fbcd-57c5-88ed-0f92-9168ecc9d208.query.consul. type=ANY class=IN latency=967.284µs client=127.0.0.1:46134 client_network=udp +49:42.753 [INFO] TestAgent: Requesting shutdown +49:42.753 [INFO] TestAgent.server: shutting down server +49:42.753 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:42.753 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:42.753 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:42.753 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:42.753 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:42.753 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:42.753 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:42.753 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:42.753 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:42.753 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:42.753 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:42.753Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.753Z [INFO] agent.router.manager: shutting down +49:42.753 [INFO] TestAgent: consul server down +49:42.753 [INFO] TestAgent: shutdown complete +49:42.753 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23780 network=tcp +49:42.753 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23780 network=udp +49:42.753 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23781 network=tcp +49:43.253 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.253 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} -2021-10-31T23:13:39.914Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:39.952Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:39.954 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:19c3da02-c370-d3d4-b756-44458d167bef Address:127.0.0.1:16555}]" -13:39.955 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-19c3da02-c370-d3d4-b756-44458d167bef.dc1 127.0.0.1 -13:39.955 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16555 [Follower]" leader= -13:39.955 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-19c3da02-c370-d3d4-b756-44458d167bef 127.0.0.1 -2021-10-31T23:13:39.955Z [INFO] agent.router: Initializing LAN area manager -13:39.956 [INFO] TestAgent.server: Adding LAN server: server="Node-19c3da02-c370-d3d4-b756-44458d167bef (Addr: tcp/127.0.0.1:16555) (DC: dc1)" -13:39.956 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16494 network=udp -13:39.956 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-19c3da02-c370-d3d4-b756-44458d167bef.dc1 area=wan -13:39.956 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16494 network=tcp -13:39.956 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16495 network=tcp -13:39.957 [INFO] TestAgent: started state syncer -13:39.957 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16486 network=tcp -13:40.014 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:40.014 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16555 [Candidate]" term=2 -13:40.014 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:40.014 [DEBUG] TestAgent.server.raft: vote granted: from=19c3da02-c370-d3d4-b756-44458d167bef term=2 tally=1 -13:40.014 [INFO] TestAgent.server.raft: election won: tally=1 -13:40.014 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16555 [Leader]" -13:40.014 [INFO] TestAgent.server: cluster leadership acquired -13:40.014 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16555 -13:40.015 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:40.016 [INFO] TestAgent.server: New leader elected: payload=Node-19c3da02-c370-d3d4-b756-44458d167bef -13:40.030 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:40.030 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:40.030 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:40.030 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:40.030 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:40.030 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-19c3da02-c370-d3d4-b756-44458d167bef -13:40.030 [INFO] TestAgent.server: member joined, marking health alive: member=Node-19c3da02-c370-d3d4-b756-44458d167bef -13:40.031 [INFO] TestAgent.server: federation state anti-entropy synced -13:40.273 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:40.274 [INFO] TestAgent: Synced node info -13:40.274 [DEBUG] TestAgent: Node info in sync -13:40.343 [INFO] TestAgent: Requesting shutdown -13:40.343 [INFO] TestAgent.server: shutting down server -13:40.343 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:40.343 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:40.343 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:40.343 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:40.343 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:40.343 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:40.343 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:40.343 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:40.343 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:40.343 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:40.344 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:40.344Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:40.344Z [INFO] agent.router.manager: shutting down -13:40.345 [INFO] TestAgent: consul server down -13:40.345 [INFO] TestAgent: shutdown complete -13:40.345 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16494 network=tcp -13:40.345 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16494 network=udp -13:40.345 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16495 network=tcp -13:40.845 [INFO] TestAgent: Waiting for endpoints to shut down -13:40.845 [INFO] TestAgent: Endpoints down ---- PASS: TestRemoteExecGetSpec (1.09s) +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:42.592Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:42.610Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:42.617 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fb9c023f-aae4-2f58-2d84-5a4b18299343 Address:127.0.0.1:23841}]" +49:42.617 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:42.618 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:42.618Z [INFO] agent.router: Initializing LAN area manager +49:42.618 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23829 network=udp +49:42.618 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23841 [Follower]" leader= +49:42.619 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23841) (DC: dc1)" +49:42.619 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:42.619 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23829 network=tcp +49:42.620 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23830 network=tcp +49:42.620 [INFO] TestAgent: started state syncer +49:42.620 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23842 network=tcp +49:42.662 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:42.662 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23841 [Candidate]" term=2 +49:42.662 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:42.662 [DEBUG] TestAgent.server.raft: vote granted: from=fb9c023f-aae4-2f58-2d84-5a4b18299343 term=2 tally=1 +49:42.662 [INFO] TestAgent.server.raft: election won: tally=1 +49:42.662 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23841 [Leader]" +49:42.662 [INFO] TestAgent.server: cluster leadership acquired +49:42.662 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23841 +49:42.662 [INFO] TestAgent.server: New leader elected: payload=test-node +49:42.663 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:42.664 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:42.664 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:42.664 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:42.664 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:42.664 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:42.664 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:42.664 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:42.665 [INFO] TestAgent.server: federation state anti-entropy synced +49:42.715 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:42.715 [INFO] TestAgent: Synced node info +49:42.715 [DEBUG] TestAgent: Node info in sync +49:42.823 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.208888ms client=127.0.0.1:45534 client_network=udp +49:42.826 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.239894ms client=127.0.0.1:44569 client_network=udp +49:42.827 [DEBUG] TestAgent.dns: request served from client: name=9d525461-6c37-e8b8-7c29-bee2aed67ca1.query.consul. type=AAAA class=IN latency=966.583µs client=127.0.0.1:58265 client_network=udp +49:42.828 [INFO] TestAgent: Requesting shutdown +49:42.828 [INFO] TestAgent.server: shutting down server +49:42.828 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:42.828 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:42.828 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:42.828 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:42.828 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:42.828 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:42.828 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:42.828 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:42.828 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:42.828 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:42.828 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:42.828Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.828Z [INFO] agent.router.manager: shutting down +49:42.828 [INFO] TestAgent: consul server down +49:42.828 [INFO] TestAgent: shutdown complete +49:42.828 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23829 network=tcp +49:42.828 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23829 network=udp +49:42.828 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23830 network=tcp +49:43.328 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.328 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} -[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:40.103Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:40.196Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:40.211 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dc9cd32b-2ac4-7482-78f4-be00b3cb0838 Address:127.0.0.1:16471}]" -13:40.212 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dc9cd32b-2ac4-7482-78f4-be00b3cb0838.dc1 127.0.0.1 -13:40.212 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dc9cd32b-2ac4-7482-78f4-be00b3cb0838 127.0.0.1 -2021-10-31T23:13:40.212Z [INFO] agent.router: Initializing LAN area manager -13:40.212 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16536 network=udp -13:40.212 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16471 [Follower]" leader= -13:40.213 [INFO] TestAgent.server: Adding LAN server: server="Node-dc9cd32b-2ac4-7482-78f4-be00b3cb0838 (Addr: tcp/127.0.0.1:16471) (DC: dc1)" -13:40.213 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dc9cd32b-2ac4-7482-78f4-be00b3cb0838.dc1 area=wan -13:40.213 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16536 network=tcp -13:40.213 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16537 network=tcp -13:40.214 [INFO] TestAgent: started state syncer -13:40.214 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16521 network=tcp -13:40.253 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:40.253 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16471 [Candidate]" term=2 -13:40.254 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:40.254 [DEBUG] TestAgent.server.raft: vote granted: from=dc9cd32b-2ac4-7482-78f4-be00b3cb0838 term=2 tally=1 -13:40.254 [INFO] TestAgent.server.raft: election won: tally=1 -13:40.254 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16471 [Leader]" -13:40.254 [INFO] TestAgent.server: cluster leadership acquired -13:40.254 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16471 -13:40.255 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:40.256 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:40.256 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:40.256 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:40.256 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:40.256 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:40.257 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dc9cd32b-2ac4-7482-78f4-be00b3cb0838 -13:40.257 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dc9cd32b-2ac4-7482-78f4-be00b3cb0838 -13:40.257 [INFO] TestAgent.server: New leader elected: payload=Node-dc9cd32b-2ac4-7482-78f4-be00b3cb0838 -13:40.321 [INFO] TestAgent.server: federation state anti-entropy synced -13:40.387 [INFO] TestAgent: Requesting shutdown -13:40.388 [INFO] TestAgent.server: shutting down server -13:40.388 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:40.388 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:40.388 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:40.388 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:40.388 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:40.388 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:40.388 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:40.388 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:40.388 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:40.388 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:40.388 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:40.388 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:40.388Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:40.388Z [INFO] agent.router.manager: shutting down -13:40.388 [INFO] TestAgent: consul server down -13:40.388 [INFO] TestAgent: shutdown complete -13:40.388 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16536 network=tcp -13:40.388 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16536 network=udp -13:40.388 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16537 network=tcp -13:40.889 [INFO] TestAgent: Waiting for endpoints to shut down -13:40.889 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionCreate_Delete (0.88s) +2022-12-04T05:49:42.583Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:42.595Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:42.596 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1ee87a85-8523-09ce-f77e-54dc09651748 Address:127.0.0.1:23869}]" +49:42.596 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23869 [Follower]" leader= +49:42.596 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:42.597 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:42.597Z [INFO] agent.router: Initializing LAN area manager +49:42.597 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23815 network=udp +49:42.597 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:42.597 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23869) (DC: dc1)" +49:42.597 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23815 network=tcp +49:42.598 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23816 network=tcp +49:42.598 [INFO] TestAgent: started state syncer +49:42.598 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23828 network=tcp +49:42.658 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:42.658 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23869 [Candidate]" term=2 +49:42.658 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:42.658 [DEBUG] TestAgent.server.raft: vote granted: from=1ee87a85-8523-09ce-f77e-54dc09651748 term=2 tally=1 +49:42.658 [INFO] TestAgent.server.raft: election won: tally=1 +49:42.658 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23869 [Leader]" +49:42.658 [INFO] TestAgent.server: cluster leadership acquired +49:42.658 [INFO] TestAgent.server: New leader elected: payload=test-node +49:42.658 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23869 +49:42.659 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:42.660 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:42.660 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:42.660 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:42.660 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:42.660 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:42.660 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:42.660 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:42.661 [INFO] TestAgent.server: federation state anti-entropy synced +49:42.826 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.361594ms client=127.0.0.1:44660 client_network=udp +49:42.828 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.212514ms client=127.0.0.1:34067 client_network=udp +49:42.829 [DEBUG] TestAgent.dns: request served from client: name=81298441-c57f-5ea1-d73a-6275b35cfd4d.query.consul. type=ANY class=IN latency=1.382612ms client=127.0.0.1:44753 client_network=udp +49:42.829 [INFO] TestAgent: Requesting shutdown +49:42.829 [INFO] TestAgent.server: shutting down server +49:42.829 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:42.829 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:42.829 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:42.829 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:42.829 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:42.829 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:42.829 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:42.829 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:42.829 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:42.829 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:42.829 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:42.829 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +2022-12-04T05:49:42.829Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.829Z [INFO] agent.router.manager: shutting down +49:42.830 [INFO] TestAgent: consul server down +49:42.830 [INFO] TestAgent: shutdown complete +49:42.830 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23815 network=tcp +49:42.830 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23815 network=udp +49:42.830 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23816 network=tcp +49:43.330 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.330 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} -2021-10-31T23:13:40.255Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:40.319Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:40.342 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0bc82b31-2f44-b70c-ba4a-36b6ea03e32a Address:127.0.0.1:16541}]" -13:40.343 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0bc82b31-2f44-b70c-ba4a-36b6ea03e32a.dc1 127.0.0.1 -13:40.344 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16541 [Follower]" leader= -13:40.344 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0bc82b31-2f44-b70c-ba4a-36b6ea03e32a 127.0.0.1 -2021-10-31T23:13:40.344Z [INFO] agent.router: Initializing LAN area manager -13:40.344 [INFO] TestAgent.server: Adding LAN server: server="Node-0bc82b31-2f44-b70c-ba4a-36b6ea03e32a (Addr: tcp/127.0.0.1:16541) (DC: dc1)" -13:40.344 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16599 network=udp -13:40.344 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0bc82b31-2f44-b70c-ba4a-36b6ea03e32a.dc1 area=wan -13:40.344 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16599 network=tcp -13:40.349 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16600 network=tcp -13:40.350 [INFO] TestAgent: started state syncer -13:40.350 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16563 network=tcp -13:40.381 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:40.381 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16541 [Candidate]" term=2 -13:40.382 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:40.382 [DEBUG] TestAgent.server.raft: vote granted: from=0bc82b31-2f44-b70c-ba4a-36b6ea03e32a term=2 tally=1 -13:40.382 [INFO] TestAgent.server.raft: election won: tally=1 -13:40.382 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16541 [Leader]" -13:40.382 [INFO] TestAgent.server: cluster leadership acquired -13:40.382 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16541 -13:40.383 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:40.384 [INFO] TestAgent.server: New leader elected: payload=Node-0bc82b31-2f44-b70c-ba4a-36b6ea03e32a -13:40.385 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:40.385 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:40.385 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:40.385 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:40.389 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:40.390 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0bc82b31-2f44-b70c-ba4a-36b6ea03e32a -13:40.390 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0bc82b31-2f44-b70c-ba4a-36b6ea03e32a -13:40.542 [INFO] TestAgent: Requesting shutdown -13:40.542 [INFO] TestAgent.server: shutting down server -13:40.542 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:40.543 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:40.543 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:40.543 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:40.543 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:40.543 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" -13:40.543 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:40.543 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:40.543 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:40.543 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:40.543 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:40.543 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:40.543Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:40.543Z [INFO] agent.router.manager: shutting down -13:40.543 [INFO] TestAgent: consul server down -13:40.543 [INFO] TestAgent: shutdown complete -13:40.543 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16599 network=tcp -13:40.543 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16599 network=udp -13:40.543 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16600 network=tcp -13:41.047 [INFO] TestAgent: Waiting for endpoints to shut down -13:41.047 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionCreate_DefaultCheck (0.85s) +2022-12-04T05:49:42.471Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:42.481Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:42.482 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a9ff1432-d3f1-2459-0f55-143ddb68edd6 Address:127.0.0.1:23806}]" +49:42.482 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23806 [Follower]" leader= +49:42.483 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:42.483 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:42.483Z [INFO] agent.router: Initializing LAN area manager +49:42.483 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23806) (DC: dc1)" +49:42.483 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:42.483 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23697 network=udp +49:42.483 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23697 network=tcp +49:42.484 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23698 network=tcp +49:42.484 [INFO] TestAgent: started state syncer +49:42.484 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23752 network=tcp +49:42.544 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:42.544 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23806 [Candidate]" term=2 +49:42.544 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:42.544 [DEBUG] TestAgent.server.raft: vote granted: from=a9ff1432-d3f1-2459-0f55-143ddb68edd6 term=2 tally=1 +49:42.544 [INFO] TestAgent.server.raft: election won: tally=1 +49:42.544 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23806 [Leader]" +49:42.544 [INFO] TestAgent.server: cluster leadership acquired +49:42.544 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23806 +49:42.548 [INFO] TestAgent.server: New leader elected: payload=test-node +49:42.548 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:42.549 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:42.549 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:42.549 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:42.549 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:42.549 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:42.549 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:42.549 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:42.550 [INFO] TestAgent.server: federation state anti-entropy synced +49:42.663 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:42.664 [INFO] TestAgent: Synced node info +49:42.664 [DEBUG] TestAgent: Node info in sync +49:42.828 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.278053ms client=127.0.0.1:51754 client_network=udp +49:42.830 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=940.499µs client=127.0.0.1:33779 client_network=udp +49:42.831 [DEBUG] TestAgent.dns: request served from client: name=2b7fdc2f-9adf-1102-5367-c7f462d6d8e7.query.consul. type=A class=IN latency=1.083385ms client=127.0.0.1:49302 client_network=udp +49:42.833 [INFO] TestAgent: Requesting shutdown +49:42.833 [INFO] TestAgent.server: shutting down server +49:42.833 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:42.833 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:42.833 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:42.833 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:42.833 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:42.833 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:42.833 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:42.833 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:42.833 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:42.833 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:42.833 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:42.833Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.833Z [INFO] agent.router.manager: shutting down +49:42.833 [INFO] TestAgent: consul server down +49:42.833 [INFO] TestAgent: shutdown complete +49:42.833 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23697 network=tcp +49:42.833 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23697 network=udp +49:42.833 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23698 network=tcp +49:43.334 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.334 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} -2021-10-31T23:13:40.044Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:40.186Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:40.188 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c66b1011-11a5-b768-45f2-4c199287164f Address:127.0.0.1:16590}]" -13:40.189 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c66b1011-11a5-b768-45f2-4c199287164f.dc1 127.0.0.1 -13:40.189 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c66b1011-11a5-b768-45f2-4c199287164f 127.0.0.1 -2021-10-31T23:13:40.189Z [INFO] agent.router: Initializing LAN area manager -13:40.189 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16590 [Follower]" leader= -13:40.190 [INFO] TestAgent.server: Adding LAN server: server="Node-c66b1011-11a5-b768-45f2-4c199287164f (Addr: tcp/127.0.0.1:16590) (DC: dc1)" -13:40.190 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c66b1011-11a5-b768-45f2-4c199287164f.dc1 area=wan -13:40.190 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16585 network=udp -13:40.190 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16585 network=tcp -13:40.191 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16586 network=tcp -13:40.191 [INFO] TestAgent: started state syncer -13:40.191 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16535 network=tcp -13:40.227 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:40.227 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16590 [Candidate]" term=2 -13:40.227 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:40.227 [DEBUG] TestAgent.server.raft: vote granted: from=c66b1011-11a5-b768-45f2-4c199287164f term=2 tally=1 -13:40.227 [INFO] TestAgent.server.raft: election won: tally=1 -13:40.227 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16590 [Leader]" -13:40.227 [INFO] TestAgent.server: cluster leadership acquired -13:40.227 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16590 -13:40.228 [INFO] TestAgent.server: New leader elected: payload=Node-c66b1011-11a5-b768-45f2-4c199287164f -13:40.228 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:40.229 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:40.229 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:40.229 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:40.229 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:40.229 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:40.230 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c66b1011-11a5-b768-45f2-4c199287164f -13:40.230 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c66b1011-11a5-b768-45f2-4c199287164f -13:40.230 [INFO] TestAgent.server: federation state anti-entropy synced -13:40.342 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:40.344 [INFO] TestAgent: Synced node info -13:40.578 [WARN] TestAgent.server: endpoint injected; this should only be used for testing -13:40.578 [INFO] TestAgent: Requesting shutdown -13:40.578 [INFO] TestAgent.server: shutting down server -13:40.578 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:40.578 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:40.578 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:40.578 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:40.578 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:40.578 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:40.578 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:40.578 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:40.578 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:40.578 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:40.578 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:40.578Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:40.579Z [INFO] agent.router.manager: shutting down -13:40.579 [INFO] TestAgent: consul server down -13:40.579 [INFO] TestAgent: shutdown complete -13:40.579 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16585 network=tcp -13:40.579 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16585 network=udp -13:40.579 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16586 network=tcp -13:41.082 [INFO] TestAgent: Waiting for endpoints to shut down -13:41.082 [INFO] TestAgent: Endpoints down ---- PASS: TestPreparedQuery_Update (1.13s) +2022-12-04T05:49:42.553Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:42.563Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:42.565 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f7189a77-c290-a8ee-ecec-771c96c1e171 Address:127.0.0.1:23848}]" +49:42.565 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:42.565 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:42.565Z [INFO] agent.router: Initializing LAN area manager +49:42.566 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23753 network=udp +49:42.566 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23848 [Follower]" leader= +49:42.566 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23848) (DC: dc1)" +49:42.566 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:42.566 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23753 network=tcp +49:42.566 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23754 network=tcp +49:42.566 [INFO] TestAgent: started state syncer +49:42.566 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23814 network=tcp +49:42.626 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:42.626 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23848 [Candidate]" term=2 +49:42.626 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:42.626 [DEBUG] TestAgent.server.raft: vote granted: from=f7189a77-c290-a8ee-ecec-771c96c1e171 term=2 tally=1 +49:42.626 [INFO] TestAgent.server.raft: election won: tally=1 +49:42.626 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23848 [Leader]" +49:42.627 [INFO] TestAgent.server: cluster leadership acquired +49:42.627 [INFO] TestAgent.server: New leader elected: payload=test-node +49:42.627 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23848 +49:42.628 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:42.630 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:42.630 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:42.630 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:42.630 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:42.630 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:42.630 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:42.630 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:42.631 [INFO] TestAgent.server: federation state anti-entropy synced +49:42.794 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:42.794 [INFO] TestAgent: Synced node info +49:42.794 [DEBUG] TestAgent: Node info in sync +49:42.994 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.334731ms client=127.0.0.1:56357 client_network=udp +49:42.996 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.120595ms client=127.0.0.1:60463 client_network=udp +49:42.997 [DEBUG] TestAgent.dns: request served from client: name=ab21310a-c6c4-2544-0de3-5d642b47ca90.query.consul. type=AAAA class=IN latency=1.222889ms client=127.0.0.1:44650 client_network=udp +49:42.999 [INFO] TestAgent: Requesting shutdown +49:42.999 [INFO] TestAgent.server: shutting down server +49:42.999 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:42.999 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:42.999 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:42.999 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:42.999 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:42.999 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:42.999 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:42.999 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:42.999 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:42.999 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:42.999 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:42.999Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:42.999Z [INFO] agent.router.manager: shutting down +49:42.999 [INFO] TestAgent: consul server down +49:42.999 [INFO] TestAgent: shutdown complete +49:42.999 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23753 network=tcp +49:42.999 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23753 network=udp +49:42.999 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23754 network=tcp +49:43.500 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.500 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{9_9_8_8_6_8_8_5_8_8_-5} -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:40.240Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:40.329Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:40.346 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:62fa45ad-f50c-acc1-5e91-78e7c4094e50 Address:127.0.0.1:16527}]" -13:40.347 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-62fa45ad-f50c-acc1-5e91-78e7c4094e50.dc1 127.0.0.1 -13:40.347 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-62fa45ad-f50c-acc1-5e91-78e7c4094e50 127.0.0.1 -2021-10-31T23:13:40.347Z [INFO] agent.router: Initializing LAN area manager -13:40.348 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16527 [Follower]" leader= -13:40.348 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16564 network=udp -13:40.348 [INFO] TestAgent.server: Adding LAN server: server="Node-62fa45ad-f50c-acc1-5e91-78e7c4094e50 (Addr: tcp/127.0.0.1:16527) (DC: dc1)" -13:40.348 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-62fa45ad-f50c-acc1-5e91-78e7c4094e50.dc1 area=wan -13:40.348 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16564 network=tcp -13:40.357 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16565 network=tcp -13:40.358 [INFO] TestAgent: started state syncer -13:40.358 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16556 network=tcp -13:40.416 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:40.416 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16527 [Candidate]" term=2 -13:40.416 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:40.416 [DEBUG] TestAgent.server.raft: vote granted: from=62fa45ad-f50c-acc1-5e91-78e7c4094e50 term=2 tally=1 -13:40.416 [INFO] TestAgent.server.raft: election won: tally=1 -13:40.416 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16527 [Leader]" -13:40.417 [INFO] TestAgent.server: cluster leadership acquired -13:40.431 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16527 -13:40.432 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:40.433 [INFO] TestAgent.server: New leader elected: payload=Node-62fa45ad-f50c-acc1-5e91-78e7c4094e50 -13:40.434 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:40.434 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:40.434 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:40.434 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:40.434 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:40.434 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-62fa45ad-f50c-acc1-5e91-78e7c4094e50 -13:40.434 [INFO] TestAgent.server: member joined, marking health alive: member=Node-62fa45ad-f50c-acc1-5e91-78e7c4094e50 -13:40.435 [INFO] TestAgent.server: federation state anti-entropy synced -13:40.502 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:40.502 [INFO] TestAgent: Synced node info -13:40.502 [DEBUG] TestAgent: Node info in sync -13:40.602 [INFO] TestAgent: Requesting shutdown -13:40.602 [INFO] TestAgent.server: shutting down server -13:40.602 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:40.602 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:40.602 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:40.602 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:40.602 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:40.602 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:40.602 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:40.602 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:40.602 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:40.602 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:40.602 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:40.603Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:40.603Z [INFO] agent.router.manager: shutting down -13:40.603 [INFO] TestAgent: consul server down -13:40.603 [INFO] TestAgent: shutdown complete -13:40.603 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16564 network=tcp -13:40.603 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16564 network=udp -13:40.603 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16565 network=tcp -13:41.103 [INFO] TestAgent: Waiting for endpoints to shut down -13:41.103 [INFO] TestAgent: Endpoints down ---- PASS: TestTxnEndpoint_KV_Actions (1.84s) - --- PASS: TestTxnEndpoint_KV_Actions/#00 (0.94s) - --- PASS: TestTxnEndpoint_KV_Actions/#01 (0.90s) -=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{9_9_8_8_6_8_8_5_8_8_-5} -2021-10-31T23:13:40.183Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:40.183Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:40.224Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:40.224Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:40.234 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3fd842e7-9679-d4b5-13af-f8397e85a460 Address:127.0.0.1:16597}]" -13:40.235 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3fd842e7-9679-d4b5-13af-f8397e85a460.dc1 127.0.0.1 -13:40.236 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3fd842e7-9679-d4b5-13af-f8397e85a460 127.0.0.1 -2021-10-31T23:13:40.236Z [INFO] agent.router: Initializing LAN area manager -13:40.236 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16522 network=udp -13:40.236 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16597 [Follower]" leader= -13:40.236 [INFO] TestAgent.server: Adding LAN server: server="Node-3fd842e7-9679-d4b5-13af-f8397e85a460 (Addr: tcp/127.0.0.1:16597) (DC: dc1)" -13:40.236 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3fd842e7-9679-d4b5-13af-f8397e85a460.dc1 area=wan -13:40.236 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16522 network=tcp -13:40.250 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16523 network=tcp -13:40.250 [INFO] TestAgent: started state syncer -13:40.250 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16598 network=tcp -13:40.310 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:40.310 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16597 [Candidate]" term=2 -13:40.310 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:40.310 [DEBUG] TestAgent.server.raft: vote granted: from=3fd842e7-9679-d4b5-13af-f8397e85a460 term=2 tally=1 -13:40.310 [INFO] TestAgent.server.raft: election won: tally=1 -13:40.310 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16597 [Leader]" -13:40.310 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=2 mode=2 found=true leader=127.0.0.1:16597 -13:40.310 [INFO] TestAgent.server: cluster leadership acquired -13:40.311 [INFO] TestAgent.server: initializing acls -13:40.311 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:40.311 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:40.312 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:40.312 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:40.312 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:40.312 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:40.312 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-3fd842e7-9679-d4b5-13af-f8397e85a460 -13:40.313 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-3fd842e7-9679-d4b5-13af-f8397e85a460.dc1 -13:40.314 [INFO] TestAgent.server: New leader elected: payload=Node-3fd842e7-9679-d4b5-13af-f8397e85a460 -13:40.314 [INFO] TestAgent.server: Updating LAN server: server="Node-3fd842e7-9679-d4b5-13af-f8397e85a460 (Addr: tcp/127.0.0.1:16597) (DC: dc1)" -13:40.314 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-3fd842e7-9679-d4b5-13af-f8397e85a460.dc1 area=wan -13:40.318 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:40.320 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:40.320 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:40.320 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:40.320 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:40.320 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:40.320 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3fd842e7-9679-d4b5-13af-f8397e85a460 -13:40.320 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3fd842e7-9679-d4b5-13af-f8397e85a460 -13:40.334 [INFO] TestAgent.server: federation state anti-entropy synced -13:40.334 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3fd842e7-9679-d4b5-13af-f8397e85a460 -13:40.353 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:40.353 [INFO] TestAgent: Synced node info -13:40.634 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-3fd842e7-9679-d4b5-13af-f8397e85a460 -13:40.634 [INFO] TestAgent: Requesting shutdown -13:40.634 [INFO] TestAgent.server: shutting down server -13:40.634 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:40.634 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:40.634 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:40.634 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:40.634 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:40.634 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:40.634 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:40.634 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:40.634 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:40.634 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:40.634 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:40.634 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:40.634 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:40.634 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:40.634 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -2021-10-31T23:13:40.634Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:40.634Z [INFO] agent.router.manager: shutting down -13:40.634 [INFO] TestAgent: consul server down -13:40.634 [INFO] TestAgent: shutdown complete -13:40.634 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16522 network=tcp -13:40.634 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16522 network=udp -13:40.634 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16523 network=tcp -13:41.135 [INFO] TestAgent: Waiting for endpoints to shut down -13:41.135 [INFO] TestAgent: Endpoints down -=== RUN TestSnapshot_Options/PUT#02 -2021-10-31T23:13:40.591Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:40.664Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:40.673 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:435f52c8-808e-6df5-d223-b56e1f332bca Address:127.0.0.1:16499}]" -13:40.673 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16499 [Follower]" leader= -13:40.674 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-435f52c8-808e-6df5-d223-b56e1f332bca.dc1 127.0.0.1 -13:40.674 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-435f52c8-808e-6df5-d223-b56e1f332bca 127.0.0.1 -2021-10-31T23:13:40.674Z [INFO] agent.router: Initializing LAN area manager -13:40.675 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16508 network=udp -13:40.675 [INFO] TestAgent.server: Adding LAN server: server="Node-435f52c8-808e-6df5-d223-b56e1f332bca (Addr: tcp/127.0.0.1:16499) (DC: dc1)" -13:40.675 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-435f52c8-808e-6df5-d223-b56e1f332bca.dc1 area=wan -13:40.675 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16508 network=tcp -13:40.676 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16509 network=tcp -13:40.676 [INFO] TestAgent: started state syncer -13:40.676 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16549 network=tcp -13:40.738 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:40.738 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16499 [Candidate]" term=2 -13:40.738 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:40.738 [DEBUG] TestAgent.server.raft: vote granted: from=435f52c8-808e-6df5-d223-b56e1f332bca term=2 tally=1 -13:40.738 [INFO] TestAgent.server.raft: election won: tally=1 -13:40.738 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16499 [Leader]" -13:40.738 [INFO] TestAgent.server: cluster leadership acquired -13:40.738 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16499 -13:40.739 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:40.740 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:40.740 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:40.740 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:40.740 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:40.740 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:40.740 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-435f52c8-808e-6df5-d223-b56e1f332bca -13:40.740 [INFO] TestAgent.server: member joined, marking health alive: member=Node-435f52c8-808e-6df5-d223-b56e1f332bca -13:40.741 [INFO] TestAgent.server: New leader elected: payload=Node-435f52c8-808e-6df5-d223-b56e1f332bca -13:40.741 [INFO] TestAgent.server: federation state anti-entropy synced -13:40.811 [INFO] TestAgent: Requesting shutdown -13:40.811 [INFO] TestAgent.server: shutting down server -13:40.811 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:40.811 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:40.811 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:40.811 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:40.811 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:40.811 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:40.811 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:40.811 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:40.811 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:40.811 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:40.811 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:40.811 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:40.811Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:40.811Z [INFO] agent.router.manager: shutting down -13:40.811 [INFO] TestAgent: consul server down -13:40.811 [INFO] TestAgent: shutdown complete -13:40.811 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16508 network=tcp -13:40.811 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16508 network=udp -13:40.811 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16509 network=tcp -13:41.313 [INFO] TestAgent: Waiting for endpoints to shut down -13:41.314 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionCreate (0.79s) +49:38.119 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-e893c8ae-d304-ad86-3305-5de182749f7c 127.0.0.1 +2022-12-04T05:49:38.119Z [INFO] agent.router: Initializing LAN area manager +49:38.119 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23378 network=udp +49:38.119 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23378 network=tcp +49:38.120 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23344 network=tcp +49:38.120 [INFO] TestAgent: started state syncer +49:38.120 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23440 network=tcp +2022-12-04T05:49:38.120Z [WARN] agent.router.manager: No servers available +49:38.120 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No known Consul servers" +49:38.121 [DEBUG] TestAgent.client.memberlist.lan: memberlist: Stream connection from=127.0.0.1:55958 +49:38.121 [INFO] TestAgent.client.serf.lan: serf: EventMemberJoin: Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb 127.0.0.1 +49:38.121 [INFO] TestAgent.client: adding server: server="Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb (Addr: tcp/127.0.0.1:23278) (DC: dc1)" +49:38.121 [INFO] TestAgent.client: New leader elected: payload=Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb +49:38.121 [INFO] TestAgent.client: client starting leave +49:38.172 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader +49:38.172 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb +49:38.372 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader +49:38.372 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb +49:38.572 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader +49:38.572 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb +49:38.572 [DEBUG] TestAgent.client.serf.lan: serf: messageLeaveType: Node-e893c8ae-d304-ad86-3305-5de182749f7c +49:38.772 [DEBUG] TestAgent.client.serf.lan: serf: messageUserEventType: consul:new-leader +49:38.772 [DEBUG] TestAgent.client.serf.lan: serf: messageJoinType: Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb +49:38.772 [DEBUG] TestAgent.client.serf.lan: serf: messageLeaveType: Node-e893c8ae-d304-ad86-3305-5de182749f7c +49:38.772 [DEBUG] TestAgent.client.serf.lan: serf: messageLeaveType: Node-e893c8ae-d304-ad86-3305-5de182749f7c +49:38.920 [INFO] TestAgent.client.serf.lan: serf: EventMemberLeave: Node-e893c8ae-d304-ad86-3305-5de182749f7c 127.0.0.1 +49:38.972 [DEBUG] TestAgent.client.serf.lan: serf: messageLeaveType: Node-e893c8ae-d304-ad86-3305-5de182749f7c +49:40.896 [INFO] TestAgent: Synced node info +49:40.896 [DEBUG] TestAgent: Node info in sync +49:42.071 [DEBUG] TestAgent: Node info in sync +49:42.519 [INFO] TestAgent: Requesting shutdown +49:42.520 [INFO] TestAgent.client: shutting down client +49:42.520 [INFO] TestAgent: consul client down +49:42.520 [INFO] TestAgent: shutdown complete +49:42.520 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23378 network=tcp +49:42.520 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23378 network=udp +49:42.520 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23344 network=tcp +49:43.020 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.020 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:37.760Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.769Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.770 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bfe4a9cb-11d3-8e4e-468d-66c2646742bb Address:127.0.0.1:23278}]" +49:37.771 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23278 [Follower]" leader= +49:37.771 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb.dc1 127.0.0.1 +49:37.771 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb 127.0.0.1 +2022-12-04T05:49:37.771Z [INFO] agent.router: Initializing LAN area manager +49:37.772 [INFO] TestAgent.server: Adding LAN server: server="Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb (Addr: tcp/127.0.0.1:23278) (DC: dc1)" +49:37.772 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23357 network=tcp +49:37.772 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb.dc1 area=wan +49:37.772 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23357 network=udp +49:37.773 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23274 network=tcp +49:37.773 [INFO] TestAgent: started state syncer +49:37.773 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23335 network=tcp +49:37.816 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:37.816 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23278 [Candidate]" term=2 +49:37.816 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:37.816 [DEBUG] TestAgent.server.raft: vote granted: from=bfe4a9cb-11d3-8e4e-468d-66c2646742bb term=2 tally=1 +49:37.816 [INFO] TestAgent.server.raft: election won: tally=1 +49:37.816 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23278 [Leader]" +49:37.816 [INFO] TestAgent.server: cluster leadership acquired +49:37.816 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23278 +49:37.816 [INFO] TestAgent.server: New leader elected: payload=Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb +49:37.817 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:37.818 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:37.818 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:37.818 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:37.818 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:37.818 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:37.818 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb +49:37.818 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb +49:37.819 [INFO] TestAgent.server: federation state anti-entropy synced +49:37.924 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:37.924 [INFO] TestAgent: Synced node info +49:38.120 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:23283] +49:38.121 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:23283 +49:38.121 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e893c8ae-d304-ad86-3305-5de182749f7c 127.0.0.1 +49:38.121 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +49:38.121 [DEBUG] TestAgent: systemd notify failed: error="No socket" +49:38.121 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e893c8ae-d304-ad86-3305-5de182749f7c +49:38.325 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:38.325 [DEBUG] TestAgent: Node info in sync +49:38.325 [DEBUG] TestAgent: Node info in sync +49:38.520 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb +49:38.520 [DEBUG] TestAgent.server.serf.lan: serf: messageLeaveType: Node-e893c8ae-d304-ad86-3305-5de182749f7c +49:38.720 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb +49:38.720 [DEBUG] TestAgent.server.serf.lan: serf: messageLeaveType: Node-e893c8ae-d304-ad86-3305-5de182749f7c +49:38.772 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb +49:38.772 [DEBUG] TestAgent.server.serf.lan: serf: messageLeaveType: Node-e893c8ae-d304-ad86-3305-5de182749f7c +49:38.920 [DEBUG] TestAgent.server.serf.lan: serf: messageJoinType: Node-bfe4a9cb-11d3-8e4e-468d-66c2646742bb +49:38.920 [DEBUG] TestAgent.server.serf.lan: serf: messageLeaveType: Node-e893c8ae-d304-ad86-3305-5de182749f7c +49:39.120 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-e893c8ae-d304-ad86-3305-5de182749f7c 127.0.0.1 +49:39.120 [INFO] TestAgent.server: deregistering member: member=Node-e893c8ae-d304-ad86-3305-5de182749f7c reason=left +49:43.020 [INFO] TestAgent: Requesting shutdown +49:43.020 [INFO] TestAgent.server: shutting down server +49:43.020 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.020 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.020 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.020 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.020 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.020 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.020 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.020 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.020 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.020 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.020 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:43.021Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.021Z [INFO] agent.router.manager: shutting down +49:43.021 [INFO] TestAgent: consul server down +49:43.021 [INFO] TestAgent: shutdown complete +49:43.021 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23357 network=tcp +49:43.021 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23357 network=udp +49:43.021 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23274 network=tcp +49:43.521 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.521 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Leave (5.77s) +=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{8_8_8_8_6_8_8_5_8_8_-5} +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:42.599Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:42.636Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:42.638 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ae52acc4-06bd-c14f-42e0-40cc59108d0b Address:127.0.0.1:23820}]" +49:42.638 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23820 [Follower]" leader= +49:42.638 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:42.638 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:42.638Z [INFO] agent.router: Initializing LAN area manager +49:42.639 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23820) (DC: dc1)" +49:42.639 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23843 network=udp +49:42.639 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:42.639 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23843 network=tcp +49:42.639 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23844 network=tcp +49:42.639 [INFO] TestAgent: started state syncer +49:42.639 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23800 network=tcp +49:42.693 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:42.693 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23820 [Candidate]" term=2 +49:42.693 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:42.693 [DEBUG] TestAgent.server.raft: vote granted: from=ae52acc4-06bd-c14f-42e0-40cc59108d0b term=2 tally=1 +49:42.693 [INFO] TestAgent.server.raft: election won: tally=1 +49:42.693 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23820 [Leader]" +49:42.693 [INFO] TestAgent.server: cluster leadership acquired +49:42.693 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23820 +49:42.693 [INFO] TestAgent.server: New leader elected: payload=test-node +49:42.694 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:42.695 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:42.695 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:42.695 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:42.695 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:42.695 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:42.695 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:42.695 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:42.695 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.044 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:43.044 [INFO] TestAgent: Synced node info +49:43.126 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.314945ms client=127.0.0.1:39167 client_network=udp +49:43.127 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.013892ms client=127.0.0.1:48013 client_network=udp +49:43.129 [DEBUG] TestAgent.dns: request served from client: name=6354dac9-be44-c3a1-ae2c-8e6ff83c56af.query.consul. type=A class=IN latency=833.347µs client=127.0.0.1:42160 client_network=udp +49:43.129 [INFO] TestAgent: Requesting shutdown +49:43.129 [INFO] TestAgent.server: shutting down server +49:43.129 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.129 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.129 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.129 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.129 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.129 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.129 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.129 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.129 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.129 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.129 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:43.129Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.129Z [INFO] agent.router.manager: shutting down +49:43.129 [INFO] TestAgent: consul server down +49:43.129 [INFO] TestAgent: shutdown complete +49:43.129 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23843 network=tcp +49:43.129 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23843 network=udp +49:43.129 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23844 network=tcp +49:43.629 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.629 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{9_9_8_8_6_8_8_5_8_8_-5} -[WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:40.508Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:40.557Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:40.559 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2ef1d913-f24b-5836-c700-33bab391c6b7 Address:127.0.0.1:16520}]" -13:40.559 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16520 [Follower]" leader= -13:40.559 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2ef1d913-f24b-5836-c700-33bab391c6b7.dc1 127.0.0.1 -13:40.560 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2ef1d913-f24b-5836-c700-33bab391c6b7 127.0.0.1 -2021-10-31T23:13:40.560Z [INFO] agent.router: Initializing LAN area manager -13:40.560 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16557 network=udp -13:40.560 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2ef1d913-f24b-5836-c700-33bab391c6b7.dc1 area=wan -13:40.560 [INFO] TestAgent.server: Adding LAN server: server="Node-2ef1d913-f24b-5836-c700-33bab391c6b7 (Addr: tcp/127.0.0.1:16520) (DC: dc1)" -13:40.561 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16557 network=tcp -13:40.561 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16558 network=tcp -13:40.561 [INFO] TestAgent: started state syncer -13:40.561 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16507 network=tcp -13:40.607 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:40.607 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16520 [Candidate]" term=2 -13:40.607 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:40.607 [DEBUG] TestAgent.server.raft: vote granted: from=2ef1d913-f24b-5836-c700-33bab391c6b7 term=2 tally=1 -13:40.607 [INFO] TestAgent.server.raft: election won: tally=1 -13:40.607 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16520 [Leader]" -13:40.607 [INFO] TestAgent.server: cluster leadership acquired -13:40.607 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16520 -13:40.608 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:40.609 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:40.609 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:40.609 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:40.609 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:40.609 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:40.609 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2ef1d913-f24b-5836-c700-33bab391c6b7 -13:40.609 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2ef1d913-f24b-5836-c700-33bab391c6b7 -13:40.609 [INFO] TestAgent.server: New leader elected: payload=Node-2ef1d913-f24b-5836-c700-33bab391c6b7 -13:40.610 [INFO] TestAgent.server: federation state anti-entropy synced -13:40.893 [INFO] TestAgent: Requesting shutdown -13:40.893 [INFO] TestAgent.server: shutting down server -13:40.893 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:40.893 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:40.893 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:40.893 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:40.893 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:40.893 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:40.893 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:40.893 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:40.893 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:40.893 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:40.893 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:40.893 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:40.893Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:40.893Z [INFO] agent.router.manager: shutting down -13:40.902 [INFO] TestAgent: consul server down -13:40.902 [INFO] TestAgent: shutdown complete -13:40.902 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16557 network=tcp -13:40.902 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16557 network=udp -13:40.902 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16558 network=tcp -13:41.409 [INFO] TestAgent: Waiting for endpoints to shut down -13:41.409 [INFO] TestAgent: Endpoints down ---- PASS: TestSessionCreate_NodeChecks (0.95s) +2022-12-04T05:49:42.857Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:42.872Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:42.873 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a0b390f5-795b-932b-5b9b-2b3f55d1c624 Address:127.0.0.1:23813}]" +49:42.873 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23813 [Follower]" leader= +49:42.874 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:42.875 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:42.875Z [INFO] agent.router: Initializing LAN area manager +49:42.875 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23801 network=udp +49:42.875 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:42.875 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23813) (DC: dc1)" +49:42.875 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23801 network=tcp +49:42.876 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23802 network=tcp +49:42.876 [INFO] TestAgent: started state syncer +49:42.876 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23891 network=tcp +49:42.924 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:42.924 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23813 [Candidate]" term=2 +49:42.924 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:42.924 [DEBUG] TestAgent.server.raft: vote granted: from=a0b390f5-795b-932b-5b9b-2b3f55d1c624 term=2 tally=1 +49:42.924 [INFO] TestAgent.server.raft: election won: tally=1 +49:42.924 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23813 [Leader]" +49:42.924 [INFO] TestAgent.server: cluster leadership acquired +49:42.924 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23813 +49:42.924 [INFO] TestAgent.server: New leader elected: payload=test-node +49:42.925 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:42.926 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:42.926 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:42.926 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:42.926 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:42.926 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:42.926 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:42.926 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:42.927 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.165 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.362321ms client=127.0.0.1:39796 client_network=udp +49:43.169 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=961.769µs client=127.0.0.1:47574 client_network=udp +49:43.171 [DEBUG] TestAgent.dns: request served from client: name=c9d96494-8c93-6280-5b1b-aa0cf6981ef4.query.consul. type=ANY class=IN latency=1.209835ms client=127.0.0.1:35031 client_network=udp +49:43.171 [INFO] TestAgent: Requesting shutdown +49:43.171 [INFO] TestAgent.server: shutting down server +49:43.171 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.171 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.171 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.171 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.171 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.171 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.171 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.171 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:43.171 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:43.171 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:49:43.172Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.172Z [INFO] agent.router.manager: shutting down +49:43.172 [INFO] TestAgent: consul server down +49:43.172 [INFO] TestAgent: shutdown complete +49:43.172 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23801 network=tcp +49:43.172 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23801 network=udp +49:43.172 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23802 network=tcp +49:43.672 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.672 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{8_8_8_8_6_8_8_5_8_8_-5} -2021-10-31T23:13:40.586Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:40.617Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:40.627 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:db5311ef-2d07-2fd4-f8cd-da25af50e4a1 Address:127.0.0.1:16261}]" -13:40.628 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16261 [Follower]" leader= -13:40.628 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-db5311ef-2d07-2fd4-f8cd-da25af50e4a1.dc1 127.0.0.1 -13:40.629 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-db5311ef-2d07-2fd4-f8cd-da25af50e4a1 127.0.0.1 -2021-10-31T23:13:40.629Z [INFO] agent.router: Initializing LAN area manager -13:40.629 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16550 network=udp -13:40.629 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-db5311ef-2d07-2fd4-f8cd-da25af50e4a1.dc1 area=wan -13:40.629 [INFO] TestAgent.server: Adding LAN server: server="Node-db5311ef-2d07-2fd4-f8cd-da25af50e4a1 (Addr: tcp/127.0.0.1:16261) (DC: dc1)" -13:40.629 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16550 network=tcp -13:40.638 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16551 network=tcp -13:40.638 [INFO] TestAgent: started state syncer -13:40.638 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16290 network=tcp -13:40.689 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:40.689 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16261 [Candidate]" term=2 -13:40.689 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:40.689 [DEBUG] TestAgent.server.raft: vote granted: from=db5311ef-2d07-2fd4-f8cd-da25af50e4a1 term=2 tally=1 -13:40.689 [INFO] TestAgent.server.raft: election won: tally=1 -13:40.689 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16261 [Leader]" -13:40.689 [INFO] TestAgent.server: cluster leadership acquired -13:40.690 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16261 -13:40.691 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:40.691 [INFO] TestAgent.server: New leader elected: payload=Node-db5311ef-2d07-2fd4-f8cd-da25af50e4a1 -13:40.692 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:40.692 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:40.692 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:40.692 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:40.692 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:40.692 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-db5311ef-2d07-2fd4-f8cd-da25af50e4a1 -13:40.692 [INFO] TestAgent.server: member joined, marking health alive: member=Node-db5311ef-2d07-2fd4-f8cd-da25af50e4a1 -13:40.693 [INFO] TestAgent.server: federation state anti-entropy synced -13:40.850 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:40.850 [INFO] TestAgent: Synced node info -13:40.850 [DEBUG] TestAgent: Node info in sync -13:40.974 [DEBUG] TestAgent.dns: request served from client: name=foo.bar.node.consul. type=ANY class=IN latency=178.436µs client=127.0.0.1:43748 client_network=udp -13:40.974 [INFO] TestAgent: Requesting shutdown -13:40.974 [INFO] TestAgent.server: shutting down server -13:40.974 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:40.974 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:40.974 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:40.974 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:40.974 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:40.974 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:40.974 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:40.974 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:40.974 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:40.974 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:40.974 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -2021-10-31T23:13:40.974Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:40.974Z [INFO] agent.router.manager: shutting down -13:40.974 [INFO] TestAgent: consul server down -13:40.974 [INFO] TestAgent: shutdown complete -13:40.974 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16550 network=tcp -13:40.974 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16550 network=udp -13:40.974 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16551 network=tcp -13:41.485 [INFO] TestAgent: Waiting for endpoints to shut down -13:41.485 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_NodeLookup_PeriodName (0.93s) +2022-12-04T05:49:43.128Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.141Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.142 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0a17c402-67b2-293a-09a1-4ff4226e9096 Address:127.0.0.1:23890}]" +49:43.142 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23890 [Follower]" leader= +49:43.143 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.143 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.143Z [INFO] agent.router: Initializing LAN area manager +49:43.143 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23890) (DC: dc1)" +49:43.143 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.144 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23871 network=udp +49:43.144 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23871 network=tcp +49:43.144 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23872 network=tcp +49:43.144 [INFO] TestAgent: started state syncer +49:43.144 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23877 network=tcp +49:43.189 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:43.189 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23890 [Candidate]" term=2 +49:43.189 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:43.189 [DEBUG] TestAgent.server.raft: vote granted: from=0a17c402-67b2-293a-09a1-4ff4226e9096 term=2 tally=1 +49:43.189 [INFO] TestAgent.server.raft: election won: tally=1 +49:43.189 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23890 [Leader]" +49:43.189 [INFO] TestAgent.server: cluster leadership acquired +49:43.189 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23890 +49:43.189 [INFO] TestAgent.server: New leader elected: payload=test-node +49:43.190 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:43.191 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:43.191 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:43.191 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:43.191 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:43.191 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:43.191 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:43.191 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:43.192 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.237 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.135328ms client=127.0.0.1:43704 client_network=udp +49:43.238 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=971.678µs client=127.0.0.1:50161 client_network=udp +49:43.239 [DEBUG] TestAgent.dns: request served from client: name=9f081403-a95f-ecf1-f88c-4f5d8ded918f.query.consul. type=AAAA class=IN latency=711.964µs client=127.0.0.1:44008 client_network=udp +49:43.239 [INFO] TestAgent: Requesting shutdown +49:43.239 [INFO] TestAgent.server: shutting down server +49:43.239 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.239 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.239 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.239 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.239 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.239 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.239 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.239 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:43.239 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.239 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.239 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.239 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:43.240Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.240Z [INFO] agent.router.manager: shutting down +49:43.240 [INFO] TestAgent: consul server down +49:43.240 [INFO] TestAgent: shutdown complete +49:43.240 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23871 network=tcp +49:43.240 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23871 network=udp +49:43.240 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23872 network=tcp +49:43.740 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.740 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{8_8_8_8_6_8_8_5_8_8_-5} -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:40.711Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:40.746Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:40.747 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:871c396e-67db-b7df-a655-046ab4c7dc83 Address:127.0.0.1:16611}]" -13:40.748 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:40.748 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:40.748Z [INFO] agent.router: Initializing LAN area manager -13:40.748 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16207 network=udp -13:40.748 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16611 [Follower]" leader= -13:40.749 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16611) (DC: dc1)" -13:40.749 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:40.749 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16207 network=tcp -13:40.749 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16208 network=tcp -13:40.749 [INFO] TestAgent: started state syncer -13:40.749 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16632 network=tcp -13:40.794 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:40.794 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16611 [Candidate]" term=2 -13:40.794 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:40.794 [DEBUG] TestAgent.server.raft: vote granted: from=871c396e-67db-b7df-a655-046ab4c7dc83 term=2 tally=1 -13:40.794 [INFO] TestAgent.server.raft: election won: tally=1 -13:40.794 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16611 [Leader]" -13:40.794 [INFO] TestAgent.server: cluster leadership acquired -13:40.794 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16611 -13:40.795 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:40.797 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:40.797 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:40.797 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:40.797 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:40.797 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:40.797 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:40.797 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:40.797 [INFO] TestAgent.server: New leader elected: payload=test-node -13:40.806 [INFO] TestAgent.server: federation state anti-entropy synced -13:41.104 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.985066ms client=127.0.0.1:43001 client_network=udp -13:41.150 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=45.408768ms client=127.0.0.1:40255 client_network=udp -13:41.154 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:41.154 [INFO] TestAgent: Synced node info -13:41.154 [DEBUG] TestAgent: Node info in sync -13:41.154 [DEBUG] TestAgent.dns: request served from client: name=1408babb-ddc8-a844-cfd2-66e2b3d7e932.query.consul. type=A class=IN latency=1.435287ms client=127.0.0.1:48186 client_network=udp -13:41.155 [INFO] TestAgent: Requesting shutdown -13:41.155 [INFO] TestAgent.server: shutting down server -13:41.155 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:41.155 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:41.155 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:41.155 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:41.155 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:41.155 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:41.155 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:41.155 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:41.155 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:41.155 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:41.155 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:41.155Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:41.155Z [INFO] agent.router.manager: shutting down -13:41.155 [INFO] TestAgent: consul server down -13:41.155 [INFO] TestAgent: shutdown complete -13:41.155 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16207 network=tcp -13:41.155 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16207 network=udp -13:41.155 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16208 network=tcp -13:41.657 [INFO] TestAgent: Waiting for endpoints to shut down -13:41.657 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{8_8_8_8_6_8_8_5_8_8_-5} -2021-10-31T23:13:40.744Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:40.796Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:40.839 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3b715268-d266-e760-7c03-4d9a1700c930 Address:127.0.0.1:16576}]" -13:40.839 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:40.840 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:40.840Z [INFO] agent.router: Initializing LAN area manager -13:40.840 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16647 network=udp -13:40.840 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16576 [Follower]" leader= -13:40.841 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16576) (DC: dc1)" -13:40.841 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:40.841 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16647 network=tcp -13:40.842 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16648 network=tcp -13:40.842 [INFO] TestAgent: started state syncer -13:40.842 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16605 network=tcp -13:40.894 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:40.894 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16576 [Candidate]" term=2 -13:40.894 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:40.894 [DEBUG] TestAgent.server.raft: vote granted: from=3b715268-d266-e760-7c03-4d9a1700c930 term=2 tally=1 -13:40.894 [INFO] TestAgent.server.raft: election won: tally=1 -13:40.894 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16576 [Leader]" -13:40.904 [INFO] TestAgent.server: cluster leadership acquired -13:40.904 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16576 -13:40.905 [INFO] TestAgent.server: New leader elected: payload=test-node -13:40.906 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:40.908 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:40.908 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:40.908 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:40.908 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:40.908 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:40.908 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:40.908 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:40.909 [INFO] TestAgent.server: federation state anti-entropy synced -13:41.187 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.409671ms client=127.0.0.1:52004 client_network=udp -13:41.189 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.147572ms client=127.0.0.1:34389 client_network=udp -13:41.190 [DEBUG] TestAgent.dns: request served from client: name=95a5c36e-55f2-28ae-269f-536acfd2be62.query.consul. type=AAAA class=IN latency=1.182296ms client=127.0.0.1:53894 client_network=udp -13:41.190 [INFO] TestAgent: Requesting shutdown -13:41.190 [INFO] TestAgent.server: shutting down server -13:41.190 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:41.190 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:41.190 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:41.190 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:41.190 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:41.190 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:41.190 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:41.190 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:41.190 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:41.190 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:41.190 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:41.190 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:41.191Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:41.191Z [INFO] agent.router.manager: shutting down -13:41.191 [INFO] TestAgent: consul server down -13:41.191 [INFO] TestAgent: shutdown complete -13:41.191 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16647 network=tcp -13:41.191 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16647 network=udp -13:41.191 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16648 network=tcp -13:41.694 [INFO] TestAgent: Waiting for endpoints to shut down -13:41.694 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{7_7_7_7_6_7_7_5_7_7_-5} -2021-10-31T23:13:40.699Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:40.724Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:40.731 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6d435426-a3bf-9e4c-67b4-a2a50362af7a Address:127.0.0.1:16233}]" -13:40.731 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:40.732 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:40.732Z [INFO] agent.router: Initializing LAN area manager -13:40.732 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16291 network=udp -13:40.732 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16233 [Follower]" leader= -13:40.732 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16233) (DC: dc1)" -13:40.733 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:40.733 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16291 network=tcp -13:40.737 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16292 network=tcp -13:40.737 [INFO] TestAgent: started state syncer -13:40.737 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16206 network=tcp -13:40.790 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:40.790 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16233 [Candidate]" term=2 -13:40.790 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:40.790 [DEBUG] TestAgent.server.raft: vote granted: from=6d435426-a3bf-9e4c-67b4-a2a50362af7a term=2 tally=1 -13:40.790 [INFO] TestAgent.server.raft: election won: tally=1 -13:40.790 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16233 [Leader]" -13:40.790 [INFO] TestAgent.server: cluster leadership acquired -13:40.790 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16233 -13:40.791 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:40.793 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:40.793 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:40.793 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:40.793 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:40.793 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:40.793 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:40.793 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:40.793 [INFO] TestAgent.server: New leader elected: payload=test-node -13:40.798 [INFO] TestAgent.server: federation state anti-entropy synced -13:40.842 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:40.842 [INFO] TestAgent: Synced node info -13:40.842 [DEBUG] TestAgent: Node info in sync -13:41.242 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=45.949227ms client=127.0.0.1:36720 client_network=udp -13:41.260 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.793625ms client=127.0.0.1:50757 client_network=udp -13:41.271 [DEBUG] TestAgent.dns: request served from client: name=25335917-90e8-6917-30d4-f3ef02bc8ab2.query.consul. type=A class=IN latency=10.610811ms client=127.0.0.1:56886 client_network=udp -13:41.271 [INFO] TestAgent: Requesting shutdown -13:41.271 [INFO] TestAgent.server: shutting down server -13:41.271 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:41.271 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:41.271 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:41.271 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:41.271 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:41.271 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:41.271 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:41.271 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:41.271 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:41.271 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:41.271 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:41.272Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:41.272Z [INFO] agent.router.manager: shutting down -13:41.272 [INFO] TestAgent: consul server down -13:41.272 [INFO] TestAgent: shutdown complete -13:41.272 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16291 network=tcp -13:41.272 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16291 network=udp -13:41.272 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16292 network=tcp -13:41.772 [INFO] TestAgent: Waiting for endpoints to shut down -13:41.772 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:43.095Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.130Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.132 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:be244133-82bd-4b0e-accf-b4251397a58b Address:127.0.0.1:23883}]" +49:43.132 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.133 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.133Z [INFO] agent.router: Initializing LAN area manager +49:43.133 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23787 network=udp +49:43.133 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23883 [Follower]" leader= +49:43.134 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23883) (DC: dc1)" +49:43.134 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.134 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23787 network=tcp +49:43.134 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23788 network=tcp +49:43.134 [INFO] TestAgent: started state syncer +49:43.134 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23870 network=tcp +49:43.176 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:43.176 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23883 [Candidate]" term=2 +49:43.177 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:43.177 [DEBUG] TestAgent.server.raft: vote granted: from=be244133-82bd-4b0e-accf-b4251397a58b term=2 tally=1 +49:43.177 [INFO] TestAgent.server.raft: election won: tally=1 +49:43.177 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23883 [Leader]" +49:43.177 [INFO] TestAgent.server: cluster leadership acquired +49:43.177 [INFO] TestAgent.server: New leader elected: payload=test-node +49:43.177 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23883 +49:43.178 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:43.179 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:43.179 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:43.179 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:43.179 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:43.179 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:43.179 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:43.179 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:43.180 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.258 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.098603ms client=127.0.0.1:60085 client_network=udp +49:43.259 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.332689ms client=127.0.0.1:52996 client_network=udp +49:43.261 [DEBUG] TestAgent.dns: request served from client: name=db715a49-4bca-746b-d826-059f6638b7b1.query.consul. type=ANY class=IN latency=1.202694ms client=127.0.0.1:59338 client_network=udp +49:43.261 [INFO] TestAgent: Requesting shutdown +49:43.261 [INFO] TestAgent.server: shutting down server +49:43.261 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.261 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.261 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.261 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.261 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:43.261 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.261 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.261 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.261 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.261 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.261 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.261 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:43.261Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.261Z [INFO] agent.router.manager: shutting down +49:43.261 [INFO] TestAgent: consul server down +49:43.261 [INFO] TestAgent: shutdown complete +49:43.261 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23787 network=tcp +49:43.261 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23787 network=udp +49:43.261 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23788 network=tcp +49:43.761 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.761 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{7_7_7_7_6_7_7_5_7_7_-5} -2021-10-31T23:13:40.729Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:40.800Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:40.810 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8667e676-aae0-cb92-493d-23507a482459 Address:127.0.0.1:16659}]" -13:40.811 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:40.811 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16659 [Follower]" leader= -13:40.812 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:40.812Z [INFO] agent.router: Initializing LAN area manager -13:40.812 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16633 network=udp -13:40.812 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16659) (DC: dc1)" -13:40.812 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:40.812 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16633 network=tcp -13:40.813 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16634 network=tcp -13:40.813 [INFO] TestAgent: started state syncer -13:40.813 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16646 network=tcp -13:40.870 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:40.870 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16659 [Candidate]" term=2 -13:40.870 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:40.870 [DEBUG] TestAgent.server.raft: vote granted: from=8667e676-aae0-cb92-493d-23507a482459 term=2 tally=1 -13:40.870 [INFO] TestAgent.server.raft: election won: tally=1 -13:40.870 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16659 [Leader]" -13:40.870 [INFO] TestAgent.server: cluster leadership acquired -13:40.870 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16659 -13:40.871 [INFO] TestAgent.server: New leader elected: payload=test-node -13:40.871 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:40.894 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:40.894 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:40.894 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:40.894 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:40.894 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:40.894 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:40.894 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:40.910 [INFO] TestAgent.server: federation state anti-entropy synced -13:40.946 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:40.946 [INFO] TestAgent: Synced node info -13:40.946 [DEBUG] TestAgent: Node info in sync -13:41.287 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.216544ms client=127.0.0.1:42099 client_network=udp -13:41.289 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.739213ms client=127.0.0.1:50962 client_network=udp -13:41.296 [DEBUG] TestAgent.dns: request served from client: name=8fcf5edc-2711-87bf-f4ea-df0b2b622a1a.query.consul. type=ANY class=IN latency=2.425639ms client=127.0.0.1:45800 client_network=udp -13:41.296 [INFO] TestAgent: Requesting shutdown -13:41.297 [INFO] TestAgent.server: shutting down server -13:41.297 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:41.297 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:41.297 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:41.297 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:41.297 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:41.297 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:41.297 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:41.297 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:41.297 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:41.297 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:41.297 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:41.297Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:41.297Z [INFO] agent.router.manager: shutting down -13:41.306 [INFO] TestAgent: consul server down -13:41.306 [INFO] TestAgent: shutdown complete -13:41.306 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16633 network=tcp -13:41.306 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16633 network=udp -13:41.306 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16634 network=tcp -13:41.814 [INFO] TestAgent: Waiting for endpoints to shut down -13:41.814 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:43.092Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.119Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.122 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8f0d1e38-0afc-df08-9715-f11f0b622469 Address:127.0.0.1:23785}]" +49:43.122 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23785 [Follower]" leader= +49:43.125 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.125 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.125Z [INFO] agent.router: Initializing LAN area manager +49:43.126 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23785) (DC: dc1)" +49:43.126 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.126 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23885 network=udp +49:43.126 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23885 network=tcp +49:43.126 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23886 network=tcp +49:43.126 [INFO] TestAgent: started state syncer +49:43.126 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23786 network=tcp +49:43.189 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:43.189 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23785 [Candidate]" term=2 +49:43.189 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:43.189 [DEBUG] TestAgent.server.raft: vote granted: from=8f0d1e38-0afc-df08-9715-f11f0b622469 term=2 tally=1 +49:43.189 [INFO] TestAgent.server.raft: election won: tally=1 +49:43.189 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23785 [Leader]" +49:43.189 [INFO] TestAgent.server: cluster leadership acquired +49:43.189 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23785 +49:43.189 [INFO] TestAgent.server: New leader elected: payload=test-node +49:43.190 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:43.191 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:43.191 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:43.191 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:43.191 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:43.191 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:43.191 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:43.191 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:43.192 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.279 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=934.544µs client=127.0.0.1:39641 client_network=udp +49:43.280 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=661.023µs client=127.0.0.1:52369 client_network=udp +49:43.281 [DEBUG] TestAgent.dns: request served from client: name=a20ae23e-325a-5195-a57d-7b961feea841.query.consul. type=AAAA class=IN latency=730.537µs client=127.0.0.1:55472 client_network=udp +49:43.281 [INFO] TestAgent: Requesting shutdown +49:43.281 [INFO] TestAgent.server: shutting down server +49:43.281 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.281 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.281 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.281 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.281 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.281 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.281 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.281 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:43.281 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.281 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.281 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.281 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:43.281Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.281Z [INFO] agent.router.manager: shutting down +49:43.281 [INFO] TestAgent: consul server down +49:43.281 [INFO] TestAgent: shutdown complete +49:43.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23885 network=tcp +49:43.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23885 network=udp +49:43.281 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23886 network=tcp +49:43.781 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.781 [INFO] TestAgent: Endpoints down +=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{7_7_7_7_6_7_7_5_7_7_-5} +[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:43.264Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.273Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.275 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8b5a3b82-fbef-8dec-ddfb-7649c04fe9b2 Address:127.0.0.1:23897}]" +49:43.275 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23897 [Follower]" leader= +49:43.275 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.276 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.276Z [INFO] agent.router: Initializing LAN area manager +49:43.276 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.276 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23897) (DC: dc1)" +49:43.276 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23878 network=udp +49:43.276 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23878 network=tcp +49:43.277 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23879 network=tcp +49:43.277 [INFO] TestAgent: started state syncer +49:43.277 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23933 network=tcp +49:43.347 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:43.347 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23897 [Candidate]" term=2 +49:43.347 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:43.347 [DEBUG] TestAgent.server.raft: vote granted: from=8b5a3b82-fbef-8dec-ddfb-7649c04fe9b2 term=2 tally=1 +49:43.347 [INFO] TestAgent.server.raft: election won: tally=1 +49:43.347 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23897 [Leader]" +49:43.347 [INFO] TestAgent.server: cluster leadership acquired +49:43.347 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23897 +49:43.347 [INFO] TestAgent.server: New leader elected: payload=test-node +49:43.348 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:43.350 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:43.350 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:43.350 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:43.350 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:43.350 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:43.350 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:43.350 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:43.350 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.413 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.364197ms client=127.0.0.1:58017 client_network=udp +49:43.415 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.138898ms client=127.0.0.1:55135 client_network=udp +49:43.418 [DEBUG] TestAgent.dns: request served from client: name=0bd693d8-dc57-ab51-1637-59253bb3564d.query.consul. type=A class=IN latency=1.224024ms client=127.0.0.1:55163 client_network=udp +49:43.418 [INFO] TestAgent: Requesting shutdown +49:43.418 [INFO] TestAgent.server: shutting down server +49:43.418 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.418 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.418 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.418 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.418 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.418 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:43.418 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.418 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.418 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:43.418 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:49:43.419Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.419Z [INFO] agent.router.manager: shutting down +49:43.419 [INFO] TestAgent: consul server down +49:43.419 [INFO] TestAgent: shutdown complete +49:43.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23878 network=tcp +49:43.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23878 network=udp +49:43.419 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23879 network=tcp +49:43.919 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.919 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{7_7_7_7_6_7_7_5_7_7_-5} -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:40.965Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:41.046Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:41.049 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b91640d0-b670-111e-ebfe-a00a4b7b17b5 Address:127.0.0.1:16142}]" -13:41.050 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16142 [Follower]" leader= -13:41.058 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:41.059 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:41.059Z [INFO] agent.router: Initializing LAN area manager -13:41.059 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16142) (DC: dc1)" -13:41.059 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:41.060 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16144 network=tcp -13:41.061 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16144 network=udp -13:41.074 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16145 network=tcp -13:41.074 [INFO] TestAgent: started state syncer -13:41.074 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16220 network=tcp -13:41.095 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:41.095 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16142 [Candidate]" term=2 -13:41.108 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:41.108 [DEBUG] TestAgent.server.raft: vote granted: from=b91640d0-b670-111e-ebfe-a00a4b7b17b5 term=2 tally=1 -13:41.108 [INFO] TestAgent.server.raft: election won: tally=1 -13:41.108 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16142 [Leader]" -13:41.108 [INFO] TestAgent.server: cluster leadership acquired -13:41.146 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16142 -13:41.146 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:41.147 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:41.147 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:41.147 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:41.148 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:41.148 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:41.148 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:41.148 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:41.148 [INFO] TestAgent.server: New leader elected: payload=test-node -13:41.148 [INFO] TestAgent.server: federation state anti-entropy synced -13:41.382 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:41.382 [INFO] TestAgent: Synced node info -13:41.387 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.485152ms client=127.0.0.1:60509 client_network=udp -13:41.392 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.206782ms client=127.0.0.1:57089 client_network=udp -13:41.393 [DEBUG] TestAgent.dns: request served from client: name=dfb24578-3b64-a118-82b9-59e2ecb50f86.query.consul. type=AAAA class=IN latency=1.107375ms client=127.0.0.1:42855 client_network=udp -13:41.393 [INFO] TestAgent: Requesting shutdown -13:41.393 [INFO] TestAgent.server: shutting down server -13:41.393 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:41.393 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:41.393 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:41.393 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:41.393 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:41.394 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:41.394 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:41.394 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:41.394 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:41.394 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:41.394 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:41.394Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:41.394Z [INFO] agent.router.manager: shutting down -13:41.394 [INFO] TestAgent: consul server down -13:41.394 [INFO] TestAgent: shutdown complete -13:41.394 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16144 network=tcp -13:41.394 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16144 network=udp -13:41.394 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16145 network=tcp -13:41.897 [INFO] TestAgent: Waiting for endpoints to shut down -13:41.897 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:43.009Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.020Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.021 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:939a7803-dc7b-284f-fe76-57ccb4cc2472 Address:127.0.0.1:23862}]" +49:43.021 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23862 [Follower]" leader= +49:43.022 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.022 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.022Z [INFO] agent.router: Initializing LAN area manager +49:43.022 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23862) (DC: dc1)" +49:43.022 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.022 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23892 network=udp +49:43.022 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23892 network=tcp +49:43.023 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23893 network=tcp +49:43.023 [INFO] TestAgent: started state syncer +49:43.023 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23884 network=tcp +49:43.065 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:43.065 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23862 [Candidate]" term=2 +49:43.065 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:43.065 [DEBUG] TestAgent.server.raft: vote granted: from=939a7803-dc7b-284f-fe76-57ccb4cc2472 term=2 tally=1 +49:43.065 [INFO] TestAgent.server.raft: election won: tally=1 +49:43.065 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23862 [Leader]" +49:43.066 [INFO] TestAgent.server: cluster leadership acquired +49:43.066 [INFO] TestAgent.server: New leader elected: payload=test-node +49:43.066 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23862 +49:43.066 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:43.067 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:43.067 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:43.067 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:43.067 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:43.067 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:43.068 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:43.068 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:43.068 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.283 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:43.283 [INFO] TestAgent: Synced node info +49:43.283 [DEBUG] TestAgent: Node info in sync +49:43.429 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.45996ms client=127.0.0.1:34806 client_network=udp +49:43.433 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.179472ms client=127.0.0.1:39718 client_network=udp +49:43.435 [DEBUG] TestAgent.dns: request served from client: name=91317347-10d3-4cbe-4413-1d4c79273d66.query.consul. type=AAAA class=IN latency=1.010941ms client=127.0.0.1:41256 client_network=udp +49:43.435 [INFO] TestAgent: Requesting shutdown +49:43.435 [INFO] TestAgent.server: shutting down server +49:43.435 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.435 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.435 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.435 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.435 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.435 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.435 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.435 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.436 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.436 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.436 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:43.436Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.436Z [INFO] agent.router.manager: shutting down +49:43.436 [INFO] TestAgent: consul server down +49:43.436 [INFO] TestAgent: shutdown complete +49:43.436 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23892 network=tcp +49:43.436 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23892 network=udp +49:43.436 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23893 network=tcp +49:43.936 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.936 [INFO] TestAgent: Endpoints down === CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{6_6_6_6_6_6_6_5_6_6_-5} -2021-10-31T23:13:41.179Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:41.179Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:41.244Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:13:41.244Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:41.258 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:739f1926-6ea9-8bee-0fb9-c9674f471844 Address:127.0.0.1:16624}]" -13:41.259 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-739f1926-6ea9-8bee-0fb9-c9674f471844.dc1 127.0.0.1 -13:41.259 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-739f1926-6ea9-8bee-0fb9-c9674f471844 127.0.0.1 -2021-10-31T23:13:41.259Z [INFO] agent.router: Initializing LAN area manager -13:41.260 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16696 network=udp -13:41.260 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16624 [Follower]" leader= -13:41.261 [INFO] TestAgent.server: Adding LAN server: server="Node-739f1926-6ea9-8bee-0fb9-c9674f471844 (Addr: tcp/127.0.0.1:16624) (DC: dc1)" -13:41.261 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-739f1926-6ea9-8bee-0fb9-c9674f471844.dc1 area=wan -13:41.262 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16696 network=tcp -13:41.263 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16697 network=tcp -13:41.263 [INFO] TestAgent: started state syncer -13:41.263 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16653 network=tcp -13:41.296 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:41.296 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16624 [Candidate]" term=2 -13:41.297 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:41.297 [DEBUG] TestAgent.server.raft: vote granted: from=739f1926-6ea9-8bee-0fb9-c9674f471844 term=2 tally=1 -13:41.297 [INFO] TestAgent.server.raft: election won: tally=1 -13:41.297 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16624 [Leader]" -13:41.297 [INFO] TestAgent.server: cluster leadership acquired -13:41.297 [INFO] TestAgent.server: initializing acls -13:41.298 [INFO] TestAgent.server: New leader elected: payload=Node-739f1926-6ea9-8bee-0fb9-c9674f471844 -13:41.307 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:41.307 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:41.318 [INFO] TestAgent.server: initializing acls -13:41.318 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:41.323 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:41.323 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:41.337 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:41.337 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:41.337 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:41.338 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:41.338 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -13:41.338 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-739f1926-6ea9-8bee-0fb9-c9674f471844 -13:41.338 [INFO] TestAgent.server: Updating LAN server: server="Node-739f1926-6ea9-8bee-0fb9-c9674f471844 (Addr: tcp/127.0.0.1:16624) (DC: dc1)" -13:41.338 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-739f1926-6ea9-8bee-0fb9-c9674f471844 -13:41.338 [INFO] TestAgent.server: Updating LAN server: server="Node-739f1926-6ea9-8bee-0fb9-c9674f471844 (Addr: tcp/127.0.0.1:16624) (DC: dc1)" -13:41.382 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-739f1926-6ea9-8bee-0fb9-c9674f471844.dc1 -13:41.383 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:41.384 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:41.384 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:41.384 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:41.384 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:41.384 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:41.384 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-739f1926-6ea9-8bee-0fb9-c9674f471844 -13:41.384 [INFO] TestAgent.server: member joined, marking health alive: member=Node-739f1926-6ea9-8bee-0fb9-c9674f471844 -13:41.385 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-739f1926-6ea9-8bee-0fb9-c9674f471844 -13:41.385 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-739f1926-6ea9-8bee-0fb9-c9674f471844 -13:41.387 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-739f1926-6ea9-8bee-0fb9-c9674f471844.dc1 area=wan -13:41.387 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-739f1926-6ea9-8bee-0fb9-c9674f471844.dc1 -13:41.387 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-739f1926-6ea9-8bee-0fb9-c9674f471844.dc1 area=wan -13:41.388 [INFO] TestAgent.server: federation state anti-entropy synced -13:41.398 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-739f1926-6ea9-8bee-0fb9-c9674f471844 -13:41.398 [INFO] TestAgent: Requesting shutdown -13:41.398 [INFO] TestAgent.server: shutting down server -13:41.398 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:41.398 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:41.398 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:41.398 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:41.398 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:41.398 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:41.398 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:41.399 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:41.399 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:41.399 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:41.399 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:41.399 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:41.399 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:41.399 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:41.399 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:41.399 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:41.399Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:41.399Z [INFO] agent.router.manager: shutting down -13:41.399 [INFO] TestAgent: consul server down -13:41.399 [INFO] TestAgent: shutdown complete -13:41.399 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16696 network=tcp -13:41.399 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16696 network=udp -13:41.399 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16697 network=tcp -13:41.900 [INFO] TestAgent: Waiting for endpoints to shut down -13:41.900 [INFO] TestAgent: Endpoints down ---- PASS: TestSnapshot_Options (5.93s) - --- PASS: TestSnapshot_Options/GET (0.94s) - --- PASS: TestSnapshot_Options/GET#01 (1.02s) - --- PASS: TestSnapshot_Options/GET#02 (1.10s) - --- PASS: TestSnapshot_Options/PUT (0.99s) - --- PASS: TestSnapshot_Options/PUT#01 (1.08s) - --- PASS: TestSnapshot_Options/PUT#02 (0.79s) -=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{6_6_6_6_6_6_6_5_6_6_-5} -2021-10-31T23:13:40.906Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:40.971Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:40.987 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:396985ca-52e5-e438-8acc-908c6dd779a8 Address:127.0.0.1:16219}]" -13:40.988 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:40.988 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16219 [Follower]" leader= -13:40.988 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:40.988Z [INFO] agent.router: Initializing LAN area manager -13:40.989 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16219) (DC: dc1)" -13:40.989 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16640 network=udp -13:40.989 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:40.989 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16640 network=tcp -13:40.990 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16641 network=tcp -13:40.990 [INFO] TestAgent: started state syncer -13:40.990 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16143 network=tcp -13:41.038 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:41.038 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16219 [Candidate]" term=2 -13:41.038 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:41.038 [DEBUG] TestAgent.server.raft: vote granted: from=396985ca-52e5-e438-8acc-908c6dd779a8 term=2 tally=1 -13:41.038 [INFO] TestAgent.server.raft: election won: tally=1 -13:41.038 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16219 [Leader]" -13:41.038 [INFO] TestAgent.server: cluster leadership acquired -13:41.038 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16219 -13:41.039 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:41.041 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:41.041 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:41.041 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:41.041 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:41.041 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:41.041 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:41.041 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:41.046 [INFO] TestAgent.server: New leader elected: payload=test-node -13:41.047 [INFO] TestAgent.server: federation state anti-entropy synced -13:41.286 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:41.287 [INFO] TestAgent: Synced node info -13:41.287 [DEBUG] TestAgent: Node info in sync -13:41.427 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.752779ms client=127.0.0.1:39858 client_network=udp -13:41.435 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.491823ms client=127.0.0.1:33594 client_network=udp -13:41.470 [DEBUG] TestAgent.dns: request served from client: name=11373d03-43b3-f158-7b57-fa3ede7c5c0a.query.consul. type=ANY class=IN latency=6.64595ms client=127.0.0.1:40432 client_network=udp -13:41.470 [INFO] TestAgent: Requesting shutdown -13:41.470 [INFO] TestAgent.server: shutting down server -13:41.470 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:41.470 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:41.470 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:41.470 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:41.470 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:41.471 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:41.471 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:41.471 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:41.471 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:41.471 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:41.471 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:41.471Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:41.471Z [INFO] agent.router.manager: shutting down -13:41.471 [INFO] TestAgent: consul server down -13:41.471 [INFO] TestAgent: shutdown complete -13:41.471 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16640 network=tcp -13:41.471 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16640 network=udp -13:41.471 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16641 network=tcp -13:41.972 [INFO] TestAgent: Waiting for endpoints to shut down -13:41.972 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:40.950Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:40.985Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:40.996 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2474c0cd-11f1-c2a8-0505-95d3af083cca Address:127.0.0.1:16645}]" -13:40.997 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:40.997 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16645 [Follower]" leader= -13:41.014 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:41.014Z [INFO] agent.router: Initializing LAN area manager -13:41.015 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16606 network=udp -13:41.016 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16645) (DC: dc1)" -13:41.016 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:41.016 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16606 network=tcp -13:41.017 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16607 network=tcp -13:41.017 [INFO] TestAgent: started state syncer -13:41.017 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16639 network=tcp -13:41.073 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:41.073 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16645 [Candidate]" term=2 -13:41.074 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:41.074 [DEBUG] TestAgent.server.raft: vote granted: from=2474c0cd-11f1-c2a8-0505-95d3af083cca term=2 tally=1 -13:41.074 [INFO] TestAgent.server.raft: election won: tally=1 -13:41.074 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16645 [Leader]" -13:41.074 [INFO] TestAgent.server: cluster leadership acquired -13:41.074 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16645 -13:41.074 [INFO] TestAgent.server: New leader elected: payload=test-node -13:41.075 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:41.086 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:41.086 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:41.086 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:41.086 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:41.086 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:41.086 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:41.086 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:41.087 [INFO] TestAgent.server: federation state anti-entropy synced -13:41.102 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:41.102 [INFO] TestAgent: Synced node info -13:41.435 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.354268ms client=127.0.0.1:51469 client_network=udp -13:41.470 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=6.924971ms client=127.0.0.1:39619 client_network=udp -13:41.472 [DEBUG] TestAgent.dns: request served from client: name=974cfe34-1c6e-4d69-66d8-b7dac9269376.query.consul. type=A class=IN latency=1.652764ms client=127.0.0.1:48087 client_network=udp -13:41.472 [INFO] TestAgent: Requesting shutdown -13:41.472 [INFO] TestAgent.server: shutting down server -13:41.472 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:41.472 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:41.472 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:41.472 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:41.472 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:41.472 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:41.472 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:41.472 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:41.472 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:41.472 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:41.472 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:41.473Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:41.473Z [INFO] agent.router.manager: shutting down -13:41.473 [INFO] TestAgent: consul server down -13:41.473 [INFO] TestAgent: shutdown complete -13:41.473 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16606 network=tcp -13:41.473 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16606 network=udp -13:41.473 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16607 network=tcp -13:41.973 [INFO] TestAgent: Waiting for endpoints to shut down -13:41.974 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{3_3_3_3_3_3_3_3_3_3_3} -=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{5_5_5_5_5_5_5_5_5_5_5} -2021-10-31T23:13:41.147Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:41.181Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:41.182 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5eca838e-565f-0db9-cb51-e7498a31f062 Address:127.0.0.1:16631}]" -13:41.183 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16631 [Follower]" leader= -13:41.183 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:41.184 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:41.184Z [INFO] agent.router: Initializing LAN area manager -13:41.184 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16221 network=udp -13:41.184 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16631) (DC: dc1)" -13:41.184 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:41.184 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16221 network=tcp -13:41.185 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16222 network=tcp -13:41.185 [INFO] TestAgent: started state syncer -13:41.185 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16660 network=tcp -13:41.242 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:41.242 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16631 [Candidate]" term=2 -13:41.242 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:41.242 [DEBUG] TestAgent.server.raft: vote granted: from=5eca838e-565f-0db9-cb51-e7498a31f062 term=2 tally=1 -13:41.242 [INFO] TestAgent.server.raft: election won: tally=1 -13:41.242 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16631 [Leader]" -13:41.243 [INFO] TestAgent.server: cluster leadership acquired -13:41.243 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16631 -13:41.244 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:41.249 [INFO] TestAgent.server: New leader elected: payload=test-node -13:41.255 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:41.255 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:41.255 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:41.255 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:41.255 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:41.255 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:41.255 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:41.256 [INFO] TestAgent.server: federation state anti-entropy synced -13:41.470 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=6.654333ms client=127.0.0.1:52160 client_network=udp -13:41.472 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.959178ms client=127.0.0.1:60224 client_network=udp -13:41.486 [DEBUG] TestAgent.dns: request served from client: name=26d5a860-760f-a5d2-1c5e-1dc81ed3429f.query.consul. type=A class=IN latency=14.127816ms client=127.0.0.1:54299 client_network=udp -13:41.486 [INFO] TestAgent: Requesting shutdown -13:41.486 [INFO] TestAgent.server: shutting down server -13:41.486 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:41.486 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:41.486 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:41.486 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:41.486 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:41.487 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:41.487 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:41.487 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:41.487 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:41.487 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:41.487 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:41.487 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:41.487Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:41.487Z [INFO] agent.router.manager: shutting down -13:41.487 [INFO] TestAgent: consul server down -13:41.487 [INFO] TestAgent: shutdown complete -13:41.487 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16221 network=tcp -13:41.487 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16221 network=udp -13:41.487 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16222 network=tcp -13:41.989 [INFO] TestAgent: Waiting for endpoints to shut down -13:41.990 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{5_5_5_5_5_5_5_5_5_5_5} -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=255)_compressed=true#01 -[WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:41.163Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:41.199Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:41.201 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:450b5df2-a8f0-2c67-5a07-1a6e19703ce1 Address:127.0.0.1:16652}]" -13:41.202 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16652 [Follower]" leader= -13:41.202 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:41.203 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:41.203Z [INFO] agent.router: Initializing LAN area manager -13:41.203 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16661 network=udp -13:41.203 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16652) (DC: dc1)" -13:41.203 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:41.203 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16661 network=tcp -13:41.204 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16662 network=tcp -13:41.204 [INFO] TestAgent: started state syncer -13:41.204 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16612 network=tcp -13:41.272 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:41.272 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16652 [Candidate]" term=2 -13:41.272 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:41.272 [DEBUG] TestAgent.server.raft: vote granted: from=450b5df2-a8f0-2c67-5a07-1a6e19703ce1 term=2 tally=1 -13:41.272 [INFO] TestAgent.server.raft: election won: tally=1 -13:41.272 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16652 [Leader]" -13:41.272 [INFO] TestAgent.server: cluster leadership acquired -13:41.273 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16652 -13:41.277 [INFO] TestAgent.server: New leader elected: payload=test-node -13:41.280 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:41.292 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:41.292 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:41.292 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:41.292 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:41.292 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:41.292 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:41.292 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:41.293 [INFO] TestAgent.server: federation state anti-entropy synced -13:41.482 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:41.482 [INFO] TestAgent: Synced node info -13:41.630 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.595935ms client=127.0.0.1:40764 client_network=udp -13:41.639 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.732866ms client=127.0.0.1:59415 client_network=udp -13:41.643 [DEBUG] TestAgent.dns: request served from client: name=6c14d4e8-d617-781c-3943-cad271456b40.query.consul. type=ANY class=IN latency=1.449193ms client=127.0.0.1:36933 client_network=udp -13:41.643 [INFO] TestAgent: Requesting shutdown -13:41.643 [INFO] TestAgent.server: shutting down server -13:41.643 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:41.643 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:41.643 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:41.643 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:41.643 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:41.643 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:41.643 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:41.643 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:41.643 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:41.643 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:41.643 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:41.643Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:41.644Z [INFO] agent.router.manager: shutting down -13:41.644 [INFO] TestAgent: consul server down -13:41.644 [INFO] TestAgent: shutdown complete -13:41.644 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16661 network=tcp -13:41.644 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16661 network=udp -13:41.644 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16662 network=tcp -13:42.145 [INFO] TestAgent: Waiting for endpoints to shut down -13:42.145 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{5_5_5_5_5_5_5_5_5_5_5} -[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:41.207Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:41.330Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:41.379 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1005dd97-25f7-a58e-e110-4872bd283608 Address:127.0.0.1:16666}]" -13:41.379 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:41.380 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:41.380Z [INFO] agent.router: Initializing LAN area manager -13:41.380 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16613 network=udp -13:41.388 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16666 [Follower]" leader= -13:41.388 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:41.388 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16666) (DC: dc1)" -13:41.388 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16613 network=tcp -13:41.389 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16614 network=tcp -13:41.389 [INFO] TestAgent: started state syncer -13:41.389 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16695 network=tcp -13:41.424 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:41.424 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16666 [Candidate]" term=2 -13:41.424 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:41.424 [DEBUG] TestAgent.server.raft: vote granted: from=1005dd97-25f7-a58e-e110-4872bd283608 term=2 tally=1 -13:41.424 [INFO] TestAgent.server.raft: election won: tally=1 -13:41.424 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16666 [Leader]" -13:41.424 [INFO] TestAgent.server: cluster leadership acquired -13:41.425 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16666 -13:41.425 [INFO] TestAgent.server: New leader elected: payload=test-node -13:41.456 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:41.458 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:41.458 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:41.458 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:41.458 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:41.459 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:41.459 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:41.459 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:41.460 [INFO] TestAgent.server: federation state anti-entropy synced -13:41.774 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:41.774 [INFO] TestAgent: Synced node info -13:41.774 [DEBUG] TestAgent: Node info in sync -13:41.856 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.621971ms client=127.0.0.1:59430 client_network=udp -13:41.858 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.416977ms client=127.0.0.1:34756 client_network=udp -13:41.867 [DEBUG] TestAgent.dns: request served from client: name=bdb35547-00b1-5f76-9ec7-64620cd825af.query.consul. type=AAAA class=IN latency=1.394565ms client=127.0.0.1:57837 client_network=udp -13:41.870 [INFO] TestAgent: Requesting shutdown -13:41.870 [INFO] TestAgent.server: shutting down server -13:41.870 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:41.870 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:41.870 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:41.870 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:41.871 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:41.871 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:41.871 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:41.871 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:41.871 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:41.871 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:41.871 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:41.871Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:41.871Z [INFO] agent.router.manager: shutting down -13:41.871 [INFO] TestAgent: consul server down -13:41.871 [INFO] TestAgent: shutdown complete -13:41.871 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16613 network=tcp -13:41.871 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16613 network=udp -13:41.871 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16614 network=tcp -13:42.373 [INFO] TestAgent: Waiting for endpoints to shut down -13:42.373 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{4_4_4_4_4_4_4_4_4_4_4} -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=255)_compressed=false#01 -2021-10-31T23:13:41.459Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:41.521Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:41.524 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bd8ad926-26a3-2f13-addd-408bf2dac100 Address:127.0.0.1:16715}]" -13:41.525 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:41.525 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16715 [Follower]" leader= -13:41.525 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:41.526Z [INFO] agent.router: Initializing LAN area manager -13:41.526 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16715) (DC: dc1)" -13:41.526 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16626 network=udp -13:41.526 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:41.526 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16626 network=tcp -13:41.528 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16627 network=tcp -13:41.528 [INFO] TestAgent: started state syncer -13:41.528 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16681 network=tcp -13:41.585 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:41.585 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16715 [Candidate]" term=2 -13:41.586 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:41.586 [DEBUG] TestAgent.server.raft: vote granted: from=bd8ad926-26a3-2f13-addd-408bf2dac100 term=2 tally=1 -13:41.586 [INFO] TestAgent.server.raft: election won: tally=1 -13:41.586 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16715 [Leader]" -13:41.586 [INFO] TestAgent.server: cluster leadership acquired -13:41.586 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16715 -13:41.587 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:41.588 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:41.588 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:41.588 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:41.588 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:41.588 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:41.588 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:41.588 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:41.589 [INFO] TestAgent.server: New leader elected: payload=test-node -13:41.589 [INFO] TestAgent.server: federation state anti-entropy synced -13:41.922 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.936208ms client=127.0.0.1:37479 client_network=udp -13:41.924 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.940644ms client=127.0.0.1:56522 client_network=udp -13:41.939 [DEBUG] TestAgent.dns: request served from client: name=8f569438-1e39-dce0-1f0d-99a25e863fb8.query.consul. type=ANY class=IN latency=14.475972ms client=127.0.0.1:47420 client_network=udp -13:41.939 [INFO] TestAgent: Requesting shutdown -13:41.939 [INFO] TestAgent.server: shutting down server -13:41.939 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:41.939 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:41.939 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:41.939 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:41.939 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:41.939 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:41.939 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:41.939 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:41.939 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:41.939 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:41.939 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:41.939 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:41.939Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:41.939Z [INFO] agent.router.manager: shutting down -13:41.940 [INFO] TestAgent: consul server down -13:41.940 [INFO] TestAgent: shutdown complete -13:41.940 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16626 network=tcp -13:41.940 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16626 network=udp -13:41.940 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16627 network=tcp -13:42.441 [INFO] TestAgent: Waiting for endpoints to shut down -13:42.441 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{4_4_4_4_4_4_4_4_4_4_4} -2021-10-31T23:13:41.572Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:41.607Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:41.609 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:074a35a5-ad37-d4af-81c5-c45ea4899bc5 Address:127.0.0.1:16701}]" -13:41.610 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16701 [Follower]" leader= -13:41.618 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:41.618 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:41.618Z [INFO] agent.router: Initializing LAN area manager -13:41.618 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16682 network=udp -13:41.619 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16701) (DC: dc1)" -13:41.619 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:41.619 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16682 network=tcp -13:41.619 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16683 network=tcp -13:41.619 [INFO] TestAgent: started state syncer -13:41.619 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16248 network=tcp -13:41.657 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:41.657 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16701 [Candidate]" term=2 -13:41.658 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:41.658 [DEBUG] TestAgent.server.raft: vote granted: from=074a35a5-ad37-d4af-81c5-c45ea4899bc5 term=2 tally=1 -13:41.658 [INFO] TestAgent.server.raft: election won: tally=1 -13:41.658 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16701 [Leader]" -13:41.658 [INFO] TestAgent.server: cluster leadership acquired -13:41.658 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16701 -13:41.659 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:41.660 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:41.660 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:41.660 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:41.660 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:41.660 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:41.660 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:41.660 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:41.661 [INFO] TestAgent.server: New leader elected: payload=test-node -13:41.661 [INFO] TestAgent.server: federation state anti-entropy synced -13:41.926 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=9.919483ms client=127.0.0.1:44831 client_network=udp -13:41.929 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.934266ms client=127.0.0.1:58835 client_network=udp -13:41.942 [DEBUG] TestAgent.dns: request served from client: name=af8d2f98-7500-a59f-993a-e746ea96fd8a.query.consul. type=AAAA class=IN latency=12.919898ms client=127.0.0.1:52834 client_network=udp -13:41.942 [INFO] TestAgent: Requesting shutdown -13:41.942 [INFO] TestAgent.server: shutting down server -13:41.942 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:41.942 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:41.942 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:41.942 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:41.942 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:41.943 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:41.943 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:41.943 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:41.943 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:41.943 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:41.943 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:41.943 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:41.943Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:41.943Z [INFO] agent.router.manager: shutting down -13:41.943 [INFO] TestAgent: consul server down -13:41.943 [INFO] TestAgent: shutdown complete -13:41.943 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16682 network=tcp -13:41.943 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16682 network=udp -13:41.943 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16683 network=tcp -13:42.446 [INFO] TestAgent: Waiting for endpoints to shut down -13:42.446 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{4_4_4_4_4_4_4_4_4_4_4} -2021-10-31T23:13:41.410Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:41.464Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:41.483 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b04ddc43-9a03-cf00-88e7-538d4f37aba7 Address:127.0.0.1:16638}]" -13:41.484 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:41.485 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:41.485Z [INFO] agent.router: Initializing LAN area manager -13:41.485 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16654 network=udp -13:41.485 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16638 [Follower]" leader= -13:41.486 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16638) (DC: dc1)" -13:41.486 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:41.487 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16654 network=tcp -13:41.488 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16655 network=tcp -13:41.488 [INFO] TestAgent: started state syncer -13:41.488 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16625 network=tcp -13:41.537 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:41.537 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16638 [Candidate]" term=2 -13:41.538 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:41.538 [DEBUG] TestAgent.server.raft: vote granted: from=b04ddc43-9a03-cf00-88e7-538d4f37aba7 term=2 tally=1 -13:41.538 [INFO] TestAgent.server.raft: election won: tally=1 -13:41.538 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16638 [Leader]" -13:41.538 [INFO] TestAgent.server: cluster leadership acquired -13:41.574 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16638 -13:41.575 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:41.576 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:41.576 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:41.576 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:41.576 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:41.576 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:41.576 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:41.576 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:41.577 [INFO] TestAgent.server: New leader elected: payload=test-node -13:41.577 [INFO] TestAgent.server: federation state anti-entropy synced -13:41.654 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:41.654 [INFO] TestAgent: Synced node info -13:41.654 [DEBUG] TestAgent: Node info in sync -13:41.972 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=2.781374ms client=127.0.0.1:43448 client_network=udp -13:41.981 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.865186ms client=127.0.0.1:36222 client_network=udp -13:41.989 [INFO] TestAgent: Requesting shutdown -13:41.994 [DEBUG] TestAgent.dns: request served from client: name=2e36fb43-935d-ab23-9e2e-d64100ed2fa0.query.consul. type=A class=IN latency=8.146621ms client=127.0.0.1:51991 client_network=udp -13:41.997 [INFO] TestAgent.server: shutting down server -13:41.998 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:41.998 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:41.998 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:41.998 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:41.998 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:41.998 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:41.998 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:42.035 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:42.035 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:42.035 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:42.035 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:42.036Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:42.036Z [INFO] agent.router.manager: shutting down -13:42.036 [INFO] TestAgent: consul server down -13:42.036 [INFO] TestAgent: shutdown complete -13:42.036 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16654 network=tcp -13:42.036 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16654 network=udp -13:42.036 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16655 network=tcp -13:42.537 [INFO] TestAgent: Waiting for endpoints to shut down -13:42.537 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{3_3_3_3_3_3_3_3_3_3_3} -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:41.777Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:41.853Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:41.868 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:10fd213f-4a6a-5edc-98de-c2f2071ec57d Address:127.0.0.1:16163}]" -13:41.868 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:41.869 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:41.869Z [INFO] agent.router: Initializing LAN area manager -13:41.869 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16571 network=udp -13:41.869 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16163 [Follower]" leader= -13:41.871 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:41.871 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16163) (DC: dc1)" -13:41.872 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16571 network=tcp -13:41.872 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16572 network=tcp -13:41.872 [INFO] TestAgent: started state syncer -13:41.872 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16730 network=tcp -13:41.914 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:41.914 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16163 [Candidate]" term=2 -13:41.915 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:41.915 [DEBUG] TestAgent.server.raft: vote granted: from=10fd213f-4a6a-5edc-98de-c2f2071ec57d term=2 tally=1 -13:41.915 [INFO] TestAgent.server.raft: election won: tally=1 -13:41.915 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16163 [Leader]" -13:41.915 [INFO] TestAgent.server: cluster leadership acquired -13:41.915 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16163 -13:41.917 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:41.917 [INFO] TestAgent.server: New leader elected: payload=test-node -13:41.927 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:41.927 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:41.927 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:41.927 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:41.927 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:41.927 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:41.927 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:41.928 [INFO] TestAgent.server: federation state anti-entropy synced -13:42.198 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:42.202 [INFO] TestAgent: Synced node info -13:42.202 [DEBUG] TestAgent: Node info in sync -13:42.238 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=22.321343ms client=127.0.0.1:37831 client_network=udp -13:42.241 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=2.085539ms client=127.0.0.1:48278 client_network=udp -13:42.259 [DEBUG] TestAgent.dns: request served from client: name=90b6d7b2-89a9-2dfb-605f-64147b634d32.query.consul. type=ANY class=IN latency=2.206803ms client=127.0.0.1:58733 client_network=udp -13:42.261 [INFO] TestAgent: Requesting shutdown -13:42.262 [INFO] TestAgent.server: shutting down server -13:42.262 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:42.262 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:42.262 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:42.262 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:42.262 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:42.262 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:42.262 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:42.262 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:42.262 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:42.262Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:42.262Z [INFO] agent.router.manager: shutting down -13:42.262 [INFO] TestAgent: consul server down -13:42.262 [INFO] TestAgent: shutdown complete -13:42.262 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16571 network=tcp -13:42.262 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16571 network=udp -13:42.262 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16572 network=tcp -13:42.262 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:42.262 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:42.763 [INFO] TestAgent: Waiting for endpoints to shut down -13:42.763 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{3_3_3_3_3_3_3_3_3_3_3} -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=true#01 -2021-10-31T23:13:41.840Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:41.912Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:41.931 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1b737147-5f46-3cb9-37a5-c839d3a8a5a7 Address:127.0.0.1:16673}]" -13:41.932 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:41.932 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16673 [Follower]" leader= -13:41.932 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:41.932Z [INFO] agent.router: Initializing LAN area manager -13:41.933 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16673) (DC: dc1)" -13:41.933 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:41.933 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16731 network=udp -13:41.933 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16731 network=tcp -13:41.947 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16732 network=tcp -13:41.947 [INFO] TestAgent: started state syncer -13:41.947 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16674 network=tcp -13:41.973 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:41.973 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16673 [Candidate]" term=2 -13:41.974 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:41.974 [DEBUG] TestAgent.server.raft: vote granted: from=1b737147-5f46-3cb9-37a5-c839d3a8a5a7 term=2 tally=1 -13:41.974 [INFO] TestAgent.server.raft: election won: tally=1 -13:41.974 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16673 [Leader]" -13:41.974 [INFO] TestAgent.server: cluster leadership acquired -13:41.975 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16673 -13:41.975 [INFO] TestAgent.server: New leader elected: payload=test-node -13:41.976 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:41.984 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:41.984 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:41.984 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:41.984 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:41.984 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:41.984 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:41.984 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:42.042 [INFO] TestAgent.server: federation state anti-entropy synced -13:42.229 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:42.235 [INFO] TestAgent: Synced node info -13:42.235 [DEBUG] TestAgent: Node info in sync -13:42.295 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.839076ms client=127.0.0.1:54759 client_network=udp -13:42.305 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=3.611379ms client=127.0.0.1:55301 client_network=udp -13:42.319 [DEBUG] TestAgent.dns: request served from client: name=18ef62bd-8420-0307-3b94-930b3f0a2f97.query.consul. type=AAAA class=IN latency=1.958341ms client=127.0.0.1:43681 client_network=udp -13:42.320 [INFO] TestAgent: Requesting shutdown -13:42.320 [INFO] TestAgent.server: shutting down server -13:42.320 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:42.320 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:42.320 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:42.320 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:42.320 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:42.320 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:42.320 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:42.320 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:42.320 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:42.320 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:42.320 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:42.320Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:42.320Z [INFO] agent.router.manager: shutting down -13:42.320 [INFO] TestAgent: consul server down -13:42.320 [INFO] TestAgent: shutdown complete -13:42.320 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16731 network=tcp -13:42.320 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16731 network=udp -13:42.320 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16732 network=tcp -13:42.821 [INFO] TestAgent: Waiting for endpoints to shut down -13:42.821 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{1_1_1_1_1_1_1_1_1_1_1} -[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:41.699Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:41.826Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:41.828 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d6a86885-98ff-2fd5-950e-aec2a578a1c7 Address:127.0.0.1:16506}]" -13:41.828 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:41.829 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:41.829Z [INFO] agent.router: Initializing LAN area manager -13:41.829 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16506 [Follower]" leader= -13:41.829 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:41.830 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16506) (DC: dc1)" -13:41.830 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16249 network=tcp -13:41.830 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16249 network=udp -13:41.830 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16250 network=tcp -13:41.830 [INFO] TestAgent: started state syncer -13:41.830 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16570 network=tcp -13:41.884 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:41.884 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16506 [Candidate]" term=2 -13:41.884 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:41.884 [DEBUG] TestAgent.server.raft: vote granted: from=d6a86885-98ff-2fd5-950e-aec2a578a1c7 term=2 tally=1 -13:41.884 [INFO] TestAgent.server.raft: election won: tally=1 -13:41.884 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16506 [Leader]" -13:41.885 [INFO] TestAgent.server: cluster leadership acquired -13:41.886 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16506 -13:41.886 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:41.887 [INFO] TestAgent.server: New leader elected: payload=test-node -13:41.888 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:41.888 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:41.888 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:41.888 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:41.888 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:41.888 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:41.888 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:41.889 [INFO] TestAgent.server: federation state anti-entropy synced -13:42.069 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:42.070 [INFO] TestAgent: Synced node info -13:42.070 [DEBUG] TestAgent: Node info in sync -13:42.399 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.352367ms client=127.0.0.1:43897 client_network=udp -13:42.403 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.083244ms client=127.0.0.1:56963 client_network=udp -13:42.404 [DEBUG] TestAgent.dns: request served from client: name=b6b96109-6e2e-8f3f-9321-fd6f15bdde4d.query.consul. type=A class=IN latency=952.885µs client=127.0.0.1:37713 client_network=udp -13:42.404 [INFO] TestAgent: Requesting shutdown -13:42.404 [INFO] TestAgent.server: shutting down server -13:42.404 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:42.404 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:42.404 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:42.404 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:42.404 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:42.404 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:42.404 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:42.404 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:42.404 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:42.404 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:42.404 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:42.404Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:42.404Z [INFO] agent.router.manager: shutting down -13:42.404 [INFO] TestAgent: consul server down -13:42.404 [INFO] TestAgent: shutdown complete -13:42.404 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16249 network=tcp -13:42.404 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16249 network=udp -13:42.405 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16250 network=tcp -13:42.905 [INFO] TestAgent: Waiting for endpoints to shut down -13:42.905 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{2_2_2_2_2_2_2_2_2_2_2} -2021-10-31T23:13:42.025Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:42.083Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:42.085 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b7ed2218-9547-55a2-fe4c-f03bc79bcf32 Address:127.0.0.1:16694}]" -13:42.085 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16694 [Follower]" leader= -13:42.095 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:42.096 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:42.096Z [INFO] agent.router: Initializing LAN area manager -13:42.096 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16717 network=udp -13:42.096 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16694) (DC: dc1)" -13:42.096 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16717 network=tcp -13:42.096 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:42.097 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16718 network=tcp -13:42.097 [INFO] TestAgent: started state syncer -13:42.097 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16667 network=tcp -13:42.137 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:42.137 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16694 [Candidate]" term=2 -13:42.138 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:42.138 [DEBUG] TestAgent.server.raft: vote granted: from=b7ed2218-9547-55a2-fe4c-f03bc79bcf32 term=2 tally=1 -13:42.138 [INFO] TestAgent.server.raft: election won: tally=1 -13:42.138 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16694 [Leader]" -13:42.138 [INFO] TestAgent.server: cluster leadership acquired -13:42.138 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16694 -13:42.139 [INFO] TestAgent.server: New leader elected: payload=test-node -13:42.139 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:42.140 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:42.140 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:42.140 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:42.140 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:42.140 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:42.140 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:42.140 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:42.141 [INFO] TestAgent.server: federation state anti-entropy synced -13:42.426 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:42.426 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.688804ms client=127.0.0.1:53762 client_network=udp -13:42.426 [INFO] TestAgent: Synced node info -13:42.428 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.263859ms client=127.0.0.1:40884 client_network=udp -13:42.435 [DEBUG] TestAgent.dns: request served from client: name=4abcd689-c484-373c-319d-eb6ac20d83cb.query.consul. type=AAAA class=IN latency=1.280634ms client=127.0.0.1:39331 client_network=udp -13:42.435 [INFO] TestAgent: Requesting shutdown -13:42.435 [INFO] TestAgent.server: shutting down server -13:42.435 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:42.435 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:42.435 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:42.435 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:42.435 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:42.435 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:42.435 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:42.435 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:42.435 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:42.435 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:42.435 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:42.435Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:42.435Z [INFO] agent.router.manager: shutting down -13:42.436 [INFO] TestAgent: consul server down -13:42.436 [INFO] TestAgent: shutdown complete -13:42.436 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16717 network=tcp -13:42.436 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16717 network=udp -13:42.436 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16718 network=tcp -13:42.936 [INFO] TestAgent: Waiting for endpoints to shut down -13:42.936 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{2_2_2_2_2_2_2_2_2_2_2} -2021-10-31T23:13:41.841Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:41.874Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:41.886 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b75e6a48-8352-b2d8-4c3e-7957577b3153 Address:127.0.0.1:16254}]" -13:41.887 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:41.887 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:41.887Z [INFO] agent.router: Initializing LAN area manager -13:41.887 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16675 network=udp -13:41.887 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16254 [Follower]" leader= -13:41.888 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16254) (DC: dc1)" -13:41.888 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:41.888 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16675 network=tcp -13:41.889 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16676 network=tcp -13:41.889 [INFO] TestAgent: started state syncer -13:41.889 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16171 network=tcp -13:41.947 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:41.947 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16254 [Candidate]" term=2 -13:41.947 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:41.947 [DEBUG] TestAgent.server.raft: vote granted: from=b75e6a48-8352-b2d8-4c3e-7957577b3153 term=2 tally=1 -13:41.947 [INFO] TestAgent.server.raft: election won: tally=1 -13:41.947 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16254 [Leader]" -13:41.947 [INFO] TestAgent.server: cluster leadership acquired -13:41.948 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16254 -13:41.949 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:41.950 [INFO] TestAgent.server: New leader elected: payload=test-node -13:41.963 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:41.963 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:41.963 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:41.963 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:41.963 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:41.964 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:41.964 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:42.034 [INFO] TestAgent.server: federation state anti-entropy synced -13:42.131 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:42.131 [INFO] TestAgent: Synced node info -13:42.131 [DEBUG] TestAgent: Node info in sync -13:42.241 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:42.241 [DEBUG] TestAgent: Node info in sync -13:42.434 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=9.601469ms client=127.0.0.1:39648 client_network=udp -13:42.436 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.263716ms client=127.0.0.1:44771 client_network=udp -13:42.439 [DEBUG] TestAgent.dns: request served from client: name=0a61a391-b7a8-d3a7-8da5-9d041c7257f8.query.consul. type=A class=IN latency=1.033291ms client=127.0.0.1:54161 client_network=udp -13:42.439 [INFO] TestAgent: Requesting shutdown -13:42.439 [INFO] TestAgent.server: shutting down server -13:42.439 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:42.439 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:42.439 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:42.439 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:42.439 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:42.439 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:42.439 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:42.439 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:42.439 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:42.439 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:42.439 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:42.439Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:42.439Z [INFO] agent.router.manager: shutting down -13:42.439 [INFO] TestAgent: consul server down -13:42.439 [INFO] TestAgent: shutdown complete -13:42.439 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16675 network=tcp -13:42.439 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16675 network=udp -13:42.439 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16676 network=tcp -13:42.941 [INFO] TestAgent: Waiting for endpoints to shut down -13:42.941 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{2_2_2_2_2_2_2_2_2_2_2} -2021-10-31T23:13:42.063Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:42.109Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:42.119 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:96ac47fc-7991-3188-4127-b296e4f57cc0 Address:127.0.0.1:16160}]" -13:42.120 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:42.120 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16160 [Follower]" leader= -13:42.120 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:42.120Z [INFO] agent.router: Initializing LAN area manager -13:42.121 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:42.121 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16970 network=udp -13:42.121 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16160) (DC: dc1)" -13:42.121 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16970 network=tcp -13:42.131 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16268 network=tcp -13:42.131 [INFO] TestAgent: started state syncer -13:42.131 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16161 network=tcp -13:42.161 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:42.162 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16160 [Candidate]" term=2 -13:42.162 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:42.162 [DEBUG] TestAgent.server.raft: vote granted: from=96ac47fc-7991-3188-4127-b296e4f57cc0 term=2 tally=1 -13:42.162 [INFO] TestAgent.server.raft: election won: tally=1 -13:42.162 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16160 [Leader]" -13:42.162 [INFO] TestAgent.server: cluster leadership acquired -13:42.162 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16160 -13:42.163 [INFO] TestAgent.server: New leader elected: payload=test-node -13:42.163 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:42.164 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:42.164 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:42.164 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:42.164 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:42.164 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:42.164 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:42.164 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:42.165 [INFO] TestAgent.server: federation state anti-entropy synced -13:42.428 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:42.434 [INFO] TestAgent: Synced node info -13:42.434 [DEBUG] TestAgent: Node info in sync -13:42.460 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.605887ms client=127.0.0.1:39410 client_network=udp -13:42.463 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.543136ms client=127.0.0.1:34782 client_network=udp -13:42.465 [DEBUG] TestAgent.dns: request served from client: name=64035af3-be35-545e-3e28-7ad3e2b135d6.query.consul. type=ANY class=IN latency=1.433349ms client=127.0.0.1:36266 client_network=udp -13:42.470 [INFO] TestAgent: Requesting shutdown -13:42.470 [INFO] TestAgent.server: shutting down server -13:42.470 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:42.470 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:42.470 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:42.470 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:42.470 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:42.470 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:42.470 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:42.470 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:42.470 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:42.470 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:42.470 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:42.470Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:42.470Z [INFO] agent.router.manager: shutting down -13:42.470 [INFO] TestAgent: consul server down -13:42.470 [INFO] TestAgent: shutdown complete -13:42.470 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16970 network=tcp -13:42.470 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16970 network=udp -13:42.471 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16268 network=tcp -13:42.973 [INFO] TestAgent: Waiting for endpoints to shut down -13:42.973 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/A_lookup_{1_1_1_1_1_1_1_1_1_1_1} -2021-10-31T23:13:42.162Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:42.237Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:42.248 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:101c6816-2e4a-7954-cd22-bf3d4e303e04 Address:127.0.0.1:16924}]" -13:42.249 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:42.249 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16924 [Follower]" leader= -13:42.258 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:42.258Z [INFO] agent.router: Initializing LAN area manager -13:42.259 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16176 network=udp -13:42.259 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16924) (DC: dc1)" -13:42.259 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:42.259 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16176 network=tcp -13:42.260 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16920 network=tcp -13:42.260 [INFO] TestAgent: started state syncer -13:42.260 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16941 network=tcp -13:42.309 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:42.309 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16924 [Candidate]" term=2 -13:42.310 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:42.310 [DEBUG] TestAgent.server.raft: vote granted: from=101c6816-2e4a-7954-cd22-bf3d4e303e04 term=2 tally=1 -13:42.310 [INFO] TestAgent.server.raft: election won: tally=1 -13:42.310 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16924 [Leader]" -13:42.310 [INFO] TestAgent.server: cluster leadership acquired -13:42.310 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16924 -13:42.311 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:42.313 [INFO] TestAgent.server: New leader elected: payload=test-node -13:42.342 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:42.342 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:42.342 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:42.342 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:42.342 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:42.342 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:42.342 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:42.343 [INFO] TestAgent.server: federation state anti-entropy synced -13:42.393 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:42.395 [INFO] TestAgent: Synced node info -13:42.395 [DEBUG] TestAgent: Node info in sync -13:42.475 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.172137ms client=127.0.0.1:60445 client_network=udp -13:42.479 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.276885ms client=127.0.0.1:58536 client_network=udp -13:42.480 [DEBUG] TestAgent.dns: request served from client: name=59333ea7-613d-68b3-3307-9103f13fa889.query.consul. type=A class=IN latency=1.141932ms client=127.0.0.1:49936 client_network=udp -13:42.480 [INFO] TestAgent: Requesting shutdown -13:42.481 [INFO] TestAgent.server: shutting down server -13:42.481 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:42.481 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:42.481 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:42.481 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:42.481 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:42.481 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:42.481 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:42.481 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:42.481 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:42.481 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:42.481 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:42.481Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:42.481Z [INFO] agent.router.manager: shutting down -13:42.481 [INFO] TestAgent: consul server down -13:42.481 [INFO] TestAgent: shutdown complete -13:42.481 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16176 network=tcp -13:42.481 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16176 network=udp -13:42.481 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16920 network=tcp -13:42.981 [INFO] TestAgent: Waiting for endpoints to shut down -13:42.981 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{1_1_1_1_1_1_1_1_1_1_1} -2021-10-31T23:13:42.061Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:42.127Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:42.129 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:676f9401-3b11-1568-9849-e3069578c908 Address:127.0.0.1:16971}]" -13:42.129 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:42.129 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16971 [Follower]" leader= -13:42.142 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:42.142Z [INFO] agent.router: Initializing LAN area manager -13:42.142 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16668 network=udp -13:42.142 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16971) (DC: dc1)" -13:42.142 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:42.143 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16668 network=tcp -13:42.143 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16669 network=tcp -13:42.143 [INFO] TestAgent: started state syncer -13:42.143 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16919 network=tcp -13:42.194 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:42.194 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16971 [Candidate]" term=2 -13:42.194 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:42.194 [DEBUG] TestAgent.server.raft: vote granted: from=676f9401-3b11-1568-9849-e3069578c908 term=2 tally=1 -13:42.194 [INFO] TestAgent.server.raft: election won: tally=1 -13:42.194 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16971 [Leader]" -13:42.195 [INFO] TestAgent.server: cluster leadership acquired -13:42.195 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16971 -13:42.196 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:42.197 [INFO] TestAgent.server: New leader elected: payload=test-node -13:42.198 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:42.198 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:42.198 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:42.198 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:42.198 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:42.198 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:42.199 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:42.200 [INFO] TestAgent.server: federation state anti-entropy synced -13:42.216 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:42.217 [INFO] TestAgent: Synced node info -13:42.528 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.222841ms client=127.0.0.1:60104 client_network=udp -13:42.539 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=9.852537ms client=127.0.0.1:52804 client_network=udp -13:42.540 [DEBUG] TestAgent.dns: request served from client: name=35cdcb6e-d936-97fd-40b6-f46bf88d50d8.query.consul. type=A class=IN latency=1.45192ms client=127.0.0.1:59917 client_network=udp -13:42.540 [INFO] TestAgent: Requesting shutdown -13:42.541 [INFO] TestAgent.server: shutting down server -13:42.541 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:42.541 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:42.541 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:42.541 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:42.541 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:42.541 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:42.541 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:42.541 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:42.541 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:42.541 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:42.541 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:42.541Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:42.541Z [INFO] agent.router.manager: shutting down -13:42.541 [INFO] TestAgent: consul server down -13:42.541 [INFO] TestAgent: shutdown complete -13:42.541 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16668 network=tcp -13:42.541 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16668 network=udp -13:42.541 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16669 network=tcp -13:43.042 [INFO] TestAgent: Waiting for endpoints to shut down -13:43.042 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{0_0_0_0_0_0_0_0_0_0_0} -[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:42.078Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:42.104Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:42.110 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b1804359-2887-ec04-7b71-1550807b8126 Address:127.0.0.1:16202}]" -13:42.111 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:42.111 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:42.111Z [INFO] agent.router: Initializing LAN area manager -13:42.111 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16281 network=udp -13:42.111 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16202 [Follower]" leader= -13:42.112 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16202) (DC: dc1)" -13:42.112 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:42.112 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16281 network=tcp -13:42.112 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16149 network=tcp -13:42.112 [INFO] TestAgent: started state syncer -13:42.113 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16203 network=tcp -13:42.201 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:42.201 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16202 [Candidate]" term=2 -13:42.202 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:42.202 [DEBUG] TestAgent.server.raft: vote granted: from=b1804359-2887-ec04-7b71-1550807b8126 term=2 tally=1 -13:42.202 [INFO] TestAgent.server.raft: election won: tally=1 -13:42.202 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16202 [Leader]" -13:42.202 [INFO] TestAgent.server: cluster leadership acquired -13:42.202 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16202 -13:42.205 [INFO] TestAgent.server: New leader elected: payload=test-node -13:42.210 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:42.212 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:42.212 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:42.212 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:42.212 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:42.212 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:42.212 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:42.212 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:42.226 [INFO] TestAgent.server: federation state anti-entropy synced -13:42.376 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:42.377 [INFO] TestAgent: Synced node info -13:42.377 [DEBUG] TestAgent: Node info in sync -13:42.539 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=13.866778ms client=127.0.0.1:55518 client_network=udp -13:42.540 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.331713ms client=127.0.0.1:57484 client_network=udp -13:42.555 [DEBUG] TestAgent.dns: request served from client: name=b3640365-1a42-2faa-0253-cb6b02ef1b28.query.consul. type=AAAA class=IN latency=13.880381ms client=127.0.0.1:49893 client_network=udp -13:42.556 [INFO] TestAgent: Requesting shutdown -13:42.556 [INFO] TestAgent.server: shutting down server -13:42.556 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:42.556 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:42.556 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:42.556 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:42.556 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:42.556 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:42.556 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:42.556 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:42.556 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:42.556 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:42.556 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:42.556Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:42.556Z [INFO] agent.router.manager: shutting down -13:42.556 [INFO] TestAgent: consul server down -13:42.556 [INFO] TestAgent: shutdown complete -13:42.556 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16281 network=tcp -13:42.556 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16281 network=udp -13:42.556 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16149 network=tcp -13:43.089 [INFO] TestAgent: Waiting for endpoints to shut down -13:43.089 [INFO] TestAgent: Endpoints down -=== CONT TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{0_0_0_0_0_0_0_0_0_0_0} -2021-10-31T23:13:41.983Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:42.059Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:42.060 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f0a36d87-0d59-255e-d3e7-bb1ec5aecb66 Address:127.0.0.1:16736}]" -13:42.061 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:42.061 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:42.061Z [INFO] agent.router: Initializing LAN area manager -13:42.061 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16736 [Follower]" leader= -13:42.074 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16172 network=udp -13:42.074 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16736) (DC: dc1)" -13:42.074 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:42.074 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16172 network=tcp -13:42.074 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16173 network=tcp -13:42.075 [INFO] TestAgent: started state syncer -13:42.075 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16716 network=tcp -13:42.117 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:42.117 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16736 [Candidate]" term=2 -13:42.118 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:42.118 [DEBUG] TestAgent.server.raft: vote granted: from=f0a36d87-0d59-255e-d3e7-bb1ec5aecb66 term=2 tally=1 -13:42.118 [INFO] TestAgent.server.raft: election won: tally=1 -13:42.118 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16736 [Leader]" -13:42.118 [INFO] TestAgent.server: cluster leadership acquired -13:42.118 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16736 -13:42.118 [INFO] TestAgent.server: New leader elected: payload=test-node -13:42.119 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:42.120 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:42.120 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:42.120 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:42.120 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:42.120 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:42.120 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:42.120 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:42.121 [INFO] TestAgent.server: federation state anti-entropy synced -13:42.194 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:42.194 [INFO] TestAgent: Synced node info -13:42.624 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=2.029078ms client=127.0.0.1:59899 client_network=udp -13:42.627 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=2.753253ms client=127.0.0.1:52636 client_network=udp -13:42.629 [DEBUG] TestAgent.dns: request served from client: name=c57add19-1863-c43e-f58a-b6d5782753e9.query.consul. type=ANY class=IN latency=1.785177ms client=127.0.0.1:36514 client_network=udp -13:42.629 [INFO] TestAgent: Requesting shutdown -13:42.629 [INFO] TestAgent.server: shutting down server -13:42.629 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:42.629 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:42.629 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:42.629 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:42.629 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:42.629 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:42.629 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:42.629 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:42.629 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:42.629 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:42.629 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:42.629Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:42.629Z [INFO] agent.router.manager: shutting down -13:42.630 [INFO] TestAgent: consul server down -13:42.630 [INFO] TestAgent: shutdown complete -13:42.630 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16172 network=tcp -13:42.630 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16172 network=udp -13:42.630 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16173 network=tcp -13:43.133 [INFO] TestAgent: Waiting for endpoints to shut down -13:43.189 [INFO] TestAgent: Endpoints down -=== CONT TestUiServices/No_Filter -2021-10-31T23:13:33.084Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:33.178Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:33.180 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:efc6aa6f-c74f-b0b0-4d33-3e9264e95ec6 Address:127.0.0.1:16896}]" -13:33.180 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-efc6aa6f-c74f-b0b0-4d33-3e9264e95ec6.dc1 127.0.0.1 -13:33.181 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-efc6aa6f-c74f-b0b0-4d33-3e9264e95ec6 127.0.0.1 -2021-10-31T23:13:33.181Z [INFO] agent.router: Initializing LAN area manager -13:33.181 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16896 [Follower]" leader= -13:33.182 [INFO] TestAgent.server: Adding LAN server: server="Node-efc6aa6f-c74f-b0b0-4d33-3e9264e95ec6 (Addr: tcp/127.0.0.1:16896) (DC: dc1)" -13:33.182 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-efc6aa6f-c74f-b0b0-4d33-3e9264e95ec6.dc1 area=wan -13:33.189 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16891 network=udp -13:33.189 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16891 network=tcp -13:33.204 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16892 network=tcp -13:33.204 [INFO] TestAgent: started state syncer -13:33.204 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16897 network=tcp -13:33.242 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:33.242 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16896 [Candidate]" term=2 -13:33.242 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:33.242 [DEBUG] TestAgent.server.raft: vote granted: from=efc6aa6f-c74f-b0b0-4d33-3e9264e95ec6 term=2 tally=1 -13:33.242 [INFO] TestAgent.server.raft: election won: tally=1 -13:33.242 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16896 [Leader]" -13:33.242 [INFO] TestAgent.server: cluster leadership acquired -13:33.242 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16896 -13:33.243 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:33.245 [INFO] TestAgent.server: New leader elected: payload=Node-efc6aa6f-c74f-b0b0-4d33-3e9264e95ec6 -13:33.258 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:33.258 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:33.258 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:33.258 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:33.258 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:33.258 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-efc6aa6f-c74f-b0b0-4d33-3e9264e95ec6 -13:33.258 [INFO] TestAgent.server: member joined, marking health alive: member=Node-efc6aa6f-c74f-b0b0-4d33-3e9264e95ec6 -13:33.259 [INFO] TestAgent.server: federation state anti-entropy synced -13:33.287 [INFO] TestAgent: Requesting shutdown -13:33.287 [INFO] TestAgent.server: shutting down server -13:33.287 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:33.287 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:33.287 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:33.287 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:33.287 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:33.287 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:33.287 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:33.287 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:33.288 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:33.288 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:33.288 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:33.288 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:33.288Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:33.288Z [INFO] agent.router.manager: shutting down -13:33.288 [INFO] TestAgent: consul server down -13:33.288 [INFO] TestAgent: shutdown complete -13:33.288 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16891 network=tcp -13:33.288 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16891 network=udp -13:33.288 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16892 network=tcp -13:33.789 [INFO] TestAgent: Waiting for endpoints to shut down -13:33.789 [INFO] TestAgent: Endpoints down ---- PASS: TestUiServices (0.80s) - --- PASS: TestUiServices/Filtered (0.00s) - --- PASS: TestUiServices/No_Filter (0.00s) -2021-10-31T23:13:42.408Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:42.458Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:42.460 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c40c77f0-369e-7948-0cd1-96a3a968c175 Address:127.0.0.1:16705}]" -13:42.461 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:42.461 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16705 [Follower]" leader= -13:42.466 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:42.466Z [INFO] agent.router: Initializing LAN area manager -13:42.466 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17000 network=udp -13:42.466 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16705) (DC: dc1)" -13:42.466 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:42.466 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17000 network=tcp -13:42.467 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16743 network=tcp -13:42.467 [INFO] TestAgent: started state syncer -13:42.467 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16706 network=tcp -13:42.512 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:42.512 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16705 [Candidate]" term=2 -13:42.512 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:42.512 [DEBUG] TestAgent.server.raft: vote granted: from=c40c77f0-369e-7948-0cd1-96a3a968c175 term=2 tally=1 -13:42.512 [INFO] TestAgent.server.raft: election won: tally=1 -13:42.512 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16705 [Leader]" -13:42.512 [INFO] TestAgent.server: cluster leadership acquired -13:42.512 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16705 -13:42.513 [INFO] TestAgent.server: New leader elected: payload=test-node -13:42.513 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:42.516 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:42.516 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:42.516 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:42.516 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:42.516 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:42.516 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:42.516 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:42.517 [INFO] TestAgent.server: federation state anti-entropy synced -13:42.719 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.754904ms client=127.0.0.1:51597 client_network=udp -13:42.724 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.569769ms client=127.0.0.1:54262 client_network=udp -13:42.725 [DEBUG] TestAgent.dns: request served from client: name=770da277-6609-c8f3-87cd-4df73222e2b6.query.consul. type=ANY class=IN latency=1.515381ms client=127.0.0.1:54706 client_network=udp -13:42.725 [INFO] TestAgent: Requesting shutdown -13:42.730 [INFO] TestAgent.server: shutting down server -13:42.730 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:42.730 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:42.730 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:42.730 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:42.730 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:42.730 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:42.730 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:42.730 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:42.730 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:42.730 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:42.730 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:42.730 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:42.730Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:42.730Z [INFO] agent.router.manager: shutting down -13:42.730 [INFO] TestAgent: consul server down -13:42.730 [INFO] TestAgent: shutdown complete -13:42.730 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17000 network=tcp -13:42.730 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17000 network=udp -13:42.730 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16743 network=tcp -13:43.233 [INFO] TestAgent: Waiting for endpoints to shut down -13:43.233 [INFO] TestAgent: Endpoints down -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:42.527Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:42.593Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:42.623 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7f5fd346-e576-2986-d71d-d1ba18ff22ab Address:127.0.0.1:16761}]" -13:42.623 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:42.624 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:42.624Z [INFO] agent.router: Initializing LAN area manager -13:42.624 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16582 network=udp -13:42.624 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16761 [Follower]" leader= -13:42.625 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16761) (DC: dc1)" -13:42.625 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:42.625 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16582 network=tcp -13:42.626 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16583 network=tcp -13:42.626 [INFO] TestAgent: started state syncer -13:42.626 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16762 network=tcp -13:42.677 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:42.677 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16761 [Candidate]" term=2 -13:42.678 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:42.678 [DEBUG] TestAgent.server.raft: vote granted: from=7f5fd346-e576-2986-d71d-d1ba18ff22ab term=2 tally=1 -13:42.678 [INFO] TestAgent.server.raft: election won: tally=1 -13:42.678 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16761 [Leader]" -13:42.678 [INFO] TestAgent.server: cluster leadership acquired -13:42.678 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16761 -13:42.679 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:42.680 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:42.680 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:42.680 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:42.680 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:42.680 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:42.680 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:42.680 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:42.681 [INFO] TestAgent.server: New leader elected: payload=test-node -13:42.681 [INFO] TestAgent.server: federation state anti-entropy synced -13:42.883 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:42.884 [INFO] TestAgent: Synced node info -13:42.884 [DEBUG] TestAgent: Node info in sync -13:43.185 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=21.531249ms client=127.0.0.1:44310 client_network=udp -13:43.187 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.153741ms client=127.0.0.1:40828 client_network=udp -13:43.188 [DEBUG] TestAgent.dns: request served from client: name=f564f74a-ea6a-df43-4614-11f66d0c0cb9.query.consul. type=A class=IN latency=1.093181ms client=127.0.0.1:48472 client_network=udp -13:43.188 [INFO] TestAgent: Requesting shutdown -13:43.188 [INFO] TestAgent.server: shutting down server -13:43.188 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:43.188 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:43.188 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:43.188 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:43.188 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:43.189 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:43.189 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:43.189 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:43.189 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:43.189 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:43.189 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:43.189Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:43.189Z [INFO] agent.router.manager: shutting down -13:43.189 [INFO] TestAgent: consul server down -13:43.189 [INFO] TestAgent: shutdown complete -13:43.189 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16582 network=tcp -13:43.189 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16582 network=udp -13:43.189 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16583 network=tcp -13:43.689 [INFO] TestAgent: Waiting for endpoints to shut down -13:43.689 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:42.622Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:42.685Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:42.687 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2f73b335-861f-1c99-e424-8c13f5053ebf Address:127.0.0.1:16754}]" -13:42.688 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:42.689 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:42.689Z [INFO] agent.router: Initializing LAN area manager -13:42.689 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16756 network=udp -13:42.689 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16754 [Follower]" leader= -13:42.689 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16754) (DC: dc1)" -13:42.689 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:42.689 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16756 network=tcp -13:42.695 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16708 network=tcp -13:42.695 [INFO] TestAgent: started state syncer -13:42.695 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16755 network=tcp -13:42.753 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:42.753 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16754 [Candidate]" term=2 -13:42.754 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:42.754 [DEBUG] TestAgent.server.raft: vote granted: from=2f73b335-861f-1c99-e424-8c13f5053ebf term=2 tally=1 -13:42.754 [INFO] TestAgent.server.raft: election won: tally=1 -13:42.754 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16754 [Leader]" -13:42.754 [INFO] TestAgent.server: cluster leadership acquired -13:42.754 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16754 -13:42.755 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:42.756 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:42.756 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:42.756 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:42.756 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:42.756 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:42.756 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:42.756 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:42.757 [INFO] TestAgent.server: New leader elected: payload=test-node -13:42.757 [INFO] TestAgent.server: federation state anti-entropy synced -13:42.773 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:42.774 [INFO] TestAgent: Synced node info -13:43.188 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.334362ms client=127.0.0.1:43227 client_network=udp -13:43.192 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=3.270505ms client=127.0.0.1:40675 client_network=udp -13:43.193 [DEBUG] TestAgent.dns: request served from client: name=429ceff5-5e0e-ecbc-81aa-91ae8f8991e8.query.consul. type=ANY class=IN latency=1.280713ms client=127.0.0.1:56905 client_network=udp -13:43.193 [INFO] TestAgent: Requesting shutdown -13:43.193 [INFO] TestAgent.server: shutting down server -13:43.193 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:43.193 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:43.193 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:43.193 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:43.193 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:43.193 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:43.193 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:43.193 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:43.193 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:43.193 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:43.193 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:43.193Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:43.194Z [INFO] agent.router.manager: shutting down -13:43.194 [INFO] TestAgent: consul server down -13:43.194 [INFO] TestAgent: shutdown complete -13:43.194 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16756 network=tcp -13:43.194 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16756 network=udp -13:43.194 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16708 network=tcp -13:43.704 [INFO] TestAgent: Waiting for endpoints to shut down -13:43.704 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:42.807Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:42.848Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:42.863 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:108a4ce2-1712-1465-47a2-beaf871b959b Address:127.0.0.1:16740}]" -13:42.863 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16740 [Follower]" leader= -13:42.863 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:42.878 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:42.878Z [INFO] agent.router: Initializing LAN area manager -13:42.878 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16763 network=udp -13:42.879 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16740) (DC: dc1)" -13:42.879 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:42.879 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16763 network=tcp -13:42.879 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16778 network=tcp -13:42.880 [INFO] TestAgent: started state syncer -13:42.880 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16741 network=tcp -13:42.908 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:42.908 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16740 [Candidate]" term=2 -13:42.908 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:42.908 [DEBUG] TestAgent.server.raft: vote granted: from=108a4ce2-1712-1465-47a2-beaf871b959b term=2 tally=1 -13:42.908 [INFO] TestAgent.server.raft: election won: tally=1 -13:42.908 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16740 [Leader]" -13:42.908 [INFO] TestAgent.server: cluster leadership acquired -13:42.908 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16740 -13:42.909 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:42.910 [INFO] TestAgent.server: New leader elected: payload=test-node -13:42.915 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:42.915 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:42.915 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:42.915 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:42.915 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:42.915 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:42.915 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:42.916 [INFO] TestAgent.server: federation state anti-entropy synced -13:43.150 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:43.151 [INFO] TestAgent: Synced node info -13:43.151 [DEBUG] TestAgent: Node info in sync -13:43.278 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=44.109075ms client=127.0.0.1:37415 client_network=udp -13:43.280 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.803943ms client=127.0.0.1:45810 client_network=udp -13:43.294 [DEBUG] TestAgent.dns: request served from client: name=64e37b74-483b-a8f7-4b55-7a9ff87a4803.query.consul. type=AAAA class=IN latency=13.874617ms client=127.0.0.1:53562 client_network=udp -13:43.294 [INFO] TestAgent: Requesting shutdown -13:43.294 [INFO] TestAgent.server: shutting down server -13:43.294 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:43.294 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:43.294 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:43.294 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:43.294 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:43.294 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:43.294 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:43.294 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:43.294 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:43.294 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:43.309 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:43.310Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:43.311Z [INFO] agent.router.manager: shutting down -13:43.312 [INFO] TestAgent: consul server down -13:43.312 [INFO] TestAgent: shutdown complete -13:43.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16763 network=tcp -13:43.312 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16763 network=udp -13:43.312 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16778 network=tcp -13:43.813 [INFO] TestAgent: Waiting for endpoints to shut down -13:43.813 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:42.851Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:42.884Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:42.898 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:70a0f35d-b413-0efd-7e64-224feeb26a1a Address:127.0.0.1:16726}]" -13:42.899 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16726 [Follower]" leader= -13:42.899 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:42.900 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:42.900Z [INFO] agent.router: Initializing LAN area manager -13:42.900 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16721 network=udp -13:42.900 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16726) (DC: dc1)" -13:42.900 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:42.900 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16721 network=tcp -13:42.901 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16722 network=tcp -13:42.901 [INFO] TestAgent: started state syncer -13:42.901 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16727 network=tcp -13:42.937 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:42.937 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16726 [Candidate]" term=2 -13:42.938 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:42.938 [DEBUG] TestAgent.server.raft: vote granted: from=70a0f35d-b413-0efd-7e64-224feeb26a1a term=2 tally=1 -13:42.938 [INFO] TestAgent.server.raft: election won: tally=1 -13:42.938 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16726 [Leader]" -13:42.938 [INFO] TestAgent.server: cluster leadership acquired -13:42.938 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16726 -13:42.939 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:42.940 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:42.940 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:42.940 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:42.940 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:42.940 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:42.940 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:42.940 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:42.941 [INFO] TestAgent.server: New leader elected: payload=test-node -13:42.941 [INFO] TestAgent.server: federation state anti-entropy synced -13:43.082 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:43.182 [INFO] TestAgent: Synced node info -13:43.322 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=3.348425ms client=127.0.0.1:37019 client_network=udp -13:43.324 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.683482ms client=127.0.0.1:49910 client_network=udp -13:43.346 [DEBUG] TestAgent.dns: request served from client: name=53d89289-d2ce-3466-f126-236c96f09d1b.query.consul. type=ANY class=IN latency=21.885891ms client=127.0.0.1:48230 client_network=udp -13:43.346 [INFO] TestAgent: Requesting shutdown -13:43.346 [INFO] TestAgent.server: shutting down server -13:43.346 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:43.346 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:43.346 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:43.346 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:43.346 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:43.346 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:43.346 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:43.346 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:43.346 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:43.346 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:43.346 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:43.346Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:43.347Z [INFO] agent.router.manager: shutting down -13:43.347 [INFO] TestAgent: consul server down -13:43.347 [INFO] TestAgent: shutdown complete -13:43.347 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16721 network=tcp -13:43.347 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16721 network=udp -13:43.347 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16722 network=tcp -13:43.849 [INFO] TestAgent: Waiting for endpoints to shut down -13:43.849 [INFO] TestAgent: Endpoints down -[WARN] freeport: 10 out of 38 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:42.517Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:42.610Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:42.651 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c019da86-ec72-1d6d-1287-10b24e1684a7 Address:127.0.0.1:16747}]" -13:42.651 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:42.652 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:42.652Z [INFO] agent.router: Initializing LAN area manager -13:42.652 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16749 network=udp -13:42.652 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16747 [Follower]" leader= -13:42.652 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16747) (DC: dc1)" -13:42.652 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:42.652 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16749 network=tcp -13:42.653 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16750 network=tcp -13:42.653 [INFO] TestAgent: started state syncer -13:42.653 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16748 network=tcp -13:42.697 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:42.697 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16747 [Candidate]" term=2 -13:42.698 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:42.698 [DEBUG] TestAgent.server.raft: vote granted: from=c019da86-ec72-1d6d-1287-10b24e1684a7 term=2 tally=1 -13:42.698 [INFO] TestAgent.server.raft: election won: tally=1 -13:42.698 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16747 [Leader]" -13:42.698 [INFO] TestAgent.server: cluster leadership acquired -13:42.698 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16747 -13:42.699 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:42.700 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:42.700 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:42.700 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:42.700 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:42.700 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:42.700 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:42.700 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:42.701 [INFO] TestAgent.server: New leader elected: payload=test-node -13:42.701 [INFO] TestAgent.server: federation state anti-entropy synced -13:42.834 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:42.834 [INFO] TestAgent: Synced node info -13:42.834 [DEBUG] TestAgent: Node info in sync -13:43.347 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=9.70135ms client=127.0.0.1:33703 client_network=udp -13:43.390 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=42.453788ms client=127.0.0.1:38657 client_network=udp -13:43.392 [DEBUG] TestAgent.dns: request served from client: name=07310efe-583e-e0ea-4e28-4a7b9c135ee9.query.consul. type=AAAA class=IN latency=1.414258ms client=127.0.0.1:41716 client_network=udp -13:43.392 [INFO] TestAgent: Requesting shutdown -13:43.392 [INFO] TestAgent.server: shutting down server -13:43.392 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:43.392 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:43.392 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:43.392 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:43.392 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:43.392 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:43.392 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:43.392 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:43.392 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:43.392 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:43.392 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:43.393Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:43.393Z [INFO] agent.router.manager: shutting down -13:43.393 [INFO] TestAgent: consul server down -13:43.393 [INFO] TestAgent: shutdown complete -13:43.393 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16749 network=tcp -13:43.393 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16749 network=udp -13:43.393 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16750 network=tcp -13:43.893 [INFO] TestAgent: Waiting for endpoints to shut down -13:43.893 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:42.947Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:42.986Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:42.988 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cd18595b-4671-288d-d8e3-bf755357bb4c Address:127.0.0.1:16691}]" -13:42.989 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16691 [Follower]" leader= -13:43.059 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:43.061 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:43.061Z [INFO] agent.router: Initializing LAN area manager -13:43.089 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16707 network=udp -13:43.098 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16691) (DC: dc1)" -13:43.098 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16707 network=tcp -13:43.110 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:43.110 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16691 [Candidate]" term=2 -13:43.110 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:43.110 [DEBUG] TestAgent.server.raft: vote granted: from=cd18595b-4671-288d-d8e3-bf755357bb4c term=2 tally=1 -13:43.110 [INFO] TestAgent.server.raft: election won: tally=1 -13:43.110 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16691 [Leader]" -13:43.150 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:43.166 [INFO] TestAgent.server: cluster leadership acquired -13:43.166 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16691 -13:43.167 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:43.168 [INFO] TestAgent.server: New leader elected: payload=test-node -13:43.169 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:43.169 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:43.169 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:43.169 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:43.169 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:43.169 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:43.169 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:43.170 [INFO] TestAgent.server: federation state anti-entropy synced -13:43.174 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16687 network=tcp -13:43.174 [INFO] TestAgent: started state syncer -13:43.177 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16692 network=tcp -13:43.178 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:43.178 [INFO] TestAgent: Synced node info -13:43.178 [DEBUG] TestAgent: Node info in sync -13:43.431 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.569345ms client=127.0.0.1:55903 client_network=udp -13:43.440 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=2.026217ms client=127.0.0.1:39230 client_network=udp -13:43.451 [DEBUG] TestAgent.dns: request served from client: name=92c0f784-73c8-6661-ef25-f5268ccb485b.query.consul. type=ANY class=IN latency=1.64587ms client=127.0.0.1:44848 client_network=udp -13:43.451 [INFO] TestAgent: Requesting shutdown -13:43.452 [INFO] TestAgent.server: shutting down server -13:43.452 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:43.452 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:43.452 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:43.452 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:43.452 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:43.452 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:43.452 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:43.452 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:43.452 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:43.452 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:43.452 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:43.452Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:43.452Z [INFO] agent.router.manager: shutting down -13:43.452 [INFO] TestAgent: consul server down -13:43.452 [INFO] TestAgent: shutdown complete -13:43.452 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16707 network=tcp -13:43.452 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16707 network=udp -13:43.475 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16687 network=tcp -13:43.977 [INFO] TestAgent: Waiting for endpoints to shut down -13:43.998 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:42.961Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:43.063Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:43.086 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c1aa71bd-70f2-b735-b7b3-c3ecb9160374 Address:127.0.0.1:16580}]" -13:43.087 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:43.088 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:43.088Z [INFO] agent.router: Initializing LAN area manager -13:43.089 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16805 network=udp -13:43.089 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16580 [Follower]" leader= -13:43.097 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:43.099 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16805 network=tcp -13:43.105 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16580) (DC: dc1)" -13:43.128 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:43.128 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16580 [Candidate]" term=2 -13:43.128 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:43.128 [DEBUG] TestAgent.server.raft: vote granted: from=c1aa71bd-70f2-b735-b7b3-c3ecb9160374 term=2 tally=1 -13:43.128 [INFO] TestAgent.server.raft: election won: tally=1 -13:43.128 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16580 [Leader]" -13:43.158 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16729 network=tcp -13:43.158 [INFO] TestAgent: started state syncer -13:43.161 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16581 network=tcp -13:43.189 [INFO] TestAgent.server: cluster leadership acquired -13:43.189 [INFO] TestAgent: Synced node info -13:43.189 [DEBUG] TestAgent: Node info in sync -13:43.190 [INFO] TestAgent.server: New leader elected: payload=test-node -13:43.190 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16580 -13:43.191 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:43.192 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:43.192 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:43.192 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:43.192 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:43.192 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:43.192 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:43.192 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:43.193 [INFO] TestAgent.server: federation state anti-entropy synced -13:43.482 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=6.316242ms client=127.0.0.1:34507 client_network=udp -13:43.484 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.650982ms client=127.0.0.1:43042 client_network=udp -13:43.502 [DEBUG] TestAgent.dns: request served from client: name=4f8f2086-5a9d-bfc2-605b-fa01122daa52.query.consul. type=AAAA class=IN latency=17.763233ms client=127.0.0.1:36252 client_network=udp -13:43.502 [INFO] TestAgent: Requesting shutdown -13:43.502 [INFO] TestAgent.server: shutting down server -13:43.502 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:43.502 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:43.502 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:43.502 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:43.502 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:43.502 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:43.502 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:43.502 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:43.502 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:43.502 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:43.502 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:43.502Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:43.502Z [INFO] agent.router.manager: shutting down -13:43.503 [INFO] TestAgent: consul server down -13:43.503 [INFO] TestAgent: shutdown complete -13:43.503 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16805 network=tcp -13:43.503 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16805 network=udp -13:43.503 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16729 network=tcp -13:44.003 [INFO] TestAgent: Waiting for endpoints to shut down -13:44.003 [INFO] TestAgent: Endpoints down -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=false#01 +2022-12-04T05:49:43.341Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.351Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.352 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1ba005d1-00ae-496b-bb89-36cdbee81189 Address:127.0.0.1:23946}]" +49:43.352 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23946 [Follower]" leader= +49:43.352 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.353 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.353Z [INFO] agent.router: Initializing LAN area manager +49:43.353 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23946) (DC: dc1)" +49:43.353 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.353 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23899 network=udp +49:43.353 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23899 network=tcp +49:43.354 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23900 network=tcp +49:43.354 [INFO] TestAgent: started state syncer +49:43.354 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23905 network=tcp +49:43.409 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:43.409 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23946 [Candidate]" term=2 +49:43.409 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:43.409 [DEBUG] TestAgent.server.raft: vote granted: from=1ba005d1-00ae-496b-bb89-36cdbee81189 term=2 tally=1 +49:43.409 [INFO] TestAgent.server.raft: election won: tally=1 +49:43.409 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23946 [Leader]" +49:43.409 [INFO] TestAgent.server: cluster leadership acquired +49:43.409 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23946 +49:43.410 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:43.412 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:43.412 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:43.412 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:43.412 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:43.412 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:43.412 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:43.412 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:43.412 [INFO] TestAgent.server: New leader elected: payload=test-node +49:43.412 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.455 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.230164ms client=127.0.0.1:46714 client_network=udp +49:43.457 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.286246ms client=127.0.0.1:52129 client_network=udp +49:43.458 [DEBUG] TestAgent.dns: request served from client: name=825ed7e0-cbed-6ab2-bb9e-ebc0f1c5ef17.query.consul. type=AAAA class=IN latency=1.079385ms client=127.0.0.1:45078 client_network=udp +49:43.458 [INFO] TestAgent: Requesting shutdown +49:43.459 [INFO] TestAgent.server: shutting down server +49:43.459 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.459 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.459 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.459 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.459 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.459 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.459 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.459 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.459 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:43.459 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:43.459 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.459 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:43.459Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.459Z [INFO] agent.router.manager: shutting down +49:43.459 [INFO] TestAgent: consul server down +49:43.459 [INFO] TestAgent: shutdown complete +49:43.459 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23899 network=tcp +49:43.459 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23899 network=udp +49:43.459 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23900 network=tcp +49:43.968 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.968 [INFO] TestAgent: Endpoints down [WARN] freeport: 10 out of 31 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:43.053Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:43.188Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:43.226 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:10f767d6-961f-6829-baf4-404e02443bce Address:127.0.0.1:16782}]" -13:43.227 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:43.227 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:43.227Z [INFO] agent.router: Initializing LAN area manager -13:43.227 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16770 network=udp -13:43.227 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16782 [Follower]" leader= -13:43.228 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16782) (DC: dc1)" -13:43.228 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:43.228 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16770 network=tcp -13:43.229 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16785 network=tcp -13:43.229 [INFO] TestAgent: started state syncer -13:43.229 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16783 network=tcp -13:43.300 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:43.300 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16782 [Candidate]" term=2 -13:43.300 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:43.300 [DEBUG] TestAgent.server.raft: vote granted: from=10f767d6-961f-6829-baf4-404e02443bce term=2 tally=1 -13:43.300 [INFO] TestAgent.server.raft: election won: tally=1 -13:43.300 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16782 [Leader]" -13:43.301 [INFO] TestAgent.server: cluster leadership acquired -13:43.301 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16782 -13:43.302 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:43.306 [INFO] TestAgent.server: New leader elected: payload=test-node -13:43.318 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:43.318 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:43.318 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:43.318 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:43.318 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:43.318 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:43.318 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:43.320 [INFO] TestAgent.server: federation state anti-entropy synced -13:43.321 [INFO] TestAgent: Synced node info -13:43.321 [DEBUG] TestAgent: Node info in sync -13:43.623 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.295865ms client=127.0.0.1:59648 client_network=udp -13:43.625 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.015304ms client=127.0.0.1:51676 client_network=udp -13:43.630 [DEBUG] TestAgent.dns: request served from client: name=ad6c6535-ae38-40b3-8ef5-90ccbbfe0e3a.query.consul. type=A class=IN latency=5.4315ms client=127.0.0.1:43754 client_network=udp -13:43.633 [INFO] TestAgent: Requesting shutdown -13:43.634 [INFO] TestAgent.server: shutting down server -13:43.634 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:43.634 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:43.634 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:43.634 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:43.634 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:43.634 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:43.634 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:43.634 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:43.634 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:43.634 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:43.634 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:43.634Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:43.634Z [INFO] agent.router.manager: shutting down -13:43.634 [INFO] TestAgent: consul server down -13:43.634 [INFO] TestAgent: shutdown complete -13:43.634 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16770 network=tcp -13:43.634 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16770 network=udp -13:43.634 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16785 network=tcp -13:44.137 [INFO] TestAgent: Waiting for endpoints to shut down -13:44.137 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:43.089Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:43.196Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:43.202 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:87f34024-ed03-4fa7-7d90-df34d4b6c32a Address:127.0.0.1:16817}]" -13:43.203 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:43.203 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:43.203Z [INFO] agent.router: Initializing LAN area manager -13:43.204 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16742 network=udp -13:43.204 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16817 [Follower]" leader= -13:43.204 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16817) (DC: dc1)" -13:43.204 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:43.204 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16742 network=tcp -13:43.205 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16792 network=tcp -13:43.205 [INFO] TestAgent: started state syncer -13:43.205 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16818 network=tcp -13:43.269 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:43.269 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16817 [Candidate]" term=2 -13:43.270 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:43.270 [DEBUG] TestAgent.server.raft: vote granted: from=87f34024-ed03-4fa7-7d90-df34d4b6c32a term=2 tally=1 -13:43.270 [INFO] TestAgent.server.raft: election won: tally=1 -13:43.270 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16817 [Leader]" -13:43.270 [INFO] TestAgent.server: cluster leadership acquired -13:43.270 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16817 -13:43.271 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:43.273 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:43.273 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:43.273 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:43.273 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:43.273 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:43.273 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:43.273 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:43.277 [INFO] TestAgent.server: New leader elected: payload=test-node -13:43.278 [INFO] TestAgent.server: federation state anti-entropy synced -13:43.570 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:43.570 [INFO] TestAgent: Synced node info -13:43.688 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.751602ms client=127.0.0.1:53271 client_network=udp -13:43.689 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.341713ms client=127.0.0.1:42532 client_network=udp -13:43.715 [DEBUG] TestAgent.dns: request served from client: name=17227496-6813-4e11-a5ba-a5412bfcafbc.query.consul. type=AAAA class=IN latency=26.167698ms client=127.0.0.1:55390 client_network=udp -13:43.716 [INFO] TestAgent: Requesting shutdown -13:43.716 [INFO] TestAgent.server: shutting down server -13:43.716 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:43.716 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:43.716 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:43.716 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:43.716 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:43.716 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:43.716 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:43.716 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:43.716 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:43.716 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:43.716 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:43.716Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:43.716Z [INFO] agent.router.manager: shutting down -13:43.716 [INFO] TestAgent: consul server down -13:43.716 [INFO] TestAgent: shutdown complete -13:43.716 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16742 network=tcp -13:43.716 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16742 network=udp -13:43.716 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16792 network=tcp -13:44.217 [INFO] TestAgent: Waiting for endpoints to shut down -13:44.217 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:43.192Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:43.304Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:43.309 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b20ee004-2766-5dfa-ddd6-df3074e30a7b Address:127.0.0.1:16803}]" -13:43.309 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16803 [Follower]" leader= -13:43.311 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:43.311 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:43.311Z [INFO] agent.router: Initializing LAN area manager -13:43.312 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16840 network=udp -13:43.315 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:43.315 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16803) (DC: dc1)" -13:43.316 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16840 network=tcp -13:43.331 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16764 network=tcp -13:43.335 [INFO] TestAgent: started state syncer -13:43.344 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16804 network=tcp -13:43.345 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:43.353 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16803 [Candidate]" term=2 -13:43.354 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:43.354 [DEBUG] TestAgent.server.raft: vote granted: from=b20ee004-2766-5dfa-ddd6-df3074e30a7b term=2 tally=1 -13:43.354 [INFO] TestAgent.server.raft: election won: tally=1 -13:43.354 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16803 [Leader]" -13:43.354 [INFO] TestAgent.server: cluster leadership acquired -13:43.394 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16803 -13:43.394 [INFO] TestAgent.server: New leader elected: payload=test-node -13:43.395 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:43.396 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:43.396 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:43.396 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:43.396 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:43.396 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:43.396 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:43.396 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:43.397 [INFO] TestAgent.server: federation state anti-entropy synced -13:43.440 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:43.440 [INFO] TestAgent: Synced node info -13:43.440 [DEBUG] TestAgent: Node info in sync -13:43.785 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.312314ms client=127.0.0.1:59744 client_network=udp -13:43.790 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=5.687605ms client=127.0.0.1:39327 client_network=udp -13:43.794 [DEBUG] TestAgent.dns: request served from client: name=8b9c8922-8609-58d2-f117-c4d17802ee75.query.consul. type=ANY class=IN latency=1.609185ms client=127.0.0.1:53435 client_network=udp -13:43.798 [INFO] TestAgent: Requesting shutdown -13:43.798 [INFO] TestAgent.server: shutting down server -13:43.798 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:43.798 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:43.798 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:43.798 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:43.798 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:43.798 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:43.798 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:43.798 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -13:43.798 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:43.798 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:43.798 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:13:43.798Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:43.798Z [INFO] agent.router.manager: shutting down -13:43.798 [INFO] TestAgent: consul server down -13:43.798 [INFO] TestAgent: shutdown complete -13:43.798 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16840 network=tcp -13:43.798 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16840 network=udp -13:43.798 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16764 network=tcp -13:44.299 [INFO] TestAgent: Waiting for endpoints to shut down -13:44.299 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:43.083Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:43.190Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:43.192 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:74da3eac-97de-6418-c243-5a0d7d5997fe Address:127.0.0.1:16775}]" -13:43.192 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:43.193 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:43.193Z [INFO] agent.router: Initializing LAN area manager -13:43.193 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16679 network=udp -13:43.193 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16775 [Follower]" leader= -13:43.193 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16679 network=tcp -13:43.193 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:43.193 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16775) (DC: dc1)" -13:43.219 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16806 network=tcp -13:43.219 [INFO] TestAgent: started state syncer -13:43.219 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16776 network=tcp -13:43.272 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:43.272 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16775 [Candidate]" term=2 -13:43.272 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:43.272 [DEBUG] TestAgent.server.raft: vote granted: from=74da3eac-97de-6418-c243-5a0d7d5997fe term=2 tally=1 -13:43.272 [INFO] TestAgent.server.raft: election won: tally=1 -13:43.272 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16775 [Leader]" -13:43.273 [INFO] TestAgent.server: cluster leadership acquired -13:43.273 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16775 -13:43.277 [INFO] TestAgent.server: New leader elected: payload=test-node -13:43.278 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:43.280 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:43.280 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:43.280 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:43.280 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:43.280 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:43.280 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:43.280 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:43.281 [INFO] TestAgent.server: federation state anti-entropy synced -13:43.422 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:43.422 [INFO] TestAgent: Synced node info -13:43.794 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.2584ms client=127.0.0.1:32884 client_network=udp -13:43.803 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.277859ms client=127.0.0.1:46655 client_network=udp -13:43.811 [DEBUG] TestAgent.dns: request served from client: name=31482a53-0a72-dce2-a80e-c510ff1b0f70.query.consul. type=A class=IN latency=1.496031ms client=127.0.0.1:55466 client_network=udp -13:43.811 [INFO] TestAgent: Requesting shutdown -13:43.811 [INFO] TestAgent.server: shutting down server -13:43.811 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:43.811 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:43.811 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:43.811 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:43.811 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:43.811 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:43.811 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:43.811 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:43.811 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:43.811 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:43.811 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:43.812Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:43.812Z [INFO] agent.router.manager: shutting down -13:43.812 [INFO] TestAgent: consul server down -13:43.812 [INFO] TestAgent: shutdown complete -13:43.812 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16679 network=tcp -13:43.812 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16679 network=udp -13:43.812 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16806 network=tcp -13:44.317 [INFO] TestAgent: Waiting for endpoints to shut down -13:44.317 [INFO] TestAgent: Endpoints down -2021-10-31T23:13:43.224Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:43.293Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:43.296 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3933e0d6-1779-37df-3620-8667fed3ffa8 Address:127.0.0.1:16768}]" -13:43.296 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -13:43.297 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:13:43.297Z [INFO] agent.router: Initializing LAN area manager -13:43.299 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16768 [Follower]" leader= -13:43.300 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -13:43.300 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:16768) (DC: dc1)" -13:43.306 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16777 network=udp -13:43.307 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16777 network=tcp -13:43.308 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16757 network=tcp -13:43.308 [INFO] TestAgent: started state syncer -13:43.308 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16769 network=tcp -13:43.365 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:43.373 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16768 [Candidate]" term=2 -13:43.374 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:43.374 [DEBUG] TestAgent.server.raft: vote granted: from=3933e0d6-1779-37df-3620-8667fed3ffa8 term=2 tally=1 -13:43.374 [INFO] TestAgent.server.raft: election won: tally=1 -13:43.374 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16768 [Leader]" -13:43.374 [INFO] TestAgent.server: cluster leadership acquired -13:43.396 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16768 -13:43.397 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:43.397 [INFO] TestAgent.server: New leader elected: payload=test-node -13:43.402 [INFO] TestAgent: Synced node info -13:43.402 [DEBUG] TestAgent: Node info in sync -13:43.407 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:43.407 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:43.407 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:43.407 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:43.407 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:43.407 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -13:43.407 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -13:43.408 [INFO] TestAgent.server: federation state anti-entropy synced -13:43.811 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=1.486369ms client=127.0.0.1:41011 client_network=udp -13:43.823 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=1.45662ms client=127.0.0.1:37943 client_network=udp -13:43.827 [DEBUG] TestAgent.dns: request served from client: name=1ea12600-fc59-a731-5cd0-cec8a4e3a52a.query.consul. type=AAAA class=IN latency=1.283386ms client=127.0.0.1:33927 client_network=udp -13:43.827 [INFO] TestAgent: Requesting shutdown -13:43.827 [INFO] TestAgent.server: shutting down server -13:43.827 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:43.827 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:43.827 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:43.827 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:43.827 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:43.827 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:43.827 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:43.827 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:43.827 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:43.827 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:43.827 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:43.827Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:43.827Z [INFO] agent.router.manager: shutting down -13:43.828 [INFO] TestAgent: consul server down -13:43.828 [INFO] TestAgent: shutdown complete -13:43.828 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16777 network=tcp -13:43.828 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16777 network=udp -13:43.828 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16757 network=tcp -13:44.332 [INFO] TestAgent: Waiting for endpoints to shut down -13:44.332 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_ServiceLookup_AnswerLimits (0.00s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{6_6_6_6_6_6_6_5_6_6_-5} (0.98s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} (0.99s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{0_0_0_0_0_0_0_0_0_0_0} (1.10s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} (1.11s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} (1.01s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} (1.14s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} (1.13s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} (1.00s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{9_9_8_8_6_8_8_5_8_8_-5} (1.06s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{9_9_8_8_6_8_8_5_8_8_-5} (1.27s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{8_8_8_8_6_8_8_5_8_8_-5} (1.03s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{8_8_8_8_6_8_8_5_8_8_-5} (0.96s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{9_9_8_8_6_8_8_5_8_8_-5} (1.22s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{7_7_7_7_6_7_7_5_7_7_-5} (1.07s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{7_7_7_7_6_7_7_5_7_7_-5} (1.05s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{8_8_8_8_6_8_8_5_8_8_-5} (1.25s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{6_6_6_6_6_6_6_5_6_6_-5} (1.01s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{7_7_7_7_6_7_7_5_7_7_-5} (1.14s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{5_5_5_5_5_5_5_5_5_5_5} (0.99s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{5_5_5_5_5_5_5_5_5_5_5} (0.84s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{3_3_3_3_3_3_3_3_3_3_3} (1.07s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{5_5_5_5_5_5_5_5_5_5_5} (1.14s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{6_6_6_6_6_6_6_5_6_6_-5} (1.29s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{4_4_4_4_4_4_4_4_4_4_4} (0.86s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{4_4_4_4_4_4_4_4_4_4_4} (1.24s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{3_3_3_3_3_3_3_3_3_3_3} (1.17s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{3_3_3_3_3_3_3_3_3_3_3} (1.05s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{1_1_1_1_1_1_1_1_1_1_1} (1.03s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{4_4_4_4_4_4_4_4_4_4_4} (1.45s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{2_2_2_2_2_2_2_2_2_2_2} (1.09s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{2_2_2_2_2_2_2_2_2_2_2} (1.07s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{1_1_1_1_1_1_1_1_1_1_1} (1.16s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{1_1_1_1_1_1_1_1_1_1_1} (1.24s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{0_0_0_0_0_0_0_0_0_0_0} (1.26s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{2_2_2_2_2_2_2_2_2_2_2} (1.36s) - --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{0_0_0_0_0_0_0_0_0_0_0} (1.15s) +2022-12-04T05:49:37.930Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.945Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.947 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:57e070de-d565-e7e4-4127-85c382a8eb71 Address:127.0.0.1:24249}]" +49:37.947 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24249 [Follower]" leader= +49:37.947 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-57e070de-d565-e7e4-4127-85c382a8eb71.dc1 127.0.0.1 +49:37.948 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-57e070de-d565-e7e4-4127-85c382a8eb71 127.0.0.1 +2022-12-04T05:49:37.948Z [INFO] agent.router: Initializing LAN area manager +49:37.948 [INFO] TestAgent.server: Adding LAN server: server="Node-57e070de-d565-e7e4-4127-85c382a8eb71 (Addr: tcp/127.0.0.1:24249) (DC: dc1)" +49:37.948 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-57e070de-d565-e7e4-4127-85c382a8eb71.dc1 area=wan +49:37.948 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24272 network=udp +49:37.948 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24272 network=tcp +49:37.948 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24245 network=tcp +49:37.949 [INFO] TestAgent: started state syncer +49:37.949 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24243 network=tcp +49:38.004 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:38.004 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24249 [Candidate]" term=2 +49:38.004 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:38.004 [DEBUG] TestAgent.server.raft: vote granted: from=57e070de-d565-e7e4-4127-85c382a8eb71 term=2 tally=1 +49:38.004 [INFO] TestAgent.server.raft: election won: tally=1 +49:38.004 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24249 [Leader]" +49:38.005 [INFO] TestAgent.server: cluster leadership acquired +49:38.005 [INFO] TestAgent.server: New leader elected: payload=Node-57e070de-d565-e7e4-4127-85c382a8eb71 +49:38.005 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24249 +49:38.005 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:38.007 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:38.007 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:38.007 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:38.007 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:38.007 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:38.007 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-57e070de-d565-e7e4-4127-85c382a8eb71 +49:38.007 [INFO] TestAgent.server: member joined, marking health alive: member=Node-57e070de-d565-e7e4-4127-85c382a8eb71 +49:38.007 [INFO] TestAgent.server: federation state anti-entropy synced +49:38.366 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:38.367 [INFO] TestAgent: Synced node info +49:38.372 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:42378 +49:38.373 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4c381542-64eb-5bd1-c5b5-f5019201b067 127.0.0.1 +49:38.373 [INFO] TestAgent.server: Adding LAN server: server="Node-4c381542-64eb-5bd1-c5b5-f5019201b067 (Addr: tcp/127.0.0.1:23979) (DC: dc1)" +49:38.373 [INFO] TestAgent.server: New leader elected: payload=Node-4c381542-64eb-5bd1-c5b5-f5019201b067 +49:38.373 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-4c381542-64eb-5bd1-c5b5-f5019201b067 other=Node-57e070de-d565-e7e4-4127-85c382a8eb71 +49:38.373 [INFO] TestAgent: Requesting shutdown +49:38.373 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4c381542-64eb-5bd1-c5b5-f5019201b067 +49:38.373 [INFO] TestAgent.server: shutting down server +49:38.373 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:38.373 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:38.373 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:38.373 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:38.373 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:38.373 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:38.373 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:38.373 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:38.373 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:38.373 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:38.373 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:38.374 [DEBUG] TestAgent.server: Failed to flood-join server at address: server=Node-4c381542-64eb-5bd1-c5b5-f5019201b067.dc1 address=127.0.0.1:23978 error="Serf can't Join after Leave or Shutdown" +2022-12-04T05:49:38.374Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.374Z [INFO] agent.router.manager: shutting down +49:38.374 [INFO] TestAgent: consul server down +49:38.374 [INFO] TestAgent: shutdown complete +49:38.374 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24272 network=tcp +49:38.374 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24272 network=udp +49:38.374 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24245 network=tcp +49:38.874 [INFO] TestAgent: Waiting for endpoints to shut down +49:38.874 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:37.547Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.566Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.569 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4c381542-64eb-5bd1-c5b5-f5019201b067 Address:127.0.0.1:23979}]" +49:37.570 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4c381542-64eb-5bd1-c5b5-f5019201b067.dc1 127.0.0.1 +49:37.570 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4c381542-64eb-5bd1-c5b5-f5019201b067 127.0.0.1 +2022-12-04T05:49:37.570Z [INFO] agent.router: Initializing LAN area manager +49:37.571 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23995 network=udp +49:37.571 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23979 [Follower]" leader= +49:37.571 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23995 network=tcp +49:37.571 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4c381542-64eb-5bd1-c5b5-f5019201b067.dc1 area=wan +49:37.571 [INFO] TestAgent.server: Adding LAN server: server="Node-4c381542-64eb-5bd1-c5b5-f5019201b067 (Addr: tcp/127.0.0.1:23979) (DC: dc1)" +49:37.572 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23975 network=tcp +49:37.572 [INFO] TestAgent: started state syncer +49:37.572 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23300 network=tcp +49:37.618 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:37.618 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23979 [Candidate]" term=2 +49:37.618 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:37.618 [DEBUG] TestAgent.server.raft: vote granted: from=4c381542-64eb-5bd1-c5b5-f5019201b067 term=2 tally=1 +49:37.618 [INFO] TestAgent.server.raft: election won: tally=1 +49:37.618 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23979 [Leader]" +49:37.618 [INFO] TestAgent.server: cluster leadership acquired +49:37.619 [INFO] TestAgent.server: New leader elected: payload=Node-4c381542-64eb-5bd1-c5b5-f5019201b067 +49:37.619 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23979 +49:37.619 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:37.621 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:37.621 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:37.621 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:37.621 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:37.621 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:37.621 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4c381542-64eb-5bd1-c5b5-f5019201b067 +49:37.621 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4c381542-64eb-5bd1-c5b5-f5019201b067 +49:37.622 [INFO] TestAgent.server: federation state anti-entropy synced +49:37.976 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:37.976 [INFO] TestAgent: Synced node info +49:38.369 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:24247] +49:38.372 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:24247 +49:38.373 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-57e070de-d565-e7e4-4127-85c382a8eb71 127.0.0.1 +49:38.373 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +49:38.373 [DEBUG] TestAgent: systemd notify failed: error="No socket" +49:38.373 [INFO] TestAgent.server: Adding LAN server: server="Node-57e070de-d565-e7e4-4127-85c382a8eb71 (Addr: tcp/127.0.0.1:24249) (DC: dc1)" +49:38.373 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-57e070de-d565-e7e4-4127-85c382a8eb71 other=Node-4c381542-64eb-5bd1-c5b5-f5019201b067 +49:38.373 [INFO] TestAgent.server: member joined, marking health alive: member=Node-57e070de-d565-e7e4-4127-85c382a8eb71 +49:38.374 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:24248: connect: connection refused +49:38.374 [DEBUG] TestAgent.server: Failed to flood-join server at address: server=Node-57e070de-d565-e7e4-4127-85c382a8eb71.dc1 address=127.0.0.1:24248 error="1 error occurred: + * Failed to join 127.0.0.1: dial tcp 127.0.0.1:24248: connect: connection refused + +" +49:38.609 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:38.609 [DEBUG] TestAgent: Node info in sync +49:38.609 [DEBUG] TestAgent: Node info in sync +49:39.071 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-57e070de-d565-e7e4-4127-85c382a8eb71 (timeout reached) +49:39.571 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-57e070de-d565-e7e4-4127-85c382a8eb71 has failed, no acks received +49:39.620 [WARN] TestAgent: error getting server health from server: server=Node-57e070de-d565-e7e4-4127-85c382a8eb71 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:24249: connect: connection refused" +49:40.619 [WARN] TestAgent: error getting server health from server: server=Node-57e070de-d565-e7e4-4127-85c382a8eb71 error="context deadline exceeded" +49:41.071 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-57e070de-d565-e7e4-4127-85c382a8eb71 (timeout reached) +49:41.619 [WARN] TestAgent: error getting server health from server: server=Node-57e070de-d565-e7e4-4127-85c382a8eb71 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:24249: connect: connection refused" +49:42.572 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-57e070de-d565-e7e4-4127-85c382a8eb71 has failed, no acks received +49:42.620 [WARN] TestAgent: error getting server health from server: server=Node-57e070de-d565-e7e4-4127-85c382a8eb71 error="context deadline exceeded" +49:43.571 [INFO] TestAgent.server.memberlist.lan: memberlist: Marking Node-57e070de-d565-e7e4-4127-85c382a8eb71 as failed, suspect timeout reached (0 peer confirmations) +49:43.571 [INFO] TestAgent.server.serf.lan: serf: EventMemberFailed: Node-57e070de-d565-e7e4-4127-85c382a8eb71 127.0.0.1 +49:43.571 [INFO] TestAgent.server: Removing LAN server: server="Node-57e070de-d565-e7e4-4127-85c382a8eb71 (Addr: tcp/127.0.0.1:24249) (DC: dc1)" +49:43.571 [INFO] TestAgent.server: member failed, marking health critical: member=Node-57e070de-d565-e7e4-4127-85c382a8eb71 +49:43.582 [INFO] TestAgent: Force leaving node: node=Node-57e070de-d565-e7e4-4127-85c382a8eb71 +49:43.582 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave (forced): Node-57e070de-d565-e7e4-4127-85c382a8eb71 127.0.0.1 +49:43.582 [INFO] TestAgent: Requesting shutdown +49:43.582 [INFO] TestAgent.server: Removing LAN server: server="Node-57e070de-d565-e7e4-4127-85c382a8eb71 (Addr: tcp/127.0.0.1:24249) (DC: dc1)" +49:43.582 [INFO] TestAgent.server: shutting down server +49:43.582 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.582 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.582 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.582 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.582 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.582 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.582 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.582 [INFO] TestAgent.server: deregistering member: member=Node-57e070de-d565-e7e4-4127-85c382a8eb71 reason=left +49:43.582 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:43.582 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.582 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.582 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:49:43.582Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.582Z [INFO] agent.router.manager: shutting down +49:43.583 [INFO] TestAgent: consul server down +49:43.583 [INFO] TestAgent: shutdown complete +49:43.583 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23995 network=tcp +49:43.583 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23995 network=udp +49:43.583 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23975 network=tcp +49:44.071 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-57e070de-d565-e7e4-4127-85c382a8eb71 (timeout reached) +49:44.083 [INFO] TestAgent: Waiting for endpoints to shut down +49:44.083 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ForceLeave (6.56s) +2022-12-04T05:49:43.342Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.354Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.356 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a17e6c85-2bdc-9cb6-1159-e65264d59ecf Address:127.0.0.1:23911}]" +49:43.356 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23911 [Follower]" leader= +49:43.356 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.357 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.357Z [INFO] agent.router: Initializing LAN area manager +49:43.357 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23911) (DC: dc1)" +49:43.357 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.357 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23934 network=udp +49:43.357 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23934 network=tcp +49:43.358 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23935 network=tcp +49:43.358 [INFO] TestAgent: started state syncer +49:43.358 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23898 network=tcp +49:43.410 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:43.410 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23911 [Candidate]" term=2 +49:43.410 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:43.410 [DEBUG] TestAgent.server.raft: vote granted: from=a17e6c85-2bdc-9cb6-1159-e65264d59ecf term=2 tally=1 +49:43.410 [INFO] TestAgent.server.raft: election won: tally=1 +49:43.410 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23911 [Leader]" +49:43.410 [INFO] TestAgent.server: cluster leadership acquired +49:43.411 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23911 +49:43.411 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:43.412 [INFO] TestAgent.server: New leader elected: payload=test-node +49:43.412 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:43.412 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:43.412 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:43.412 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:43.412 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:43.413 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:43.413 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:43.413 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.642 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.314331ms client=127.0.0.1:57507 client_network=udp +49:43.643 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.058464ms client=127.0.0.1:59362 client_network=udp +49:43.644 [DEBUG] TestAgent.dns: request served from client: name=1ac511cf-07e1-d91e-eb72-777e17dddc63.query.consul. type=ANY class=IN latency=1.473032ms client=127.0.0.1:43028 client_network=udp +49:43.645 [INFO] TestAgent: Requesting shutdown +49:43.645 [INFO] TestAgent.server: shutting down server +49:43.645 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.645 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.645 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.645 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.645 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.645 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.645 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.645 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:43.645 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.645 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.645 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.645 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:43.645Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.645Z [INFO] agent.router.manager: shutting down +49:43.645 [INFO] TestAgent: consul server down +49:43.645 [INFO] TestAgent: shutdown complete +49:43.645 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23934 network=tcp +49:43.645 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23934 network=udp +49:43.645 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23935 network=tcp +49:44.145 [INFO] TestAgent: Waiting for endpoints to shut down +49:44.145 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:43.513Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.525Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.527 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d1b436da-166a-5816-8f6d-41f871aac75f Address:127.0.0.1:23932}]" +49:43.527 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23932 [Follower]" leader= +49:43.527 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.527 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.527Z [INFO] agent.router: Initializing LAN area manager +49:43.528 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23403 network=udp +49:43.528 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.528 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23932) (DC: dc1)" +49:43.528 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23403 network=tcp +49:43.528 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23404 network=tcp +49:43.528 [INFO] TestAgent: started state syncer +49:43.529 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23926 network=tcp +49:43.583 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:43.583 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23932 [Candidate]" term=2 +49:43.583 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:43.583 [DEBUG] TestAgent.server.raft: vote granted: from=d1b436da-166a-5816-8f6d-41f871aac75f term=2 tally=1 +49:43.583 [INFO] TestAgent.server.raft: election won: tally=1 +49:43.583 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23932 [Leader]" +49:43.583 [INFO] TestAgent.server: cluster leadership acquired +49:43.583 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23932 +49:43.583 [INFO] TestAgent.server: New leader elected: payload=test-node +49:43.584 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:43.585 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:43.585 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:43.585 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:43.585 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:43.585 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:43.585 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:43.585 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:43.586 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.669 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.080476ms client=127.0.0.1:34483 client_network=udp +49:43.670 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=1.08231ms client=127.0.0.1:59838 client_network=udp +49:43.671 [DEBUG] TestAgent.dns: request served from client: name=7609102a-4913-5302-f929-628a28eae622.query.consul. type=ANY class=IN latency=884.319µs client=127.0.0.1:52314 client_network=udp +49:43.671 [INFO] TestAgent: Requesting shutdown +49:43.671 [INFO] TestAgent.server: shutting down server +49:43.671 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.671 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.671 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.671 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.671 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.671 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.671 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.671 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.671 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.671 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:43.671 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.671 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:43.671Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.671Z [INFO] agent.router.manager: shutting down +49:43.672 [INFO] TestAgent: consul server down +49:43.672 [INFO] TestAgent: shutdown complete +49:43.672 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23403 network=tcp +49:43.672 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23403 network=udp +49:43.672 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23404 network=tcp +49:44.172 [INFO] TestAgent: Waiting for endpoints to shut down +49:44.172 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:43.347Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.360Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.361 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:894a1e20-0d29-33c1-2521-1f90a82e7304 Address:127.0.0.1:23967}]" +49:43.361 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23967 [Follower]" leader= +49:43.362 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.362 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.362Z [INFO] agent.router: Initializing LAN area manager +49:43.362 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23906 network=udp +49:43.362 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23967) (DC: dc1)" +49:43.363 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.363 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23906 network=tcp +49:43.367 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23907 network=tcp +49:43.367 [INFO] TestAgent: started state syncer +49:43.367 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23402 network=tcp +49:43.431 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:43.431 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23967 [Candidate]" term=2 +49:43.432 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:43.432 [DEBUG] TestAgent.server.raft: vote granted: from=894a1e20-0d29-33c1-2521-1f90a82e7304 term=2 tally=1 +49:43.432 [INFO] TestAgent.server.raft: election won: tally=1 +49:43.432 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23967 [Leader]" +49:43.432 [INFO] TestAgent.server: cluster leadership acquired +49:43.432 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23967 +49:43.432 [INFO] TestAgent.server: New leader elected: payload=test-node +49:43.433 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:43.435 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:43.435 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:43.435 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:43.435 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:43.435 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:43.435 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:43.435 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:43.436 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.637 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:43.637 [INFO] TestAgent: Synced node info +49:43.677 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=811.152µs client=127.0.0.1:42445 client_network=udp +49:43.678 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=650.685µs client=127.0.0.1:43648 client_network=udp +49:43.678 [DEBUG] TestAgent.dns: request served from client: name=82559d77-500e-0aaf-9555-8ead2e8b1877.query.consul. type=A class=IN latency=704.664µs client=127.0.0.1:49590 client_network=udp +49:43.678 [INFO] TestAgent: Requesting shutdown +49:43.679 [INFO] TestAgent.server: shutting down server +49:43.679 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.679 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.679 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.679 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.679 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.679 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.679 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.679 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.679 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.679 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:43.679 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:49:43.679Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.679Z [INFO] agent.router.manager: shutting down +49:43.679 [INFO] TestAgent: consul server down +49:43.679 [INFO] TestAgent: shutdown complete +49:43.679 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23906 network=tcp +49:43.679 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23906 network=udp +49:43.679 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23907 network=tcp +49:44.179 [INFO] TestAgent: Waiting for endpoints to shut down +49:44.179 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:43.532Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.541Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.542 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:467dab82-224d-c90f-d568-2c52332c650b Address:127.0.0.1:23758}]" +49:43.543 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23758 [Follower]" leader= +49:43.543 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.543 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.543Z [INFO] agent.router: Initializing LAN area manager +49:43.543 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23758) (DC: dc1)" +49:43.544 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.544 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23927 network=udp +49:43.544 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23927 network=tcp +49:43.544 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23928 network=tcp +49:43.544 [INFO] TestAgent: started state syncer +49:43.544 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23807 network=tcp +49:43.612 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:43.612 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23758 [Candidate]" term=2 +49:43.612 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:43.612 [DEBUG] TestAgent.server.raft: vote granted: from=467dab82-224d-c90f-d568-2c52332c650b term=2 tally=1 +49:43.612 [INFO] TestAgent.server.raft: election won: tally=1 +49:43.612 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23758 [Leader]" +49:43.612 [INFO] TestAgent.server: cluster leadership acquired +49:43.612 [INFO] TestAgent.server: New leader elected: payload=test-node +49:43.612 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23758 +49:43.613 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:43.614 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:43.614 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:43.614 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:43.614 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:43.614 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:43.614 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:43.614 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:43.614 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.785 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.178771ms client=127.0.0.1:34231 client_network=udp +49:43.786 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=1.102402ms client=127.0.0.1:56826 client_network=udp +49:43.787 [DEBUG] TestAgent.dns: request served from client: name=7dd49aa5-0287-d279-0b90-8734e1a74fca.query.consul. type=A class=IN latency=865.174µs client=127.0.0.1:53217 client_network=udp +49:43.787 [INFO] TestAgent: Requesting shutdown +49:43.788 [INFO] TestAgent.server: shutting down server +49:43.788 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.788 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.788 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.788 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.788 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.788 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.788 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.788 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:43.788 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.788 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.788 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.788 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:43.788Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.788Z [INFO] agent.router.manager: shutting down +49:43.788 [INFO] TestAgent: consul server down +49:43.788 [INFO] TestAgent: shutdown complete +49:43.788 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23927 network=tcp +49:43.788 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23927 network=udp +49:43.788 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23928 network=tcp +49:44.288 [INFO] TestAgent: Waiting for endpoints to shut down +49:44.288 [INFO] TestAgent: Endpoints down +[WARN] freeport: 10 out of 52 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:43.793Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.803Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.804 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e8cbae23-08b3-e4bc-7ac9-74a58ebd5852 Address:127.0.0.1:23925}]" +49:43.804 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23925 [Follower]" leader= +49:43.804 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.805 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.805Z [INFO] agent.router: Initializing LAN area manager +49:43.805 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.805 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23925) (DC: dc1)" +49:43.805 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23948 network=udp +49:43.805 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23948 network=tcp +49:43.806 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23949 network=tcp +49:43.806 [INFO] TestAgent: started state syncer +49:43.806 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23912 network=tcp +49:43.845 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:43.845 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23925 [Candidate]" term=2 +49:43.845 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:43.845 [DEBUG] TestAgent.server.raft: vote granted: from=e8cbae23-08b3-e4bc-7ac9-74a58ebd5852 term=2 tally=1 +49:43.845 [INFO] TestAgent.server.raft: election won: tally=1 +49:43.845 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23925 [Leader]" +49:43.845 [INFO] TestAgent.server: cluster leadership acquired +49:43.845 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23925 +49:43.846 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:43.846 [INFO] TestAgent.server: New leader elected: payload=test-node +49:43.847 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:43.847 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:43.847 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:43.847 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:43.847 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:43.847 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:43.847 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:43.847 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.865 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:43.865 [INFO] TestAgent: Synced node info +49:43.865 [DEBUG] TestAgent: Node info in sync +49:43.909 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.097814ms client=127.0.0.1:57466 client_network=udp +49:43.910 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=790.269µs client=127.0.0.1:59670 client_network=udp +49:43.911 [DEBUG] TestAgent.dns: request served from client: name=a3068089-984f-f433-4f80-0fbd5dbacc9a.query.consul. type=ANY class=IN latency=887.161µs client=127.0.0.1:46933 client_network=udp +49:43.911 [INFO] TestAgent: Requesting shutdown +49:43.911 [INFO] TestAgent.server: shutting down server +49:43.911 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.911 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.911 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.911 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.911 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.911 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.911 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.911 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.911 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.911 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.911 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:43.912Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.912Z [INFO] agent.router.manager: shutting down +49:43.912 [INFO] TestAgent: consul server down +49:43.912 [INFO] TestAgent: shutdown complete +49:43.912 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23948 network=tcp +49:43.912 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23948 network=udp +49:43.912 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23949 network=tcp +49:44.412 [INFO] TestAgent: Waiting for endpoints to shut down +49:44.412 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:43.639Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.649Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.650 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:947b3c42-d543-abc9-e30e-d1477d54d41f Address:127.0.0.1:23953}]" +49:43.650 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23953 [Follower]" leader= +49:43.651 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.651 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.651Z [INFO] agent.router: Initializing LAN area manager +49:43.651 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23953) (DC: dc1)" +49:43.651 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23808 network=udp +49:43.651 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.651 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23808 network=tcp +49:43.652 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23809 network=tcp +49:43.652 [INFO] TestAgent: started state syncer +49:43.652 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23968 network=tcp +49:43.717 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:43.717 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23953 [Candidate]" term=2 +49:43.717 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:43.717 [DEBUG] TestAgent.server.raft: vote granted: from=947b3c42-d543-abc9-e30e-d1477d54d41f term=2 tally=1 +49:43.717 [INFO] TestAgent.server.raft: election won: tally=1 +49:43.717 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23953 [Leader]" +49:43.717 [INFO] TestAgent.server: cluster leadership acquired +49:43.718 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23953 +49:43.719 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:43.720 [INFO] TestAgent.server: New leader elected: payload=test-node +49:43.729 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:43.729 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:43.729 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:43.729 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:43.729 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:43.729 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:43.729 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:43.730 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.776 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:43.776 [INFO] TestAgent: Synced node info +49:43.957 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=922.715µs client=127.0.0.1:32795 client_network=udp +49:43.957 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=675.424µs client=127.0.0.1:52442 client_network=udp +49:43.965 [DEBUG] TestAgent.dns: request served from client: name=e30a9a31-dab2-010c-f77b-aa312d5214b1.query.consul. type=A class=IN latency=7.674084ms client=127.0.0.1:51812 client_network=udp +49:43.965 [INFO] TestAgent: Requesting shutdown +49:43.966 [INFO] TestAgent.server: shutting down server +49:43.966 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.966 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.966 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.966 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.966 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.966 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.966 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.966 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.966 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.966 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.966 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:43.966Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.966Z [INFO] agent.router.manager: shutting down +49:43.967 [INFO] TestAgent: consul server down +49:43.967 [INFO] TestAgent: shutdown complete +49:43.967 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23808 network=tcp +49:43.967 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23808 network=udp +49:43.967 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23809 network=tcp +49:44.467 [INFO] TestAgent: Waiting for endpoints to shut down +49:44.467 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:43.754Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.767Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.768 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d9e0c48e-2871-1daa-a855-cbaed7fe0374 Address:127.0.0.1:23939}]" +49:43.768 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23939 [Follower]" leader= +49:43.769 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.769 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.769Z [INFO] agent.router: Initializing LAN area manager +49:43.770 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23939) (DC: dc1)" +49:43.770 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.770 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23941 network=udp +49:43.770 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23941 network=tcp +49:43.770 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23942 network=tcp +49:43.770 [INFO] TestAgent: started state syncer +49:43.770 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23989 network=tcp +49:43.837 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:43.837 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23939 [Candidate]" term=2 +49:43.837 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:43.837 [DEBUG] TestAgent.server.raft: vote granted: from=d9e0c48e-2871-1daa-a855-cbaed7fe0374 term=2 tally=1 +49:43.837 [INFO] TestAgent.server.raft: election won: tally=1 +49:43.837 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23939 [Leader]" +49:43.837 [INFO] TestAgent.server: cluster leadership acquired +49:43.837 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23939 +49:43.838 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:43.838 [INFO] TestAgent.server: New leader elected: payload=test-node +49:43.839 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:43.839 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:43.839 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:43.839 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:43.839 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:43.839 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:43.839 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:43.840 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.973 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=2.593838ms client=127.0.0.1:56717 client_network=udp +49:43.979 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=2.341722ms client=127.0.0.1:54560 client_network=udp +49:43.982 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:43.982 [INFO] TestAgent: Synced node info +49:43.982 [DEBUG] TestAgent: Node info in sync +49:43.983 [DEBUG] TestAgent.dns: request served from client: name=315fead1-e5e9-076d-02c3-df93918420d9.query.consul. type=AAAA class=IN latency=1.212271ms client=127.0.0.1:43643 client_network=udp +49:43.984 [INFO] TestAgent: Requesting shutdown +49:43.984 [INFO] TestAgent.server: shutting down server +49:43.984 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.984 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.984 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.984 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.984 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.984 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.984 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.984 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.984 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.984 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.984 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:43.984Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.984Z [INFO] agent.router.manager: shutting down +49:43.984 [INFO] TestAgent: consul server down +49:43.984 [INFO] TestAgent: shutdown complete +49:43.984 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23941 network=tcp +49:43.984 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23941 network=udp +49:43.984 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23942 network=tcp +49:44.484 [INFO] TestAgent: Waiting for endpoints to shut down +49:44.484 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:43.773Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.784Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.786 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cc9a842e-7841-1daf-cbc5-1336840c5475 Address:127.0.0.1:23359}]" +49:43.786 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23359 [Follower]" leader= +49:43.786 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.787 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.787Z [INFO] agent.router: Initializing LAN area manager +49:43.787 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.787 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23359) (DC: dc1)" +49:43.787 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23990 network=udp +49:43.787 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23990 network=tcp +49:43.788 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23991 network=tcp +49:43.788 [INFO] TestAgent: started state syncer +49:43.788 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23947 network=tcp +49:43.852 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:43.853 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23359 [Candidate]" term=2 +49:43.853 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:43.853 [DEBUG] TestAgent.server.raft: vote granted: from=cc9a842e-7841-1daf-cbc5-1336840c5475 term=2 tally=1 +49:43.853 [INFO] TestAgent.server.raft: election won: tally=1 +49:43.853 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23359 [Leader]" +49:43.853 [INFO] TestAgent.server: cluster leadership acquired +49:43.853 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23359 +49:43.853 [INFO] TestAgent.server: New leader elected: payload=test-node +49:43.853 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:43.854 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:43.854 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:43.854 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:43.854 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:43.854 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:43.855 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:43.855 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:43.855 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.985 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=AAAA class=IN latency=966.355µs client=127.0.0.1:53512 client_network=udp +49:43.986 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=AAAA class=IN latency=891.028µs client=127.0.0.1:40658 client_network=udp +49:43.988 [DEBUG] TestAgent.dns: request served from client: name=71337769-3dff-9ec2-f01a-2101f5ccf54f.query.consul. type=AAAA class=IN latency=968.478µs client=127.0.0.1:38439 client_network=udp +49:43.988 [INFO] TestAgent: Requesting shutdown +49:43.988 [INFO] TestAgent.server: shutting down server +49:43.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.988 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.988 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.988 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.988 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:43.988 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.988 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.988 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.988 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.988 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.988 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:43.988Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.988Z [INFO] agent.router.manager: shutting down +49:43.988 [INFO] TestAgent: consul server down +49:43.988 [INFO] TestAgent: shutdown complete +49:43.988 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23990 network=tcp +49:43.988 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23990 network=udp +49:43.988 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23991 network=tcp +49:44.488 [INFO] TestAgent: Waiting for endpoints to shut down +49:44.488 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:43.682Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.695Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.705 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fb8b7324-250f-3b87-b2af-888ff5cb9917 Address:127.0.0.1:23981}]" +49:43.705 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23981 [Follower]" leader= +49:43.706 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.706 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.706Z [INFO] agent.router: Initializing LAN area manager +49:43.706 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23981) (DC: dc1)" +49:43.706 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23969 network=udp +49:43.706 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.706 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23969 network=tcp +49:43.707 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23970 network=tcp +49:43.707 [INFO] TestAgent: started state syncer +49:43.713 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23940 network=tcp +49:43.747 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:43.747 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23981 [Candidate]" term=2 +49:43.747 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:43.747 [DEBUG] TestAgent.server.raft: vote granted: from=fb8b7324-250f-3b87-b2af-888ff5cb9917 term=2 tally=1 +49:43.747 [INFO] TestAgent.server.raft: election won: tally=1 +49:43.747 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23981 [Leader]" +49:43.747 [INFO] TestAgent.server: cluster leadership acquired +49:43.747 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23981 +49:43.748 [INFO] TestAgent.server: New leader elected: payload=test-node +49:43.748 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:43.750 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:43.750 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:43.750 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:43.750 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:43.750 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:43.750 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:43.750 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:43.751 [INFO] TestAgent.server: federation state anti-entropy synced +49:44.006 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=1.190303ms client=127.0.0.1:40834 client_network=udp +49:44.007 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=802.362µs client=127.0.0.1:50405 client_network=udp +49:44.008 [DEBUG] TestAgent.dns: request served from client: name=ff073a26-de23-28c0-16b1-b9fd412ef6a0.query.consul. type=ANY class=IN latency=1.184531ms client=127.0.0.1:33040 client_network=udp +49:44.008 [INFO] TestAgent: Requesting shutdown +49:44.008 [INFO] TestAgent.server: shutting down server +49:44.008 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:44.008 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:44.008 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:44.008 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:44.008 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:44.008 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:44.008 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:44.008 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:44.008 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:44.008 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:44.008 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:44.008 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +2022-12-04T05:49:44.009Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:44.009Z [INFO] agent.router.manager: shutting down +49:44.009 [INFO] TestAgent: consul server down +49:44.009 [INFO] TestAgent: shutdown complete +49:44.009 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23969 network=tcp +49:44.009 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23969 network=udp +49:44.009 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23970 network=tcp +49:44.509 [INFO] TestAgent: Waiting for endpoints to shut down +49:44.509 [INFO] TestAgent: Endpoints down [WARN] freeport: 10 out of 45 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:43.931Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.943Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.950 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:279f9178-613d-3472-d637-706dd5565c4c Address:127.0.0.1:23415}]" +49:43.950 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23415 [Follower]" leader= +49:43.951 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.952 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.952Z [INFO] agent.router: Initializing LAN area manager +49:43.952 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23415) (DC: dc1)" +49:43.952 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.952 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23913 network=tcp +49:43.952 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23913 network=udp +49:43.953 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23914 network=tcp +49:43.953 [INFO] TestAgent: started state syncer +49:43.953 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23339 network=tcp +49:44.000 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:44.000 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23415 [Candidate]" term=2 +49:44.000 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:44.000 [DEBUG] TestAgent.server.raft: vote granted: from=279f9178-613d-3472-d637-706dd5565c4c term=2 tally=1 +49:44.000 [INFO] TestAgent.server.raft: election won: tally=1 +49:44.000 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23415 [Leader]" +49:44.000 [INFO] TestAgent.server: cluster leadership acquired +49:44.000 [INFO] TestAgent.server: New leader elected: payload=test-node +49:44.000 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23415 +49:44.001 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:44.002 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:44.002 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:44.003 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:44.003 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:44.003 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:44.003 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:44.003 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:44.003 [INFO] TestAgent.server: federation state anti-entropy synced +49:44.140 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=A class=IN latency=1.190801ms client=127.0.0.1:41540 client_network=udp +49:44.141 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=A class=IN latency=877.522µs client=127.0.0.1:51338 client_network=udp +49:44.142 [DEBUG] TestAgent.dns: request served from client: name=44a962a6-d766-2c87-6f22-7cd0268aee64.query.consul. type=A class=IN latency=963.94µs client=127.0.0.1:44746 client_network=udp +49:44.142 [INFO] TestAgent: Requesting shutdown +49:44.143 [INFO] TestAgent.server: shutting down server +49:44.143 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:44.143 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:44.143 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:44.143 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:44.143 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:44.143 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:44.143 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:44.143 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:44.143 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:44.143 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:44.143 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:44.143 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:44.143Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:44.143Z [INFO] agent.router.manager: shutting down +49:44.143 [INFO] TestAgent: consul server down +49:44.143 [INFO] TestAgent: shutdown complete +49:44.143 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23913 network=tcp +49:44.143 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23913 network=udp +49:44.143 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23914 network=tcp +49:44.643 [INFO] TestAgent: Waiting for endpoints to shut down +49:44.643 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:43.948Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.967Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.973 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:801ffc24-97e8-a5c2-44c7-ccb9cb8a4fcc Address:127.0.0.1:23876}]" +49:43.973 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23876 [Follower]" leader= +49:43.977 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:43.978 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:43.978Z [INFO] agent.router: Initializing LAN area manager +49:43.978 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23340 network=udp +49:43.981 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:43.981 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23340 network=tcp +49:43.982 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23341 network=tcp +49:43.982 [INFO] TestAgent: started state syncer +49:43.982 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23982 network=tcp +49:43.982 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:23876) (DC: dc1)" +49:44.030 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:44.030 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23876 [Candidate]" term=2 +49:44.031 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:44.031 [DEBUG] TestAgent.server.raft: vote granted: from=801ffc24-97e8-a5c2-44c7-ccb9cb8a4fcc term=2 tally=1 +49:44.031 [INFO] TestAgent.server.raft: election won: tally=1 +49:44.031 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23876 [Leader]" +49:44.031 [INFO] TestAgent.server: cluster leadership acquired +49:44.031 [INFO] TestAgent.server: New leader elected: payload=test-node +49:44.031 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23876 +49:44.032 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:44.033 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:44.033 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:44.033 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:44.033 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:44.033 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:44.033 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:44.033 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:44.033 [INFO] TestAgent.server: federation state anti-entropy synced +49:44.195 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:44.196 [INFO] TestAgent: Synced node info +49:44.196 [DEBUG] TestAgent: Node info in sync +49:44.297 [DEBUG] TestAgent.dns: request served from client: name=api-tier.service.consul. type=ANY class=IN latency=970.115µs client=127.0.0.1:41678 client_network=udp +49:44.298 [DEBUG] TestAgent.dns: request served from client: name=api-tier.query.consul. type=ANY class=IN latency=796.856µs client=127.0.0.1:53106 client_network=udp +49:44.300 [DEBUG] TestAgent.dns: request served from client: name=cc8710a7-724f-878e-daff-de181e025975.query.consul. type=ANY class=IN latency=877.211µs client=127.0.0.1:38637 client_network=udp +49:44.300 [INFO] TestAgent: Requesting shutdown +49:44.300 [INFO] TestAgent.server: shutting down server +49:44.300 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:44.300 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:44.300 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:44.300 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:44.300 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:44.300 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:44.300 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:44.300 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:44.300 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:44.300 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:44.300 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:44.300Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:44.300Z [INFO] agent.router.manager: shutting down +49:44.300 [INFO] TestAgent: consul server down +49:44.300 [INFO] TestAgent: shutdown complete +49:44.300 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23340 network=tcp +49:44.300 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23340 network=udp +49:44.300 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23341 network=tcp +49:44.801 [INFO] TestAgent: Waiting for endpoints to shut down +49:44.801 [INFO] TestAgent: Endpoints down +--- PASS: TestDNS_ServiceLookup_AnswerLimits (0.00s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{0_0_0_0_0_0_0_0_0_0_0} (0.75s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{6_6_6_6_6_6_6_5_6_6_-5} (0.89s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{5_5_5_5_5_5_5_5_5_5_5} (0.75s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{5_5_5_5_5_5_5_5_5_5_5} (0.85s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{5_5_5_5_5_5_5_5_5_5_5} (0.68s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{4_4_4_4_4_4_4_4_4_4_4} (0.70s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{3_3_3_3_3_3_3_3_3_3_3} (0.63s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{4_4_4_4_4_4_4_4_4_4_4} (0.76s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{4_4_4_4_4_4_4_4_4_4_4} (0.97s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{3_3_3_3_3_3_3_3_3_3_3} (0.88s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{3_3_3_3_3_3_3_3_3_3_3} (0.82s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{2_2_2_2_2_2_2_2_2_2_2} (0.63s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{2_2_2_2_2_2_2_2_2_2_2} (0.95s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{1_1_1_1_1_1_1_1_1_1_1} (0.75s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{1_1_1_1_1_1_1_1_1_1_1} (0.76s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{2_2_2_2_2_2_2_2_2_2_2} (0.87s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{6_6_6_6_6_6_6_5_6_6_-5} (0.96s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{1_1_1_1_1_1_1_1_1_1_1} (1.04s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{0_0_0_0_0_0_0_0_0_0_0} (0.83s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} (0.65s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} (0.68s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{9_9_8_8_6_8_8_5_8_8_-5} (0.70s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{30_30_8_8_6_8_8_5_8_-5_-5} (0.67s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{0_0_0_0_0_0_0_0_0_0_0} (0.94s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} (0.64s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} (0.82s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{9_9_8_8_6_8_8_5_8_8_-5} (0.67s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{20_20_8_8_6_8_8_5_8_-5_-5} (0.85s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{8_8_8_8_6_8_8_5_8_8_-5} (0.77s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{7_7_7_7_6_7_7_5_7_7_-5} (0.63s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{9_9_8_8_6_8_8_5_8_8_-5} (0.84s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{8_8_8_8_6_8_8_5_8_8_-5} (0.74s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/AAAA_lookup_{7_7_7_7_6_7_7_5_7_7_-5} (0.73s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{8_8_8_8_6_8_8_5_8_8_-5} (0.84s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/A_lookup_{7_7_7_7_6_7_7_5_7_7_-5} (0.72s) + --- PASS: TestDNS_ServiceLookup_AnswerLimits/ANY_lookup_{6_6_6_6_6_6_6_5_6_6_-5} (0.86s) +[WARN] freeport: 10 out of 24 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=true#01 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=false#01 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=1)_compressed=true#01 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=1)_compressed=false#01 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=1)_compressed=true#01 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=1)_compressed=false#01 -[WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=true#01 +2022-12-04T05:49:33.570Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:33.582Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:33.584 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aad9f172-f308-21b8-56b6-f39258c2608d Address:127.0.0.1:23965}]" +49:33.584 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23965 [Follower]" leader= +49:33.584 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aad9f172-f308-21b8-56b6-f39258c2608d.dc1 127.0.0.1 +49:33.585 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aad9f172-f308-21b8-56b6-f39258c2608d 127.0.0.1 +2022-12-04T05:49:33.585Z [INFO] agent.router: Initializing LAN area manager +49:33.585 [INFO] TestAgent.server: Adding LAN server: server="Node-aad9f172-f308-21b8-56b6-f39258c2608d (Addr: tcp/127.0.0.1:23965) (DC: dc1)" +49:33.585 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23904 network=udp +49:33.585 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aad9f172-f308-21b8-56b6-f39258c2608d.dc1 area=wan +49:33.585 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23904 network=tcp +49:33.586 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23961 network=tcp +49:33.586 [INFO] TestAgent: started state syncer +49:33.586 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23917 network=tcp +49:33.644 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:33.645 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23965 [Candidate]" term=2 +49:33.645 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:33.645 [DEBUG] TestAgent.server.raft: vote granted: from=aad9f172-f308-21b8-56b6-f39258c2608d term=2 tally=1 +49:33.645 [INFO] TestAgent.server.raft: election won: tally=1 +49:33.645 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23965 [Leader]" +49:33.645 [INFO] TestAgent.server: cluster leadership acquired +49:33.645 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23965 +49:33.645 [INFO] TestAgent.server: New leader elected: payload=Node-aad9f172-f308-21b8-56b6-f39258c2608d +49:33.646 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:33.648 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:33.648 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:33.648 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:33.648 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:33.648 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:33.648 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aad9f172-f308-21b8-56b6-f39258c2608d +49:33.648 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aad9f172-f308-21b8-56b6-f39258c2608d +49:33.648 [INFO] TestAgent.server: federation state anti-entropy synced +49:33.713 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:33.714 [INFO] TestAgent: Synced node info +49:33.714 [DEBUG] TestAgent: Node info in sync +49:33.920 [INFO] TestAgent.server: server starting leave +49:33.920 [INFO] TestAgent.server.serf.wan: serf: EventMemberLeave: Node-aad9f172-f308-21b8-56b6-f39258c2608d.dc1 127.0.0.1 +49:33.921 [INFO] TestAgent.server: Handled event for server in area: event=member-leave server=Node-aad9f172-f308-21b8-56b6-f39258c2608d.dc1 area=wan +2022-12-04T05:49:33.921Z [INFO] agent.router.manager: shutting down +49:35.087 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.087 [DEBUG] TestAgent: Node info in sync +49:36.924 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-aad9f172-f308-21b8-56b6-f39258c2608d 127.0.0.1 +49:36.924 [INFO] TestAgent.server: Removing LAN server: server="Node-aad9f172-f308-21b8-56b6-f39258c2608d (Addr: tcp/127.0.0.1:23965) (DC: dc1)" +49:36.924 [WARN] TestAgent.server: deregistering self should be done by follower: name=Node-aad9f172-f308-21b8-56b6-f39258c2608d +49:37.648 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +49:39.648 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +49:39.925 [INFO] TestAgent.server: Waiting to drain RPC traffic: drain_time=5s +49:41.648 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +49:43.648 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +49:43.648 [ERROR] TestAgent.server.autopilot: Error promoting servers: error="error getting server raft protocol versions: No servers found" +49:44.925 [INFO] TestAgent: Requesting shutdown +49:44.925 [INFO] TestAgent.server: shutting down server +49:44.925 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:44.925 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:44.925 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:44.925 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:44.925 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:44.926 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:44.926 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:44.926 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:44.926 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:44.926Z [INFO] agent.router.manager: shutting down +49:44.926 [INFO] TestAgent: consul server down +49:44.926 [INFO] TestAgent: shutdown complete +49:44.926 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23904 network=tcp +49:44.926 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23904 network=udp +49:44.926 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23961 network=tcp +49:45.426 [INFO] TestAgent: Waiting for endpoints to shut down +49:45.426 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_StartStop (11.87s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=false#01 [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=true#01 [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=false#01 [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=33)_compressed=true#01 [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=33)_compressed=false#01 [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! +2022-12-04T05:49:37.581Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:37.581Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.596Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:49:37.596Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.598 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:38425ea2-e08d-0bc1-635f-78f75fb32ee8 Address:127.0.0.1:23383}]" +49:37.598 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23383 [Follower]" leader= +49:37.599 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8.dc1 127.0.0.1 +49:37.599 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8 127.0.0.1 +2022-12-04T05:49:37.599Z [INFO] agent.router: Initializing LAN area manager +49:37.600 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23301 network=udp +49:37.600 [INFO] TestAgent.server: Adding LAN server: server="Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8 (Addr: tcp/127.0.0.1:23383) (DC: dc1)" +49:37.600 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8.dc1 area=wan +49:37.600 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23301 network=tcp +49:37.600 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23302 network=tcp +49:37.601 [INFO] TestAgent: started state syncer +49:37.601 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23356 network=tcp +49:37.649 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:37.649 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23383 [Candidate]" term=2 +49:37.649 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:37.649 [DEBUG] TestAgent.server.raft: vote granted: from=38425ea2-e08d-0bc1-635f-78f75fb32ee8 term=2 tally=1 +49:37.649 [INFO] TestAgent.server.raft: election won: tally=1 +49:37.649 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23383 [Leader]" +49:37.649 [INFO] TestAgent.server: cluster leadership acquired +49:37.649 [INFO] TestAgent.server: New leader elected: payload=Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8 +49:37.649 [INFO] TestAgent.server: initializing acls +49:37.649 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:37.649 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:37.650 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:37.650 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:37.650 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:37.650 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:37.650 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8 +49:37.650 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8.dc1 +49:37.650 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:37.651 [INFO] TestAgent.server: Updating LAN server: server="Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8 (Addr: tcp/127.0.0.1:23383) (DC: dc1)" +49:37.651 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8.dc1 area=wan +49:37.652 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:37.652 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:37.652 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:37.652 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:37.652 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:37.652 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8 +49:37.652 [INFO] TestAgent.server: member joined, marking health alive: member=Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8 +49:37.652 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8 +49:37.652 [INFO] TestAgent.server: federation state anti-entropy synced +49:37.830 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8 +49:37.831 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8 +49:37.832 [INFO] TestAgent.server: server starting leave +49:37.832 [INFO] TestAgent.server.serf.wan: serf: EventMemberLeave: Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8.dc1 127.0.0.1 +49:37.832 [INFO] TestAgent.server: Handled event for server in area: event=member-leave server=Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8.dc1 area=wan +2022-12-04T05:49:37.832Z [INFO] agent.router.manager: shutting down +49:38.039 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:38.039 [INFO] TestAgent: Synced node info +49:39.392 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:39.392 [DEBUG] TestAgent: Node info in sync +49:39.392 [DEBUG] TestAgent: Node info in sync +49:40.832 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8 127.0.0.1 +49:40.832 [INFO] TestAgent.server: Removing LAN server: server="Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8 (Addr: tcp/127.0.0.1:23383) (DC: dc1)" +49:40.832 [WARN] TestAgent.server: deregistering self should be done by follower: name=Node-38425ea2-e08d-0bc1-635f-78f75fb32ee8 +49:41.651 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +49:43.651 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +49:43.832 [INFO] TestAgent.server: Waiting to drain RPC traffic: drain_time=5s +49:45.651 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +49:47.651 [ERROR] TestAgent.server.autopilot: Error promoting servers: error="error getting server raft protocol versions: No servers found" +49:47.651 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +49:48.834 [INFO] TestAgent: Requesting shutdown +49:48.834 [INFO] TestAgent.server: shutting down server +49:48.834 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:48.834 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:48.834 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:48.834 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:48.834 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:48.834 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:48.834 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:48.834 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:48.834 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:48.834 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:48.834 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:48.834 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:48.834 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +2022-12-04T05:49:48.834Z [INFO] agent.router.manager: shutting down +49:48.834 [INFO] TestAgent: consul server down +49:48.834 [INFO] TestAgent: shutdown complete +49:48.834 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23301 network=tcp +49:48.835 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23301 network=udp +49:48.835 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23302 network=tcp +49:49.335 [INFO] TestAgent: Waiting for endpoints to shut down +49:49.335 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_Leave_ACLDeny (11.77s) + --- PASS: TestAgent_Leave_ACLDeny/no_token (0.00s) + --- PASS: TestAgent_Leave_ACLDeny/read-only_token (0.00s) + --- PASS: TestAgent_Leave_ACLDeny/agent_master_token (11.00s) +[WARN] freeport: 10 out of 17 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=33)_compressed=true#01 [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -=== RUN TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=33)_compressed=false#01 [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! [WARN] freeport: 10 out of 10 pending ports are still in use; something probably didn't wait around for the port to be closed! -2021-10-31T23:13:11.548Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:11.584Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:11.585 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3232d438-40ea-d395-ed72-c8e1c9bd5d45 Address:127.0.0.1:16917}]" -13:11.585 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16917 [Follower]" leader= -13:11.586 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3232d438-40ea-d395-ed72-c8e1c9bd5d45.dc1 127.0.0.1 -13:11.586 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3232d438-40ea-d395-ed72-c8e1c9bd5d45 127.0.0.1 -2021-10-31T23:13:11.586Z [INFO] agent.router: Initializing LAN area manager -13:11.586 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16912 network=udp -13:11.586 [INFO] TestAgent.server: Adding LAN server: server="Node-3232d438-40ea-d395-ed72-c8e1c9bd5d45 (Addr: tcp/127.0.0.1:16917) (DC: dc1)" -13:11.586 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3232d438-40ea-d395-ed72-c8e1c9bd5d45.dc1 area=wan -13:11.586 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16912 network=tcp -13:11.587 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16913 network=tcp -13:11.587 [INFO] TestAgent: started state syncer -13:11.587 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16904 network=tcp -13:11.629 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:11.630 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16917 [Candidate]" term=2 -13:11.630 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:11.630 [DEBUG] TestAgent.server.raft: vote granted: from=3232d438-40ea-d395-ed72-c8e1c9bd5d45 term=2 tally=1 -13:11.630 [INFO] TestAgent.server.raft: election won: tally=1 -13:11.630 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16917 [Leader]" -13:11.630 [INFO] TestAgent.server: cluster leadership acquired -13:11.630 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16917 -13:11.631 [INFO] TestAgent.server: New leader elected: payload=Node-3232d438-40ea-d395-ed72-c8e1c9bd5d45 -13:11.631 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:11.632 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:11.632 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:11.632 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:11.632 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:11.632 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:11.632 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3232d438-40ea-d395-ed72-c8e1c9bd5d45 -13:11.632 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3232d438-40ea-d395-ed72-c8e1c9bd5d45 -13:11.633 [INFO] TestAgent.server: federation state anti-entropy synced -13:11.737 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:11.737 [INFO] TestAgent: Synced node info -13:11.746 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=207.116µs client=127.0.0.1:38874 client_network=tcp -13:11.747 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=101.448µs client=127.0.0.1:38876 client_network=tcp -13:11.747 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=88.425µs client=127.0.0.1:43102 client_network=udp -13:11.747 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=75.072µs client=127.0.0.1:49677 client_network=udp -13:11.748 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=117.266µs client=127.0.0.1:38878 client_network=tcp -13:11.748 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=103.077µs client=127.0.0.1:38880 client_network=tcp -13:11.748 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=73.231µs client=127.0.0.1:59944 client_network=udp -13:11.749 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=70.391µs client=127.0.0.1:60541 client_network=udp -13:11.749 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=119.626µs client=127.0.0.1:38882 client_network=tcp -13:11.755 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=183.732µs client=127.0.0.1:38884 client_network=tcp -13:11.762 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=244.369µs client=127.0.0.1:46904 client_network=udp -13:11.762 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=90.443µs client=127.0.0.1:60195 client_network=udp -13:11.763 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=265.096µs client=127.0.0.1:38886 client_network=tcp -13:11.770 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=218.475µs client=127.0.0.1:38888 client_network=tcp -13:11.770 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=90.065µs client=127.0.0.1:36659 client_network=udp -13:11.771 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=70.097µs client=127.0.0.1:54382 client_network=udp -13:11.771 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=234.432µs client=127.0.0.1:38890 client_network=tcp -13:11.786 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=205.776µs client=127.0.0.1:38892 client_network=tcp -13:11.788 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=127.505µs client=127.0.0.1:43384 client_network=udp -13:11.788 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=61.795µs client=127.0.0.1:40809 client_network=udp -13:11.788 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=146.034µs client=127.0.0.1:38894 client_network=tcp -13:11.790 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=186.427µs client=127.0.0.1:38896 client_network=tcp -13:11.791 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=89.345µs client=127.0.0.1:38984 client_network=udp -13:11.791 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=60.686µs client=127.0.0.1:42118 client_network=udp -13:11.792 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=248.449µs client=127.0.0.1:38898 client_network=tcp -13:11.803 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=148.844µs client=127.0.0.1:38900 client_network=tcp -13:11.803 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=101.305µs client=127.0.0.1:51708 client_network=udp -13:11.804 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=56.422µs client=127.0.0.1:56486 client_network=udp -13:11.804 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=185.092µs client=127.0.0.1:38902 client_network=tcp -13:11.805 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=306.451µs client=127.0.0.1:38904 client_network=tcp -13:11.806 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=235.315µs client=127.0.0.1:40695 client_network=udp -13:11.814 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=203.927µs client=127.0.0.1:47106 client_network=udp -13:11.815 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=123.191µs client=127.0.0.1:38906 client_network=tcp -13:11.826 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=280.434µs client=127.0.0.1:38908 client_network=tcp -13:11.826 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=78.699µs client=127.0.0.1:37714 client_network=udp -13:11.827 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=67.381µs client=127.0.0.1:38829 client_network=udp -13:11.827 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=114.856µs client=127.0.0.1:38910 client_network=tcp -13:11.828 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=106.234µs client=127.0.0.1:38912 client_network=tcp -13:11.828 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=75.206µs client=127.0.0.1:52266 client_network=udp -13:11.828 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=71.264µs client=127.0.0.1:49136 client_network=udp -13:11.828 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=86.019µs client=127.0.0.1:38914 client_network=tcp -13:11.829 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=455.364µs client=127.0.0.1:38916 client_network=tcp -13:11.830 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=120.328µs client=127.0.0.1:56718 client_network=udp -13:11.830 [DEBUG] TestAgent.dns: request served from client: name=normal.service.consul. type=ANY class=IN latency=68.556µs client=127.0.0.1:54758 client_network=udp -13:11.831 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=114.161µs client=127.0.0.1:38918 client_network=tcp -13:11.831 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=100.805µs client=127.0.0.1:38920 client_network=tcp -13:11.832 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=165.093µs client=127.0.0.1:48723 client_network=udp -13:11.839 [DEBUG] TestAgent.dns: request served from client: name=5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul. type=ANY class=IN latency=204.183µs client=127.0.0.1:36569 client_network=udp -13:13.245 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:13.245 [DEBUG] TestAgent: Node info in sync -13:13.245 [DEBUG] TestAgent: Node info in sync -13:18.994 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -13:18.996 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=398.350368ms client=127.0.0.1:38942 client_network=tcp -13:20.125 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -13:20.131 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=108.934765ms client=127.0.0.1:38950 client_network=tcp -13:21.034 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=220.146653ms client=127.0.0.1:44765 client_network=udp -13:21.227 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=173.067818ms client=127.0.0.1:44060 client_network=udp -13:21.400 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -13:21.406 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=168.487333ms client=127.0.0.1:38962 client_network=tcp -13:22.717 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -13:22.728 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=262.891069ms client=127.0.0.1:38988 client_network=tcp -13:24.079 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=229.153889ms client=127.0.0.1:43270 client_network=udp -13:24.307 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=201.562773ms client=127.0.0.1:55551 client_network=udp -13:24.524 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -13:24.534 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=207.604293ms client=127.0.0.1:38998 client_network=tcp -13:25.772 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -13:25.774 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=132.22862ms client=127.0.0.1:39042 client_network=tcp -13:26.799 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=261.823317ms client=127.0.0.1:60254 client_network=udp -13:27.015 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=173.878718ms client=127.0.0.1:39318 client_network=udp -13:27.203 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -13:27.211 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=165.655602ms client=127.0.0.1:39054 client_network=tcp -13:28.408 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -13:28.411 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=185.798112ms client=127.0.0.1:39064 client_network=tcp -13:29.594 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=155.031061ms client=127.0.0.1:50876 client_network=udp -13:29.789 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=171.629339ms client=127.0.0.1:55102 client_network=udp -13:30.057 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -13:30.062 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=240.407085ms client=127.0.0.1:39066 client_network=tcp -13:30.981 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -13:30.994 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=232.33997ms client=127.0.0.1:39084 client_network=tcp -13:32.006 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=172.592ms client=127.0.0.1:46092 client_network=udp -13:32.339 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=316.084657ms client=127.0.0.1:36454 client_network=udp -13:32.747 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -13:32.750 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=395.065944ms client=127.0.0.1:39094 client_network=tcp -13:34.409 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -13:34.424 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=286.120451ms client=127.0.0.1:39112 client_network=tcp -13:35.753 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=231.828116ms client=127.0.0.1:52239 client_network=udp -13:36.023 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=261.623534ms client=127.0.0.1:51902 client_network=udp -13:36.235 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -13:36.236 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=154.607815ms client=127.0.0.1:39122 client_network=tcp -13:37.314 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -13:37.315 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=213.750533ms client=127.0.0.1:39128 client_network=tcp -13:38.295 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=201.287275ms client=127.0.0.1:39983 client_network=udp -13:38.449 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=143.664021ms client=127.0.0.1:53855 client_network=udp -13:38.775 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -13:38.790 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=300.334533ms client=127.0.0.1:39136 client_network=tcp -13:40.744 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -13:40.746 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=310.80588ms client=127.0.0.1:39138 client_network=tcp -13:42.398 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=331.411957ms client=127.0.0.1:37896 client_network=udp -13:42.739 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=305.705108ms client=127.0.0.1:56278 client_network=udp -13:42.968 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -13:42.969 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=201.897847ms client=127.0.0.1:39140 client_network=tcp -13:44.235 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -13:44.237 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=206.785198ms client=127.0.0.1:39142 client_network=tcp -13:45.095 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=161.147597ms client=127.0.0.1:58394 client_network=udp -13:45.383 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=269.508969ms client=127.0.0.1:48947 client_network=udp -13:45.632 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -13:45.635 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=240.172436ms client=127.0.0.1:39144 client_network=tcp -13:46.979 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -13:46.990 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=316.023284ms client=127.0.0.1:39146 client_network=tcp -13:48.026 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=193.313648ms client=127.0.0.1:33041 client_network=udp -13:48.211 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=173.472374ms client=127.0.0.1:54117 client_network=udp -13:48.500 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -13:48.510 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=283.16379ms client=127.0.0.1:39148 client_network=tcp -13:49.393 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{truncated.service.consul. 255 1}"] records=4092/5001 size=65514/80058 -13:49.395 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=281.365018ms client=127.0.0.1:39202 client_network=tcp -13:50.187 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=196.749792ms client=127.0.0.1:54983 client_network=udp -13:50.414 [DEBUG] TestAgent.dns: request served from client: name=truncated.service.consul. type=ANY class=IN latency=205.77826ms client=127.0.0.1:41956 client_network=udp -13:50.714 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -13:50.721 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=284.019398ms client=127.0.0.1:39212 client_network=tcp -13:51.900 [DEBUG] TestAgent.dns: TCP answer to question too large, truncated: question=["{63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. 255 1}"] records=4091/5001 size=65523/80083 -13:51.908 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=262.641954ms client=127.0.0.1:39214 client_network=tcp -13:53.321 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=275.630661ms client=127.0.0.1:37710 client_network=udp -13:53.617 [DEBUG] TestAgent.dns: request served from client: name=63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul. type=ANY class=IN latency=275.548493ms client=127.0.0.1:33340 client_network=udp -13:53.652 [INFO] TestAgent: Requesting shutdown -13:53.652 [INFO] TestAgent.server: shutting down server -13:53.652 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:53.652 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:53.652 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:53.652 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:53.652 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:53.652 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:53.652 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:53.652 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:53.652 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:53.652 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:53.653 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:53.653Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:53.653Z [INFO] agent.router.manager: shutting down -13:53.653 [INFO] TestAgent: consul server down -13:53.653 [INFO] TestAgent: shutdown complete -13:53.653 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16912 network=tcp -13:53.653 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16912 network=udp -13:53.653 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16913 network=tcp -13:54.153 [INFO] TestAgent: Waiting for endpoints to shut down -13:54.153 [INFO] TestAgent: Endpoints down ---- PASS: TestDNS_TCP_and_UDP_Truncate (42.64s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=255)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=255)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=255)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=255)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=true (0.01s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=1)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=1)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=1)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=1)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=true (0.01s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=false (0.01s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=33)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=33)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=33)_compressed=true (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=33)_compressed=false (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=true#01 (0.01s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=255)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=255)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=255)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=255)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=255)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=255)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=true#01 (0.01s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=1)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=1)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=1)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=1)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=1)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=1)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._tcp_(qType:=33)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_normal.service.consul._udp_(qType:=33)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=33)_compressed=true#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._tcp_(qType:=33)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=33)_compressed=true#01 (0.01s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_5970f412-21fc-e6fb-c79c-b4036a767a4d.query.consul._udp_(qType:=33)_compressed=false#01 (0.00s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=true (1.42s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=false (0.79s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=true (0.24s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=false (0.19s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=255)_compressed=true (1.21s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=255)_compressed=false (1.39s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=255)_compressed=true (0.26s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=255)_compressed=false (0.22s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=true (1.32s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=false (0.88s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=true (0.31s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=false (0.21s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=1)_compressed=true (1.17s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=1)_compressed=false (1.22s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=1)_compressed=true (0.18s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=1)_compressed=false (0.19s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=true (0.95s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=false (1.07s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=true (0.19s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=false (0.33s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=33)_compressed=true (1.78s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=33)_compressed=false (1.38s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=33)_compressed=true (0.24s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=33)_compressed=false (0.32s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=true#01 (1.02s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=255)_compressed=false#01 (0.99s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=true#01 (0.21s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=255)_compressed=false#01 (0.19s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=255)_compressed=true#01 (1.95s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=255)_compressed=false#01 (1.63s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=255)_compressed=true#01 (0.37s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=255)_compressed=false#01 (0.34s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=true#01 (1.26s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=1)_compressed=false#01 (0.90s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=true#01 (0.19s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=1)_compressed=false#01 (0.28s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=1)_compressed=true#01 (1.28s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=1)_compressed=false#01 (1.16s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=1)_compressed=true#01 (0.20s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=1)_compressed=false#01 (0.19s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=true#01 (0.88s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._tcp_(qType:=33)_compressed=false#01 (0.88s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=true#01 (0.21s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_truncated.service.consul._udp_(qType:=33)_compressed=false#01 (0.23s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=33)_compressed=true#01 (1.20s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._tcp_(qType:=33)_compressed=false#01 (1.41s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=33)_compressed=true#01 (0.30s) - --- PASS: TestDNS_TCP_and_UDP_Truncate/lookup_63946e60-a42c-47ae-c36e-0f9e5a25b55a.query.consul._udp_(qType:=33)_compressed=false#01 (0.31s) +2022-12-04T05:49:37.866Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.888Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.899 [INFO] Agent2.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:66c6f232-2296-af88-2cd2-7732c7a8d7ab Address:127.0.0.1:23418}]" +49:37.900 [INFO] Agent2.server.serf.wan: serf: EventMemberJoin: Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab.dc1 127.0.0.1 +49:37.900 [INFO] Agent2.server.raft: entering follower state: follower="Node at 127.0.0.1:23418 [Follower]" leader= +49:37.900 [INFO] Agent2.server.serf.lan: serf: EventMemberJoin: Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab 127.0.0.1 +2022-12-04T05:49:37.900Z [INFO] agent.router: Initializing LAN area manager +49:37.901 [INFO] Agent2.server: Adding LAN server: server="Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab (Addr: tcp/127.0.0.1:23418) (DC: dc1)" +49:37.901 [INFO] Agent2.server: Handled event for server in area: event=member-join server=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab.dc1 area=wan +49:37.901 [INFO] Agent2: Started DNS server: address=127.0.0.1:23308 network=tcp +49:37.901 [INFO] Agent2: Started DNS server: address=127.0.0.1:23308 network=udp +49:37.902 [INFO] Agent2: Started HTTP server: address=127.0.0.1:23400 network=tcp +49:37.902 [INFO] Agent2: started state syncer +49:37.902 [INFO] Agent2: Started gRPC server: address=127.0.0.1:23363 network=tcp +49:37.939 [WARN] Agent2.server.raft: heartbeat timeout reached, starting election: last-leader= +49:37.939 [INFO] Agent2.server.raft: entering candidate state: node="Node at 127.0.0.1:23418 [Candidate]" term=2 +49:37.939 [DEBUG] Agent2.server.raft: votes: needed=1 +49:37.939 [DEBUG] Agent2.server.raft: vote granted: from=66c6f232-2296-af88-2cd2-7732c7a8d7ab term=2 tally=1 +49:37.939 [INFO] Agent2.server.raft: election won: tally=1 +49:37.939 [INFO] Agent2.server.raft: entering leader state: leader="Node at 127.0.0.1:23418 [Leader]" +49:37.939 [INFO] Agent2.server: cluster leadership acquired +49:37.939 [DEBUG] Agent2.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23418 +49:37.940 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:37.942 [INFO] Agent2.server.connect: initialized primary datacenter CA with provider: provider=consul +49:37.942 [INFO] Agent2.leader: started routine: routine="federation state anti-entropy" +49:37.942 [INFO] Agent2.leader: started routine: routine="federation state pruning" +49:37.942 [INFO] Agent2.leader: started routine: routine="intermediate cert renew watch" +49:37.942 [INFO] Agent2.leader: started routine: routine="CA root pruning" +49:37.942 [DEBUG] Agent2.server: Skipping self join check for node since the cluster is too small: node=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab +49:37.942 [INFO] Agent2.server: member joined, marking health alive: member=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab +49:37.942 [INFO] Agent2.server: New leader elected: payload=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab +49:37.942 [INFO] Agent2.server: federation state anti-entropy synced +49:38.000 [DEBUG] Agent2.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:48796 +49:38.001 [INFO] Agent2.server.serf.lan: serf: EventMemberJoin: Node-fdcc13f9-036e-de1a-f26b-4aa337311521 127.0.0.1 +49:38.001 [INFO] Agent2.server: Adding LAN server: server="Node-fdcc13f9-036e-de1a-f26b-4aa337311521 (Addr: tcp/127.0.0.1:24242) (DC: dc1)" +49:38.001 [INFO] Agent2.server: New leader elected: payload=Node-fdcc13f9-036e-de1a-f26b-4aa337311521 +49:38.001 [INFO] Agent2: Requesting shutdown +49:38.001 [ERROR] Agent2.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-fdcc13f9-036e-de1a-f26b-4aa337311521 other=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab +49:38.001 [INFO] Agent2.server: shutting down server +49:38.001 [DEBUG] Agent2.leader: stopping routine: routine="federation state anti-entropy" +49:38.001 [DEBUG] Agent2.leader: stopping routine: routine="federation state pruning" +49:38.001 [DEBUG] Agent2.leader: stopping routine: routine="intermediate cert renew watch" +49:38.001 [INFO] Agent2.server: member joined, marking health alive: member=Node-fdcc13f9-036e-de1a-f26b-4aa337311521 +49:38.001 [DEBUG] Agent2.leader: stopped routine: routine="intermediate cert renew watch" +49:38.001 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No cluster leader" +49:38.001 [DEBUG] Agent2.leader: stopped routine: routine="federation state pruning" +49:38.001 [DEBUG] Agent2.leader: stopping routine: routine="CA root pruning" +49:38.002 [WARN] Agent2.server.serf.lan: serf: Shutdown without a Leave +49:38.002 [ERROR] Agent2.server: error performing anti-entropy sync of federation state: error="context canceled" +49:38.002 [DEBUG] Agent2.leader: stopped routine: routine="federation state anti-entropy" +49:38.002 [DEBUG] Agent2.leader: stopped routine: routine="CA root pruning" +49:38.002 [WARN] Agent2.server.serf.wan: serf: Shutdown without a Leave +49:38.002 [DEBUG] Agent2.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-fdcc13f9-036e-de1a-f26b-4aa337311521.dc1 127.0.0.1:24241 +49:38.002 [DEBUG] Agent2.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:33470 +2022-12-04T05:49:38.002Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:38.002Z [INFO] agent.router.manager: shutting down +49:38.002 [INFO] Agent2: consul server down +49:38.002 [INFO] Agent2: shutdown complete +49:38.002 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:23308 network=tcp +49:38.002 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:23308 network=udp +49:38.002 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:23400 network=tcp +49:38.002 [INFO] Agent2.server.serf.wan: serf: EventMemberJoin: Node-fdcc13f9-036e-de1a-f26b-4aa337311521.dc1 127.0.0.1 +49:38.002 [DEBUG] Agent2.server: Successfully performed flood-join for server at address: server=Node-fdcc13f9-036e-de1a-f26b-4aa337311521.dc1 address=127.0.0.1:24241 +49:38.502 [INFO] Agent2: Waiting for endpoints to shut down +49:38.502 [INFO] Agent2: Endpoints down +2022-12-04T05:49:37.423Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:37.433Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:37.435 [INFO] Agent1.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fdcc13f9-036e-de1a-f26b-4aa337311521 Address:127.0.0.1:24242}]" +49:37.435 [INFO] Agent1.server.raft: entering follower state: follower="Node at 127.0.0.1:24242 [Follower]" leader= +49:37.435 [INFO] Agent1.server.serf.wan: serf: EventMemberJoin: Node-fdcc13f9-036e-de1a-f26b-4aa337311521.dc1 127.0.0.1 +49:37.435 [INFO] Agent1.server.serf.lan: serf: EventMemberJoin: Node-fdcc13f9-036e-de1a-f26b-4aa337311521 127.0.0.1 +2022-12-04T05:49:37.435Z [INFO] agent.router: Initializing LAN area manager +49:37.436 [INFO] Agent1.server: Adding LAN server: server="Node-fdcc13f9-036e-de1a-f26b-4aa337311521 (Addr: tcp/127.0.0.1:24242) (DC: dc1)" +49:37.436 [INFO] Agent1: Started DNS server: address=127.0.0.1:23280 network=udp +49:37.436 [INFO] Agent1.server: Handled event for server in area: event=member-join server=Node-fdcc13f9-036e-de1a-f26b-4aa337311521.dc1 area=wan +49:37.436 [INFO] Agent1: Started DNS server: address=127.0.0.1:23280 network=tcp +49:37.436 [INFO] Agent1: Started HTTP server: address=127.0.0.1:24238 network=tcp +49:37.436 [INFO] Agent1: started state syncer +49:37.436 [INFO] Agent1: Started gRPC server: address=127.0.0.1:24194 network=tcp +49:37.502 [WARN] Agent1.server.raft: heartbeat timeout reached, starting election: last-leader= +49:37.502 [INFO] Agent1.server.raft: entering candidate state: node="Node at 127.0.0.1:24242 [Candidate]" term=2 +49:37.503 [DEBUG] Agent1.server.raft: votes: needed=1 +49:37.503 [DEBUG] Agent1.server.raft: vote granted: from=fdcc13f9-036e-de1a-f26b-4aa337311521 term=2 tally=1 +49:37.503 [INFO] Agent1.server.raft: election won: tally=1 +49:37.503 [INFO] Agent1.server.raft: entering leader state: leader="Node at 127.0.0.1:24242 [Leader]" +49:37.503 [INFO] Agent1.server: cluster leadership acquired +49:37.504 [DEBUG] Agent1.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24242 +49:37.504 [INFO] Agent1.server: New leader elected: payload=Node-fdcc13f9-036e-de1a-f26b-4aa337311521 +49:37.505 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:37.516 [INFO] Agent1.server.connect: initialized primary datacenter CA with provider: provider=consul +49:37.516 [INFO] Agent1.leader: started routine: routine="federation state anti-entropy" +49:37.516 [INFO] Agent1.leader: started routine: routine="federation state pruning" +49:37.516 [INFO] Agent1.leader: started routine: routine="intermediate cert renew watch" +49:37.516 [INFO] Agent1.leader: started routine: routine="CA root pruning" +49:37.517 [DEBUG] Agent1.server: Skipping self join check for node since the cluster is too small: node=Node-fdcc13f9-036e-de1a-f26b-4aa337311521 +49:37.517 [INFO] Agent1.server: member joined, marking health alive: member=Node-fdcc13f9-036e-de1a-f26b-4aa337311521 +49:37.518 [INFO] Agent1.server: federation state anti-entropy synced +49:37.844 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth +49:37.845 [INFO] Agent1: Synced node info +49:37.999 [INFO] Agent1: (LAN) joining: lan_addresses=[127.0.0.1:23416] +49:38.000 [DEBUG] Agent1.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:23416 +49:38.001 [INFO] Agent1.server.serf.lan: serf: EventMemberJoin: Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab 127.0.0.1 +49:38.001 [INFO] Agent1: (LAN) joined: number_of_nodes=1 +49:38.001 [DEBUG] Agent1: systemd notify failed: error="No socket" +49:38.001 [INFO] Agent1.server: Adding LAN server: server="Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab (Addr: tcp/127.0.0.1:23418) (DC: dc1)" +49:38.001 [ERROR] Agent1.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab other=Node-fdcc13f9-036e-de1a-f26b-4aa337311521 +49:38.001 [INFO] Agent1.server: member joined, marking health alive: member=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab +49:38.002 [DEBUG] Agent1.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab.dc1 127.0.0.1:23417 +49:38.002 [DEBUG] Agent1.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:50846 +49:38.002 [INFO] Agent1.server.serf.wan: serf: EventMemberJoin: Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab.dc1 127.0.0.1 +49:38.002 [INFO] Agent1.server: Handled event for server in area: event=member-join server=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab.dc1 area=wan +49:38.003 [DEBUG] Agent1.server: Successfully performed flood-join for server at address: server=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab.dc1 address=127.0.0.1:23417 +49:38.936 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab (timeout reached) +49:39.201 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth +49:39.201 [DEBUG] Agent1: Node info in sync +49:39.201 [DEBUG] Agent1: Node info in sync +49:39.436 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab has failed, no acks received +49:39.511 [WARN] Agent1: error getting server health from server: server=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:23418: connect: connection refused" +49:40.511 [WARN] Agent1: error getting server health from server: server=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab error="context deadline exceeded" +49:40.936 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab (timeout reached) +49:41.511 [WARN] Agent1: error getting server health from server: server=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:23418: connect: connection refused" +49:42.436 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab has failed, no acks received +49:42.511 [WARN] Agent1: error getting server health from server: server=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab error="context deadline exceeded" +49:43.437 [INFO] Agent1.server.memberlist.lan: memberlist: Marking Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab as failed, suspect timeout reached (0 peer confirmations) +49:43.437 [INFO] Agent1.server.serf.lan: serf: EventMemberFailed: Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab 127.0.0.1 +49:43.437 [INFO] Agent1.server: Removing LAN server: server="Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab (Addr: tcp/127.0.0.1:23418) (DC: dc1)" +49:43.437 [INFO] Agent1.server: member failed, marking health critical: member=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab +49:43.445 [INFO] Agent1: Force leaving node: node=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab +49:43.445 [INFO] Agent1.server.serf.lan: serf: EventMemberLeave (forced): Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab 127.0.0.1 +49:43.445 [INFO] Agent1.server.serf.lan: serf: EventMemberReap (forced): Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab 127.0.0.1 +49:43.445 [INFO] Agent1.server: Removing LAN server: server="Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab (Addr: tcp/127.0.0.1:23418) (DC: dc1)" +49:43.445 [INFO] Agent1.server: Removing LAN server: server="Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab (Addr: tcp/127.0.0.1:23418) (DC: dc1)" +49:43.445 [INFO] Agent1.server: deregistering member: member=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab reason=left +49:43.936 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab (timeout reached) +49:45.436 [DEBUG] Agent1.server.memberlist.wan: memberlist: Failed ping: Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab.dc1 (timeout reached) +49:46.436 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab has failed, no acks received +49:47.435 [INFO] Agent1.server.memberlist.wan: memberlist: Suspect Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab.dc1 has failed, no acks received +49:51.445 [INFO] Agent1.server.serf.wan: serf: EventMemberReap (forced): Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab.dc1 127.0.0.1 +49:51.445 [INFO] Agent1: Requesting shutdown +49:51.446 [INFO] Agent1.server: Handled event for server in area: event=member-reap server=Node-66c6f232-2296-af88-2cd2-7732c7a8d7ab.dc1 area=wan +49:51.446 [INFO] Agent1.server: shutting down server +49:51.446 [DEBUG] Agent1.leader: stopping routine: routine="CA root pruning" +49:51.446 [DEBUG] Agent1.leader: stopping routine: routine="federation state anti-entropy" +49:51.446 [DEBUG] Agent1.leader: stopping routine: routine="federation state pruning" +49:51.446 [DEBUG] Agent1.leader: stopping routine: routine="intermediate cert renew watch" +49:51.446 [WARN] Agent1.server.serf.lan: serf: Shutdown without a Leave +49:51.446 [ERROR] Agent1.server: error performing anti-entropy sync of federation state: error="context canceled" +49:51.446 [DEBUG] Agent1.leader: stopped routine: routine="federation state pruning" +49:51.446 [DEBUG] Agent1.leader: stopped routine: routine="CA root pruning" +49:51.446 [DEBUG] Agent1.leader: stopped routine: routine="federation state anti-entropy" +49:51.446 [WARN] Agent1.server.serf.wan: serf: Shutdown without a Leave +49:51.446 [DEBUG] Agent1.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:49:51.446Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:51.446Z [INFO] agent.router.manager: shutting down +49:51.446 [INFO] Agent1: consul server down +49:51.446 [INFO] Agent1: shutdown complete +49:51.446 [INFO] Agent1: Stopping server: protocol=DNS address=127.0.0.1:23280 network=tcp +49:51.446 [INFO] Agent1: Stopping server: protocol=DNS address=127.0.0.1:23280 network=udp +49:51.446 [INFO] Agent1: Stopping server: protocol=HTTP address=127.0.0.1:24238 network=tcp +49:51.946 [INFO] Agent1: Waiting for endpoints to shut down +49:51.946 [INFO] Agent1: Endpoints down +--- PASS: TestAgent_ForceLeavePrune (14.55s) PASS -ok github.com/hashicorp/consul/agent 142.207s +ok github.com/hashicorp/consul/agent 120.503s === RUN TestAE_scaleFactor === PAUSE TestAE_scaleFactor === RUN TestAE_Pause_nestedPauseResume @@ -56423,7 +56549,7 @@ === RUN TestAE_FSM/fullSyncState === RUN TestAE_FSM/fullSyncState/Paused_->_retryFullSyncState === RUN TestAE_FSM/fullSyncState/SyncFull()_error_->_retryFullSyncState -2021-10-31T23:11:01.808Z [ERROR] anti_entropy: failed to sync remote state: error=boom +2022-12-04T05:47:38.093Z [ERROR] anti_entropy: failed to sync remote state: error=boom === RUN TestAE_FSM/fullSyncState/SyncFull()_OK_->_partialSyncState === RUN TestAE_FSM/retryFullSyncState === RUN TestAE_FSM/retryFullSyncState/shutdownEvent_->_doneState @@ -56436,7 +56562,7 @@ === RUN TestAE_FSM/partialSyncState/syncFullTimerEvent_->_fullSyncState === RUN TestAE_FSM/partialSyncState/syncChangesEvent+Paused_->_partialSyncState === RUN TestAE_FSM/partialSyncState/syncChangesEvent+SyncChanges()_error_->_partialSyncState -2021-10-31T23:11:01.809Z [ERROR] anti_entropy: failed to sync changes: error=boom +2022-12-04T05:47:38.094Z [ERROR] anti_entropy: failed to sync changes: error=boom === RUN TestAE_FSM/partialSyncState/syncChangesEvent+SyncChanges()_OK_->_partialSyncState === RUN TestAE_FSM/partialSyncState/invalid_event_->_panic_ === RUN TestAE_FSM/invalid_state_->_panic_ @@ -56464,9 +56590,9 @@ === RUN TestAE_RetrySyncFullEvent/trigger_shutdownEvent_during_FullNotif === RUN TestAE_RetrySyncFullEvent/trigger_syncFullNotifEvent === RUN TestAE_RetrySyncFullEvent/trigger_syncFullTimerEvent ---- PASS: TestAE_RetrySyncFullEvent (0.13s) +--- PASS: TestAE_RetrySyncFullEvent (0.14s) --- PASS: TestAE_RetrySyncFullEvent/trigger_shutdownEvent (0.00s) - --- PASS: TestAE_RetrySyncFullEvent/trigger_shutdownEvent_during_FullNotif (0.10s) + --- PASS: TestAE_RetrySyncFullEvent/trigger_shutdownEvent_during_FullNotif (0.11s) --- PASS: TestAE_RetrySyncFullEvent/trigger_syncFullNotifEvent (0.01s) --- PASS: TestAE_RetrySyncFullEvent/trigger_syncFullTimerEvent (0.02s) === RUN TestAE_SyncChangesEvent @@ -56494,7 +56620,7 @@ === CONT TestAE_Pause_nestedPauseResume --- PASS: TestAE_Pause_nestedPauseResume (0.00s) PASS -ok github.com/hashicorp/consul/agent/ae 2.358s +ok github.com/hashicorp/consul/agent/ae 2.295s === RUN TestNew === RUN TestNew/no-cache === RUN TestNew/no-tls-configurator @@ -56512,39 +56638,39 @@ === RUN TestReadConfig --- PASS: TestReadConfig (0.00s) === RUN TestInitialConfiguration_disabled ---- PASS: TestInitialConfiguration_disabled (0.02s) +--- PASS: TestInitialConfiguration_disabled (0.01s) === RUN TestInitialConfiguration_cancelled -2021-10-31T23:11:31.215Z [DEBUG] TestInitialConfiguration_cancelled.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_cancelled-auto-config215084522/auto-config.json -2021-10-31T23:11:31.215Z [INFO] TestInitialConfiguration_cancelled.auto_config: retrieving initial agent auto configuration remotely -2021-10-31T23:11:31.241Z [DEBUG] TestInitialConfiguration_cancelled.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:8300 -2021-10-31T23:11:31.241Z [ERROR] TestInitialConfiguration_cancelled.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=127.0.0.1:8300 error="injected error" -2021-10-31T23:11:31.241Z [ERROR] TestInitialConfiguration_cancelled.auto_config: No server successfully responded to the auto-config request -2021-10-31T23:11:31.241Z [DEBUG] TestInitialConfiguration_cancelled.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:8300 -2021-10-31T23:11:31.243Z [ERROR] TestInitialConfiguration_cancelled.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=127.0.0.1:8300 error="injected error" -2021-10-31T23:11:31.243Z [ERROR] TestInitialConfiguration_cancelled.auto_config: No server successfully responded to the auto-config request -2021-10-31T23:11:31.281Z [INFO] TestInitialConfiguration_cancelled.auto_config: interrupted during initial auto configuration: err="context deadline exceeded" +2022-12-04T05:47:45.866Z [DEBUG] TestInitialConfiguration_cancelled.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_cancelled-auto-config243099609/auto-config.json +2022-12-04T05:47:45.866Z [INFO] TestInitialConfiguration_cancelled.auto_config: retrieving initial agent auto configuration remotely +2022-12-04T05:47:45.899Z [DEBUG] TestInitialConfiguration_cancelled.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:8300 +2022-12-04T05:47:45.899Z [ERROR] TestInitialConfiguration_cancelled.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=127.0.0.1:8300 error="injected error" +2022-12-04T05:47:45.899Z [ERROR] TestInitialConfiguration_cancelled.auto_config: No server successfully responded to the auto-config request +2022-12-04T05:47:45.900Z [DEBUG] TestInitialConfiguration_cancelled.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:8300 +2022-12-04T05:47:45.900Z [ERROR] TestInitialConfiguration_cancelled.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=127.0.0.1:8300 error="injected error" +2022-12-04T05:47:45.900Z [ERROR] TestInitialConfiguration_cancelled.auto_config: No server successfully responded to the auto-config request +2022-12-04T05:47:45.960Z [INFO] TestInitialConfiguration_cancelled.auto_config: interrupted during initial auto configuration: err="context deadline exceeded" mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,string) mock_test.go:242: PASS: FindLANServer() ---- PASS: TestInitialConfiguration_cancelled (0.10s) +--- PASS: TestInitialConfiguration_cancelled (0.11s) === RUN TestInitialConfiguration_restored -2021-10-31T23:11:31.306Z [DEBUG] TestInitialConfiguration_restored.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_restored-auto-config197528129/auto-config.json -2021-10-31T23:11:31.307Z [INFO] TestInitialConfiguration_restored.auto_config: read persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_restored-auto-config197528129/auto-config.json -2021-10-31T23:11:31.307Z [DEBUG] TestInitialConfiguration_restored.auto_config: updating auto-config settings -2021-10-31T23:11:31.323Z [DEBUG] TestInitialConfiguration_restored.auto_config: auto-config settings were persisted to disk +2022-12-04T05:47:45.991Z [DEBUG] TestInitialConfiguration_restored.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_restored-auto-config341749604/auto-config.json +2022-12-04T05:47:45.993Z [INFO] TestInitialConfiguration_restored.auto_config: read persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_restored-auto-config341749604/auto-config.json +2022-12-04T05:47:45.993Z [DEBUG] TestInitialConfiguration_restored.auto_config: updating auto-config settings +2022-12-04T05:47:46.005Z [DEBUG] TestInitialConfiguration_restored.auto_config: auto-config settings were persisted to disk mock_test.go:243: PASS: Prepopulate(string,cache.FetchResult,string,string,string) mock_test.go:243: PASS: Prepopulate(string,cache.FetchResult,string,string,string) mock_test.go:244: PASS: AgentToken() mock_test.go:244: PASS: UpdateAgentToken(string,token.TokenSource) mock_test.go:244: PASS: AgentToken() mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) ---- PASS: TestInitialConfiguration_restored (0.06s) +--- PASS: TestInitialConfiguration_restored (0.05s) === RUN TestInitialConfiguration_success -2021-10-31T23:11:31.361Z [DEBUG] TestInitialConfiguration_success.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_success-auto-config186384556/auto-config.json -2021-10-31T23:11:31.361Z [INFO] TestInitialConfiguration_success.auto_config: retrieving initial agent auto configuration remotely -2021-10-31T23:11:31.362Z [DEBUG] TestInitialConfiguration_success.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:8300 -2021-10-31T23:11:31.363Z [DEBUG] TestInitialConfiguration_success.auto_config: AutoConfig.InitialConfiguration RPC was successful -2021-10-31T23:11:31.363Z [DEBUG] TestInitialConfiguration_success.auto_config: updating auto-config settings -2021-10-31T23:11:31.414Z [DEBUG] TestInitialConfiguration_success.auto_config: auto-config settings were persisted to disk +2022-12-04T05:47:46.032Z [DEBUG] TestInitialConfiguration_success.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_success-auto-config535016307/auto-config.json +2022-12-04T05:47:46.032Z [INFO] TestInitialConfiguration_success.auto_config: retrieving initial agent auto configuration remotely +2022-12-04T05:47:46.033Z [DEBUG] TestInitialConfiguration_success.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:8300 +2022-12-04T05:47:46.034Z [DEBUG] TestInitialConfiguration_success.auto_config: AutoConfig.InitialConfiguration RPC was successful +2022-12-04T05:47:46.034Z [DEBUG] TestInitialConfiguration_success.auto_config: updating auto-config settings +2022-12-04T05:47:46.057Z [DEBUG] TestInitialConfiguration_success.auto_config: auto-config settings were persisted to disk mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) mock_test.go:242: PASS: FindLANServer() mock_test.go:243: PASS: Prepopulate(string,cache.FetchResult,string,string,string) @@ -56553,29 +56679,29 @@ mock_test.go:244: PASS: UpdateAgentToken(string,token.TokenSource) mock_test.go:244: PASS: AgentToken() mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) ---- PASS: TestInitialConfiguration_success (0.09s) +--- PASS: TestInitialConfiguration_success (0.05s) === RUN TestInitialConfiguration_retries -2021-10-31T23:11:31.466Z [DEBUG] TestInitialConfiguration_retries.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_retries-auto-config307855387/auto-config.json -2021-10-31T23:11:31.466Z [INFO] TestInitialConfiguration_retries.auto_config: retrieving initial agent auto configuration remotely -2021-10-31T23:11:31.470Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 -2021-10-31T23:11:31.470Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.1:8300 error="injected failure" -2021-10-31T23:11:31.470Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.2:8398 -2021-10-31T23:11:31.470Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.2:8398 error="injected failure" -2021-10-31T23:11:31.470Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.3:8399 -2021-10-31T23:11:31.470Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.3:8399 error="injected failure" -2021-10-31T23:11:31.470Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:1234 -2021-10-31T23:11:31.474Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=127.0.0.1:1234 error="injected failure" -2021-10-31T23:11:31.474Z [ERROR] TestInitialConfiguration_retries.auto_config: No server successfully responded to the auto-config request -2021-10-31T23:11:31.474Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 -2021-10-31T23:11:31.474Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.1:8300 error="injected failure" -2021-10-31T23:11:31.474Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.2:8398 -2021-10-31T23:11:31.474Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.2:8398 error="injected failure" -2021-10-31T23:11:31.474Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.3:8399 -2021-10-31T23:11:31.474Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.3:8399 error="injected failure" -2021-10-31T23:11:31.474Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:1234 -2021-10-31T23:11:31.475Z [DEBUG] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC was successful -2021-10-31T23:11:31.475Z [DEBUG] TestInitialConfiguration_retries.auto_config: updating auto-config settings -2021-10-31T23:11:31.487Z [DEBUG] TestInitialConfiguration_retries.auto_config: auto-config settings were persisted to disk +2022-12-04T05:47:46.080Z [DEBUG] TestInitialConfiguration_retries.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestInitialConfiguration_retries-auto-config448389686/auto-config.json +2022-12-04T05:47:46.080Z [INFO] TestInitialConfiguration_retries.auto_config: retrieving initial agent auto configuration remotely +2022-12-04T05:47:46.080Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 +2022-12-04T05:47:46.081Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.1:8300 error="injected failure" +2022-12-04T05:47:46.081Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.2:8398 +2022-12-04T05:47:46.081Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.2:8398 error="injected failure" +2022-12-04T05:47:46.081Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.3:8399 +2022-12-04T05:47:46.081Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.3:8399 error="injected failure" +2022-12-04T05:47:46.081Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:1234 +2022-12-04T05:47:46.081Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=127.0.0.1:1234 error="injected failure" +2022-12-04T05:47:46.081Z [ERROR] TestInitialConfiguration_retries.auto_config: No server successfully responded to the auto-config request +2022-12-04T05:47:46.081Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 +2022-12-04T05:47:46.081Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.1:8300 error="injected failure" +2022-12-04T05:47:46.081Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.2:8398 +2022-12-04T05:47:46.082Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.2:8398 error="injected failure" +2022-12-04T05:47:46.082Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.3:8399 +2022-12-04T05:47:46.082Z [ERROR] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC failed: addr=198.18.0.3:8399 error="injected failure" +2022-12-04T05:47:46.082Z [DEBUG] TestInitialConfiguration_retries.auto_config: making AutoConfig.InitialConfiguration RPC: addr=127.0.0.1:1234 +2022-12-04T05:47:46.083Z [DEBUG] TestInitialConfiguration_retries.auto_config: AutoConfig.InitialConfiguration RPC was successful +2022-12-04T05:47:46.083Z [DEBUG] TestInitialConfiguration_retries.auto_config: updating auto-config settings +2022-12-04T05:47:46.108Z [DEBUG] TestInitialConfiguration_retries.auto_config: auto-config settings were persisted to disk mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) @@ -56588,33 +56714,33 @@ mock_test.go:244: PASS: UpdateAgentToken(string,token.TokenSource) mock_test.go:244: PASS: AgentToken() mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) ---- PASS: TestInitialConfiguration_retries (0.08s) +--- PASS: TestInitialConfiguration_retries (0.06s) === RUN TestGoRoutineManagement -2021-10-31T23:11:31.561Z [INFO] TestGoRoutineManagement.auto_config: auto-config started -2021-10-31T23:11:31.573Z [DEBUG] TestGoRoutineManagement.auto_config: stopping auto-config -2021-10-31T23:11:31.574Z [DEBUG] TestGoRoutineManagement.auto_config: auto-config has been stopped -2021-10-31T23:11:31.574Z [INFO] TestGoRoutineManagement.auto_config: auto-config started -2021-10-31T23:11:31.574Z [DEBUG] TestGoRoutineManagement.auto_config: stopping auto-config -2021-10-31T23:11:31.574Z [DEBUG] TestGoRoutineManagement.auto_config: auto-config has been stopped +2022-12-04T05:47:46.139Z [INFO] TestGoRoutineManagement.auto_config: auto-config started +2022-12-04T05:47:46.149Z [DEBUG] TestGoRoutineManagement.auto_config: stopping auto-config +2022-12-04T05:47:46.149Z [DEBUG] TestGoRoutineManagement.auto_config: auto-config has been stopped +2022-12-04T05:47:46.150Z [INFO] TestGoRoutineManagement.auto_config: auto-config started +2022-12-04T05:47:46.150Z [DEBUG] TestGoRoutineManagement.auto_config: stopping auto-config +2022-12-04T05:47:46.150Z [DEBUG] TestGoRoutineManagement.auto_config: auto-config has been stopped mock_test.go:243: PASS: Notify(string,string,*structs.DCSpecificRequest,string,string) mock_test.go:243: PASS: Notify(string,string,*cachetype.ConnectCALeafRequest,string,string) mock_test.go:244: PASS: AgentToken() mock_test.go:244: PASS: Notify(token.TokenKind) mock_test.go:244: PASS: StopNotify(token.Notifier) mock_test.go:245: PASS: AutoEncryptCertNotAfter() ---- PASS: TestGoRoutineManagement (0.06s) +--- PASS: TestGoRoutineManagement (0.02s) === RUN TestTokenUpdate -2021-10-31T23:11:31.613Z [DEBUG] TestTokenUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestTokenUpdate-auto-config723437829/auto-config.json -2021-10-31T23:11:31.613Z [INFO] TestTokenUpdate.auto_config: retrieving initial agent auto configuration remotely -2021-10-31T23:11:31.614Z [DEBUG] TestTokenUpdate.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 -2021-10-31T23:11:31.615Z [DEBUG] TestTokenUpdate.auto_config: AutoConfig.InitialConfiguration RPC was successful -2021-10-31T23:11:31.615Z [DEBUG] TestTokenUpdate.auto_config: updating auto-config settings -2021-10-31T23:11:31.638Z [DEBUG] TestTokenUpdate.auto_config: auto-config settings were persisted to disk -2021-10-31T23:11:31.663Z [INFO] TestTokenUpdate.auto_config: auto-config started -2021-10-31T23:11:31.663Z [DEBUG] TestTokenUpdate.auto_config: handling a token update event -2021-10-31T23:11:31.663Z [DEBUG] TestTokenUpdate.auto_config: Agent token updated - resetting watches -2021-10-31T23:11:31.664Z [DEBUG] TestTokenUpdate.auto_config: stopping auto-config -2021-10-31T23:11:31.664Z [DEBUG] TestTokenUpdate.auto_config: auto-config has been stopped +2022-12-04T05:47:46.169Z [DEBUG] TestTokenUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestTokenUpdate-auto-config253880024/auto-config.json +2022-12-04T05:47:46.169Z [INFO] TestTokenUpdate.auto_config: retrieving initial agent auto configuration remotely +2022-12-04T05:47:46.169Z [DEBUG] TestTokenUpdate.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 +2022-12-04T05:47:46.170Z [DEBUG] TestTokenUpdate.auto_config: AutoConfig.InitialConfiguration RPC was successful +2022-12-04T05:47:46.170Z [DEBUG] TestTokenUpdate.auto_config: updating auto-config settings +2022-12-04T05:47:46.179Z [DEBUG] TestTokenUpdate.auto_config: auto-config settings were persisted to disk +2022-12-04T05:47:46.189Z [INFO] TestTokenUpdate.auto_config: auto-config started +2022-12-04T05:47:46.189Z [DEBUG] TestTokenUpdate.auto_config: handling a token update event +2022-12-04T05:47:46.189Z [DEBUG] TestTokenUpdate.auto_config: Agent token updated - resetting watches +2022-12-04T05:47:46.189Z [DEBUG] TestTokenUpdate.auto_config: stopping auto-config +2022-12-04T05:47:46.189Z [DEBUG] TestTokenUpdate.auto_config: auto-config has been stopped mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) mock_test.go:242: PASS: FindLANServer() mock_test.go:243: PASS: Notify(string,string,*structs.DCSpecificRequest,string,string) @@ -56631,22 +56757,24 @@ mock_test.go:244: PASS: AgentToken() mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) mock_test.go:245: PASS: AutoEncryptCertNotAfter() ---- PASS: TestTokenUpdate (0.10s) +--- PASS: TestTokenUpdate (0.04s) === RUN TestRootsUpdate -2021-10-31T23:11:31.694Z [DEBUG] TestRootsUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestRootsUpdate-auto-config385440928/auto-config.json -2021-10-31T23:11:31.694Z [INFO] TestRootsUpdate.auto_config: retrieving initial agent auto configuration remotely -2021-10-31T23:11:31.695Z [DEBUG] TestRootsUpdate.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 -2021-10-31T23:11:31.695Z [DEBUG] TestRootsUpdate.auto_config: AutoConfig.InitialConfiguration RPC was successful -2021-10-31T23:11:31.695Z [DEBUG] TestRootsUpdate.auto_config: updating auto-config settings -2021-10-31T23:11:31.739Z [DEBUG] TestRootsUpdate.auto_config: auto-config settings were persisted to disk -2021-10-31T23:11:31.772Z [INFO] TestRootsUpdate.auto_config: auto-config started -2021-10-31T23:11:31.773Z [DEBUG] TestRootsUpdate.auto_config: handling a cache update event: correlation_id=roots -2021-10-31T23:11:31.773Z [DEBUG] TestRootsUpdate.auto_config: roots watch fired - updating CA certificates -2021-10-31T23:11:31.774Z [DEBUG] TestRootsUpdate.auto_config: auto-config settings were persisted to disk -2021-10-31T23:11:31.774Z [DEBUG] TestRootsUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestRootsUpdate-auto-config385440928/auto-config.json -2021-10-31T23:11:31.782Z [INFO] TestRootsUpdate.auto_config: read persisted configuration: path=/tmp/consul-test/TestRootsUpdate-auto-config385440928/auto-config.json -2021-10-31T23:11:31.782Z [DEBUG] TestRootsUpdate.auto_config: stopping auto-config -2021-10-31T23:11:31.782Z [DEBUG] TestRootsUpdate.auto_config: auto-config has been stopped +2022-12-04T05:47:46.199Z [DEBUG] TestRootsUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestRootsUpdate-auto-config238664791/auto-config.json +2022-12-04T05:47:46.199Z [INFO] TestRootsUpdate.auto_config: retrieving initial agent auto configuration remotely +2022-12-04T05:47:46.199Z [DEBUG] TestRootsUpdate.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 +2022-12-04T05:47:46.200Z [DEBUG] TestRootsUpdate.auto_config: AutoConfig.InitialConfiguration RPC was successful +2022-12-04T05:47:46.200Z [DEBUG] TestRootsUpdate.auto_config: updating auto-config settings +2022-12-04T05:47:46.211Z [DEBUG] TestRootsUpdate.auto_config: auto-config settings were persisted to disk +2022-12-04T05:47:46.219Z [INFO] TestRootsUpdate.auto_config: auto-config started +2022-12-04T05:47:46.222Z [DEBUG] TestRootsUpdate.auto_config: handling a cache update event: correlation_id=roots +2022-12-04T05:47:46.222Z [DEBUG] TestRootsUpdate.auto_config: roots watch fired - updating CA certificates +2022-12-04T05:47:46.224Z [DEBUG] TestRootsUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestRootsUpdate-auto-config238664791/auto-config.json +2022-12-04T05:47:46.224Z [DEBUG] TestRootsUpdate.auto_config: auto-config settings were persisted to disk +2022-12-04T05:47:46.225Z [INFO] TestRootsUpdate.auto_config: read persisted configuration: path=/tmp/consul-test/TestRootsUpdate-auto-config238664791/auto-config.json +2022-12-04T05:47:46.250Z [DEBUG] TestRootsUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestRootsUpdate-auto-config238664791/auto-config.json +2022-12-04T05:47:46.252Z [INFO] TestRootsUpdate.auto_config: read persisted configuration: path=/tmp/consul-test/TestRootsUpdate-auto-config238664791/auto-config.json +2022-12-04T05:47:46.252Z [DEBUG] TestRootsUpdate.auto_config: stopping auto-config +2022-12-04T05:47:46.252Z [DEBUG] TestRootsUpdate.auto_config: auto-config has been stopped mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) mock_test.go:242: PASS: FindLANServer() mock_test.go:243: PASS: Notify(string,string,*structs.DCSpecificRequest,string,string) @@ -56662,22 +56790,22 @@ mock_test.go:245: PASS: AutoEncryptCertNotAfter() mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) mock_test.go:245: PASS: AutoEncryptCertNotAfter() ---- PASS: TestRootsUpdate (0.11s) +--- PASS: TestRootsUpdate (0.06s) === RUN TestCertUpdate -2021-10-31T23:11:31.809Z [DEBUG] TestCertUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestCertUpdate-auto-config989374847/auto-config.json -2021-10-31T23:11:31.809Z [INFO] TestCertUpdate.auto_config: retrieving initial agent auto configuration remotely -2021-10-31T23:11:31.834Z [DEBUG] TestCertUpdate.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 -2021-10-31T23:11:31.835Z [DEBUG] TestCertUpdate.auto_config: AutoConfig.InitialConfiguration RPC was successful -2021-10-31T23:11:31.835Z [DEBUG] TestCertUpdate.auto_config: updating auto-config settings -2021-10-31T23:11:31.856Z [DEBUG] TestCertUpdate.auto_config: auto-config settings were persisted to disk -2021-10-31T23:11:31.887Z [INFO] TestCertUpdate.auto_config: auto-config started -2021-10-31T23:11:31.888Z [DEBUG] TestCertUpdate.auto_config: handling a cache update event: correlation_id=leaf -2021-10-31T23:11:31.888Z [DEBUG] TestCertUpdate.auto_config: leaf certificate watch fired - updating TLS certificate -2021-10-31T23:11:31.897Z [DEBUG] TestCertUpdate.auto_config: auto-config settings were persisted to disk -2021-10-31T23:11:31.897Z [DEBUG] TestCertUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestCertUpdate-auto-config989374847/auto-config.json -2021-10-31T23:11:31.898Z [INFO] TestCertUpdate.auto_config: read persisted configuration: path=/tmp/consul-test/TestCertUpdate-auto-config989374847/auto-config.json -2021-10-31T23:11:31.898Z [DEBUG] TestCertUpdate.auto_config: stopping auto-config -2021-10-31T23:11:31.899Z [DEBUG] TestCertUpdate.auto_config: auto-config has been stopped +2022-12-04T05:47:46.262Z [DEBUG] TestCertUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestCertUpdate-auto-config525099978/auto-config.json +2022-12-04T05:47:46.262Z [INFO] TestCertUpdate.auto_config: retrieving initial agent auto configuration remotely +2022-12-04T05:47:46.262Z [DEBUG] TestCertUpdate.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 +2022-12-04T05:47:46.263Z [DEBUG] TestCertUpdate.auto_config: AutoConfig.InitialConfiguration RPC was successful +2022-12-04T05:47:46.263Z [DEBUG] TestCertUpdate.auto_config: updating auto-config settings +2022-12-04T05:47:46.277Z [DEBUG] TestCertUpdate.auto_config: auto-config settings were persisted to disk +2022-12-04T05:47:46.286Z [INFO] TestCertUpdate.auto_config: auto-config started +2022-12-04T05:47:46.297Z [DEBUG] TestCertUpdate.auto_config: handling a cache update event: correlation_id=leaf +2022-12-04T05:47:46.297Z [DEBUG] TestCertUpdate.auto_config: leaf certificate watch fired - updating TLS certificate +2022-12-04T05:47:46.299Z [DEBUG] TestCertUpdate.auto_config: auto-config settings were persisted to disk +2022-12-04T05:47:46.299Z [DEBUG] TestCertUpdate.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestCertUpdate-auto-config525099978/auto-config.json +2022-12-04T05:47:46.301Z [INFO] TestCertUpdate.auto_config: read persisted configuration: path=/tmp/consul-test/TestCertUpdate-auto-config525099978/auto-config.json +2022-12-04T05:47:46.301Z [DEBUG] TestCertUpdate.auto_config: stopping auto-config +2022-12-04T05:47:46.301Z [DEBUG] TestCertUpdate.auto_config: auto-config has been stopped mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) mock_test.go:242: PASS: FindLANServer() mock_test.go:243: PASS: Notify(string,string,*structs.DCSpecificRequest,string,string) @@ -56693,30 +56821,28 @@ mock_test.go:245: PASS: AutoEncryptCertNotAfter() mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) mock_test.go:245: PASS: AutoEncryptCertNotAfter() ---- PASS: TestCertUpdate (0.12s) +--- PASS: TestCertUpdate (0.05s) === RUN TestFallback -2021-10-31T23:11:31.917Z [DEBUG] TestFallback.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestFallback-auto-config690932434/auto-config.json -2021-10-31T23:11:31.917Z [INFO] TestFallback.auto_config: retrieving initial agent auto configuration remotely -2021-10-31T23:11:31.917Z [DEBUG] TestFallback.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 -2021-10-31T23:11:31.918Z [DEBUG] TestFallback.auto_config: AutoConfig.InitialConfiguration RPC was successful -2021-10-31T23:11:31.918Z [DEBUG] TestFallback.auto_config: updating auto-config settings -2021-10-31T23:11:31.955Z [DEBUG] TestFallback.auto_config: auto-config settings were persisted to disk -2021-10-31T23:11:31.973Z [INFO] TestFallback.auto_config: auto-config started -2021-10-31T23:11:31.980Z [DEBUG] TestFallback.auto_config: handling a cache update event: correlation_id=leaf -2021-10-31T23:11:31.980Z [DEBUG] TestFallback.auto_config: leaf certificate watch fired - updating TLS certificate -2021-10-31T23:11:31.981Z [DEBUG] TestFallback.auto_config: auto-config settings were persisted to disk -2021-10-31T23:11:31.981Z [WARN] TestFallback.auto_config: agent's client certificate has expired -2021-10-31T23:11:31.993Z [DEBUG] TestFallback.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.23.2:8300 -2021-10-31T23:11:31.995Z [DEBUG] TestFallback.auto_config: AutoConfig.InitialConfiguration RPC was successful -2021-10-31T23:11:31.995Z [DEBUG] TestFallback.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestFallback-auto-config690932434/auto-config.json -2021-10-31T23:11:31.996Z [INFO] TestFallback.auto_config: read persisted configuration: path=/tmp/consul-test/TestFallback-auto-config690932434/auto-config.json -2021-10-31T23:11:32.025Z [DEBUG] TestFallback.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestFallback-auto-config690932434/auto-config.json -2021-10-31T23:11:32.033Z [INFO] TestFallback.auto_config: read persisted configuration: path=/tmp/consul-test/TestFallback-auto-config690932434/auto-config.json -2021-10-31T23:11:32.033Z [DEBUG] TestFallback.auto_config: auto-config settings were persisted to disk -2021-10-31T23:11:32.061Z [DEBUG] TestFallback.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestFallback-auto-config690932434/auto-config.json -2021-10-31T23:11:32.063Z [INFO] TestFallback.auto_config: read persisted configuration: path=/tmp/consul-test/TestFallback-auto-config690932434/auto-config.json -2021-10-31T23:11:32.063Z [DEBUG] TestFallback.auto_config: stopping auto-config -2021-10-31T23:11:32.064Z [DEBUG] TestFallback.auto_config: auto-config has been stopped +2022-12-04T05:47:46.339Z [DEBUG] TestFallback.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestFallback-auto-config864680097/auto-config.json +2022-12-04T05:47:46.339Z [INFO] TestFallback.auto_config: retrieving initial agent auto configuration remotely +2022-12-04T05:47:46.340Z [DEBUG] TestFallback.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.0.1:8300 +2022-12-04T05:47:46.345Z [DEBUG] TestFallback.auto_config: AutoConfig.InitialConfiguration RPC was successful +2022-12-04T05:47:46.345Z [DEBUG] TestFallback.auto_config: updating auto-config settings +2022-12-04T05:47:46.382Z [DEBUG] TestFallback.auto_config: auto-config settings were persisted to disk +2022-12-04T05:47:46.420Z [INFO] TestFallback.auto_config: auto-config started +2022-12-04T05:47:46.423Z [DEBUG] TestFallback.auto_config: handling a cache update event: correlation_id=leaf +2022-12-04T05:47:46.423Z [DEBUG] TestFallback.auto_config: leaf certificate watch fired - updating TLS certificate +2022-12-04T05:47:46.424Z [DEBUG] TestFallback.auto_config: auto-config settings were persisted to disk +2022-12-04T05:47:46.424Z [WARN] TestFallback.auto_config: agent's client certificate has expired +2022-12-04T05:47:46.432Z [DEBUG] TestFallback.auto_config: making AutoConfig.InitialConfiguration RPC: addr=198.18.23.2:8300 +2022-12-04T05:47:46.433Z [DEBUG] TestFallback.auto_config: AutoConfig.InitialConfiguration RPC was successful +2022-12-04T05:47:46.436Z [DEBUG] TestFallback.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestFallback-auto-config864680097/auto-config.json +2022-12-04T05:47:46.438Z [INFO] TestFallback.auto_config: read persisted configuration: path=/tmp/consul-test/TestFallback-auto-config864680097/auto-config.json +2022-12-04T05:47:46.463Z [DEBUG] TestFallback.auto_config: auto-config settings were persisted to disk +2022-12-04T05:47:46.465Z [DEBUG] TestFallback.auto_config: attempting to restore any persisted configuration: path=/tmp/consul-test/TestFallback-auto-config864680097/auto-config.json +2022-12-04T05:47:46.467Z [INFO] TestFallback.auto_config: read persisted configuration: path=/tmp/consul-test/TestFallback-auto-config864680097/auto-config.json +2022-12-04T05:47:46.467Z [DEBUG] TestFallback.auto_config: stopping auto-config +2022-12-04T05:47:46.467Z [DEBUG] TestFallback.auto_config: auto-config has been stopped mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*pbautoconf.AutoConfigRequest,*pbautoconf.AutoConfigResponse) mock_test.go:242: PASS: FindLANServer() @@ -56743,35 +56869,35 @@ mock_test.go:245: PASS: AutoEncryptCertNotAfter() --- PASS: TestFallback (0.17s) === RUN TestIntroToken +=== RUN TestIntroToken/config === RUN TestIntroToken/file === RUN TestIntroToken/file-empty -=== RUN TestIntroToken/config --- PASS: TestIntroToken (0.00s) + --- PASS: TestIntroToken/config (0.00s) --- PASS: TestIntroToken/file (0.00s) --- PASS: TestIntroToken/file-empty (0.00s) - --- PASS: TestIntroToken/config (0.00s) === RUN TestAutoEncrypt_generateCSR === RUN TestAutoEncrypt_generateCSR/ip-sans === RUN TestAutoEncrypt_generateCSR/dns-sans ---- PASS: TestAutoEncrypt_generateCSR (0.00s) - --- PASS: TestAutoEncrypt_generateCSR/ip-sans (0.00s) +--- PASS: TestAutoEncrypt_generateCSR (0.01s) + --- PASS: TestAutoEncrypt_generateCSR/ip-sans (0.01s) --- PASS: TestAutoEncrypt_generateCSR/dns-sans (0.00s) === RUN TestAutoEncrypt_hosts === RUN TestAutoEncrypt_hosts/router-override === RUN TestAutoEncrypt_hosts/various-addresses === RUN TestAutoEncrypt_hosts/split-host-port-error -2021-10-31T23:11:32.074Z [WARN] TestAutoEncrypt_hosts/split-host-port-error: error splitting host address into IP and port: address=this-is-not:a:ip:and_port error="address this-is-not:a:ip:and_port: too many colons in address" +2022-12-04T05:47:46.486Z [WARN] TestAutoEncrypt_hosts/split-host-port-error: error splitting host address into IP and port: address=this-is-not:a:ip:and_port error="address this-is-not:a:ip:and_port: too many colons in address" --- PASS: TestAutoEncrypt_hosts (0.00s) --- PASS: TestAutoEncrypt_hosts/router-override (0.00s) --- PASS: TestAutoEncrypt_hosts/various-addresses (0.00s) --- PASS: TestAutoEncrypt_hosts/split-host-port-error (0.00s) === RUN TestAutoEncrypt_InitialCerts -2021-10-31T23:11:32.076Z [DEBUG] TestAutoEncrypt_InitialCerts: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 -2021-10-31T23:11:32.076Z [ERROR] TestAutoEncrypt_InitialCerts: AutoEncrypt.Sign RPC failed: addr=198.18.0.1:8300 error="injected error" -2021-10-31T23:11:32.076Z [DEBUG] TestAutoEncrypt_InitialCerts: making AutoEncrypt.Sign RPC: addr=198.18.0.2:8300 -2021-10-31T23:11:32.076Z [ERROR] TestAutoEncrypt_InitialCerts: AutoEncrypt.Sign RPC failed: addr=198.18.0.2:8300 error="injected error" -2021-10-31T23:11:32.076Z [ERROR] TestAutoEncrypt_InitialCerts: No servers successfully responded to the auto-encrypt request -2021-10-31T23:11:32.076Z [DEBUG] TestAutoEncrypt_InitialCerts: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 +2022-12-04T05:47:46.493Z [DEBUG] TestAutoEncrypt_InitialCerts: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 +2022-12-04T05:47:46.493Z [ERROR] TestAutoEncrypt_InitialCerts: AutoEncrypt.Sign RPC failed: addr=198.18.0.1:8300 error="injected error" +2022-12-04T05:47:46.493Z [DEBUG] TestAutoEncrypt_InitialCerts: making AutoEncrypt.Sign RPC: addr=198.18.0.2:8300 +2022-12-04T05:47:46.493Z [ERROR] TestAutoEncrypt_InitialCerts: AutoEncrypt.Sign RPC failed: addr=198.18.0.2:8300 error="injected error" +2022-12-04T05:47:46.493Z [ERROR] TestAutoEncrypt_InitialCerts: No servers successfully responded to the auto-encrypt request +2022-12-04T05:47:46.494Z [DEBUG] TestAutoEncrypt_InitialCerts: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) @@ -56779,8 +56905,8 @@ mock_test.go:244: PASS: AgentToken() --- PASS: TestAutoEncrypt_InitialCerts (0.01s) === RUN TestAutoEncrypt_InitialConfiguration -2021-10-31T23:11:32.128Z [DEBUG] TestAutoEncrypt_InitialConfiguration.auto_config: making AutoEncrypt.Sign RPC: addr=127.0.0.1:8300 -2021-10-31T23:11:32.129Z [INFO] TestAutoEncrypt_InitialConfiguration.auto_config: automatically upgraded to TLS +2022-12-04T05:47:46.516Z [DEBUG] TestAutoEncrypt_InitialConfiguration.auto_config: making AutoEncrypt.Sign RPC: addr=127.0.0.1:8300 +2022-12-04T05:47:46.517Z [INFO] TestAutoEncrypt_InitialConfiguration.auto_config: automatically upgraded to TLS mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) mock_test.go:242: PASS: FindLANServer() mock_test.go:243: PASS: Prepopulate(string,cache.FetchResult,string,string,string) @@ -56788,15 +56914,15 @@ mock_test.go:244: PASS: AgentToken() mock_test.go:244: PASS: AgentToken() mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) ---- PASS: TestAutoEncrypt_InitialConfiguration (0.04s) +--- PASS: TestAutoEncrypt_InitialConfiguration (0.02s) === RUN TestAutoEncrypt_TokenUpdate -2021-10-31T23:11:32.150Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 -2021-10-31T23:11:32.151Z [INFO] TestAutoEncrypt_TokenUpdate.auto_config: automatically upgraded to TLS -2021-10-31T23:11:32.152Z [INFO] TestAutoEncrypt_TokenUpdate.auto_config: auto-config started -2021-10-31T23:11:32.152Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: handling a token update event -2021-10-31T23:11:32.152Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: Agent token updated - resetting watches -2021-10-31T23:11:32.152Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: stopping auto-config -2021-10-31T23:11:32.152Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: auto-config has been stopped +2022-12-04T05:47:46.529Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 +2022-12-04T05:47:46.530Z [INFO] TestAutoEncrypt_TokenUpdate.auto_config: automatically upgraded to TLS +2022-12-04T05:47:46.531Z [INFO] TestAutoEncrypt_TokenUpdate.auto_config: auto-config started +2022-12-04T05:47:46.531Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: handling a token update event +2022-12-04T05:47:46.531Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: Agent token updated - resetting watches +2022-12-04T05:47:46.531Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: stopping auto-config +2022-12-04T05:47:46.531Z [DEBUG] TestAutoEncrypt_TokenUpdate.auto_config: auto-config has been stopped mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) mock_test.go:242: PASS: FindLANServer() mock_test.go:243: PASS: Notify(string,string,*structs.DCSpecificRequest,string,string) @@ -56815,13 +56941,13 @@ mock_test.go:245: PASS: AutoEncryptCertNotAfter() --- PASS: TestAutoEncrypt_TokenUpdate (0.02s) === RUN TestAutoEncrypt_RootsUpdate -2021-10-31T23:11:32.188Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 -2021-10-31T23:11:32.189Z [INFO] TestAutoEncrypt_RootsUpdate.auto_config: automatically upgraded to TLS -2021-10-31T23:11:32.190Z [INFO] TestAutoEncrypt_RootsUpdate.auto_config: auto-config started -2021-10-31T23:11:32.191Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: handling a cache update event: correlation_id=roots -2021-10-31T23:11:32.191Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: roots watch fired - updating CA certificates -2021-10-31T23:11:32.191Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: stopping auto-config -2021-10-31T23:11:32.191Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: auto-config has been stopped +2022-12-04T05:47:46.549Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 +2022-12-04T05:47:46.549Z [INFO] TestAutoEncrypt_RootsUpdate.auto_config: automatically upgraded to TLS +2022-12-04T05:47:46.550Z [INFO] TestAutoEncrypt_RootsUpdate.auto_config: auto-config started +2022-12-04T05:47:46.551Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: handling a cache update event: correlation_id=roots +2022-12-04T05:47:46.551Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: roots watch fired - updating CA certificates +2022-12-04T05:47:46.551Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: stopping auto-config +2022-12-04T05:47:46.551Z [DEBUG] TestAutoEncrypt_RootsUpdate.auto_config: auto-config has been stopped mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) mock_test.go:242: PASS: FindLANServer() mock_test.go:243: PASS: Notify(string,string,*structs.DCSpecificRequest,string,string) @@ -56837,15 +56963,15 @@ mock_test.go:245: PASS: AutoEncryptCertNotAfter() mock_test.go:245: PASS: UpdateAutoTLSCA([]string) mock_test.go:245: PASS: AutoEncryptCertNotAfter() ---- PASS: TestAutoEncrypt_RootsUpdate (0.04s) +--- PASS: TestAutoEncrypt_RootsUpdate (0.02s) === RUN TestAutoEncrypt_CertUpdate -2021-10-31T23:11:32.215Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 -2021-10-31T23:11:32.216Z [INFO] TestAutoEncrypt_CertUpdate.auto_config: automatically upgraded to TLS -2021-10-31T23:11:32.216Z [INFO] TestAutoEncrypt_CertUpdate.auto_config: auto-config started -2021-10-31T23:11:32.217Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: handling a cache update event: correlation_id=leaf -2021-10-31T23:11:32.217Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: leaf certificate watch fired - updating TLS certificate -2021-10-31T23:11:32.217Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: stopping auto-config -2021-10-31T23:11:32.217Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: auto-config has been stopped +2022-12-04T05:47:46.566Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 +2022-12-04T05:47:46.567Z [INFO] TestAutoEncrypt_CertUpdate.auto_config: automatically upgraded to TLS +2022-12-04T05:47:46.568Z [INFO] TestAutoEncrypt_CertUpdate.auto_config: auto-config started +2022-12-04T05:47:46.570Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: handling a cache update event: correlation_id=leaf +2022-12-04T05:47:46.570Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: leaf certificate watch fired - updating TLS certificate +2022-12-04T05:47:46.570Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: stopping auto-config +2022-12-04T05:47:46.570Z [DEBUG] TestAutoEncrypt_CertUpdate.auto_config: auto-config has been stopped mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) mock_test.go:242: PASS: FindLANServer() mock_test.go:243: PASS: Notify(string,string,*structs.DCSpecificRequest,string,string) @@ -56861,17 +56987,17 @@ mock_test.go:245: PASS: AutoEncryptCertNotAfter() mock_test.go:245: PASS: UpdateAutoTLSCert(string,string) mock_test.go:245: PASS: AutoEncryptCertNotAfter() ---- PASS: TestAutoEncrypt_CertUpdate (0.04s) +--- PASS: TestAutoEncrypt_CertUpdate (0.02s) === RUN TestAutoEncrypt_Fallback -2021-10-31T23:11:32.268Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 -2021-10-31T23:11:32.275Z [INFO] TestAutoEncrypt_Fallback.auto_config: automatically upgraded to TLS -2021-10-31T23:11:32.275Z [INFO] TestAutoEncrypt_Fallback.auto_config: auto-config started -2021-10-31T23:11:32.287Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: handling a cache update event: correlation_id=leaf -2021-10-31T23:11:32.287Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: leaf certificate watch fired - updating TLS certificate -2021-10-31T23:11:32.287Z [WARN] TestAutoEncrypt_Fallback.auto_config: agent's client certificate has expired -2021-10-31T23:11:32.288Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.23.2:8300 -2021-10-31T23:11:32.289Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: stopping auto-config -2021-10-31T23:11:32.289Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: auto-config has been stopped +2022-12-04T05:47:46.584Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.0.1:8300 +2022-12-04T05:47:46.585Z [INFO] TestAutoEncrypt_Fallback.auto_config: automatically upgraded to TLS +2022-12-04T05:47:46.585Z [INFO] TestAutoEncrypt_Fallback.auto_config: auto-config started +2022-12-04T05:47:46.588Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: handling a cache update event: correlation_id=leaf +2022-12-04T05:47:46.588Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: leaf certificate watch fired - updating TLS certificate +2022-12-04T05:47:46.588Z [WARN] TestAutoEncrypt_Fallback.auto_config: agent's client certificate has expired +2022-12-04T05:47:46.589Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: making AutoEncrypt.Sign RPC: addr=198.18.23.2:8300 +2022-12-04T05:47:46.590Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: stopping auto-config +2022-12-04T05:47:46.590Z [DEBUG] TestAutoEncrypt_Fallback.auto_config: auto-config has been stopped mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) mock_test.go:241: PASS: RPC(string,string,*net.TCPAddr,string,*structs.CASignRequest,*structs.SignedResponse) mock_test.go:242: PASS: FindLANServer() @@ -56896,18 +57022,19 @@ mock_test.go:245: PASS: AutoEncryptCertExpired() mock_test.go:245: PASS: UpdateAutoTLS([]string,[]string,string,string,bool) mock_test.go:245: PASS: AutoEncryptCertNotAfter() ---- PASS: TestAutoEncrypt_Fallback (0.07s) +--- PASS: TestAutoEncrypt_Fallback (0.02s) === RUN TestTranslateConfig --- PASS: TestTranslateConfig (0.00s) === RUN TestCArootsTranslation --- PASS: TestCArootsTranslation (0.00s) PASS -ok github.com/hashicorp/consul/agent/auto-config 1.232s +ok github.com/hashicorp/consul/agent/auto-config 0.782s === RUN TestBuildAndValidate_HTTPMaxConnsPerClientExceedsRLimit === PAUSE TestBuildAndValidate_HTTPMaxConnsPerClientExceedsRLimit === RUN TestBuilder_validateEnterpriseConfigKeys === RUN TestBuilder_validateEnterpriseConfigKeys/non_voting_server === RUN TestBuilder_validateEnterpriseConfigKeys/segment +=== RUN TestBuilder_validateEnterpriseConfigKeys/segments === RUN TestBuilder_validateEnterpriseConfigKeys/autopilot.redundancy_zone_tag === RUN TestBuilder_validateEnterpriseConfigKeys/autopilot.upgrade_version_tag === RUN TestBuilder_validateEnterpriseConfigKeys/autopilot.disable_upgrade_migration @@ -56915,10 +57042,10 @@ === RUN TestBuilder_validateEnterpriseConfigKeys/acl.msp_disable_bootstrap === RUN TestBuilder_validateEnterpriseConfigKeys/acl.tokens.managed_service_provider === RUN TestBuilder_validateEnterpriseConfigKeys/multi -=== RUN TestBuilder_validateEnterpriseConfigKeys/segments --- PASS: TestBuilder_validateEnterpriseConfigKeys (0.00s) --- PASS: TestBuilder_validateEnterpriseConfigKeys/non_voting_server (0.00s) --- PASS: TestBuilder_validateEnterpriseConfigKeys/segment (0.00s) + --- PASS: TestBuilder_validateEnterpriseConfigKeys/segments (0.00s) --- PASS: TestBuilder_validateEnterpriseConfigKeys/autopilot.redundancy_zone_tag (0.00s) --- PASS: TestBuilder_validateEnterpriseConfigKeys/autopilot.upgrade_version_tag (0.00s) --- PASS: TestBuilder_validateEnterpriseConfigKeys/autopilot.disable_upgrade_migration (0.00s) @@ -56926,7 +57053,6 @@ --- PASS: TestBuilder_validateEnterpriseConfigKeys/acl.msp_disable_bootstrap (0.00s) --- PASS: TestBuilder_validateEnterpriseConfigKeys/acl.tokens.managed_service_provider (0.00s) --- PASS: TestBuilder_validateEnterpriseConfigKeys/multi (0.00s) - --- PASS: TestBuilder_validateEnterpriseConfigKeys/segments (0.00s) === RUN TestLoad --- PASS: TestLoad (0.02s) === RUN TestShouldParseFile @@ -57332,343 +57458,343 @@ === RUN TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_invalid_claim_assertion === RUN TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_ok === RUN TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_ok ---- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases (11.78s) +--- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases (7.99s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-advertise (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-advertise-wan (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-advertise_and_-advertise-wan (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-bind (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-bootstrap (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-bootstrap-expect (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-client (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-dir (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-advertise-wan (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-advertise_and_-advertise-wan (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-bind (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-bootstrap (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-bootstrap-expect (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-client (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-dir (0.05s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-file_json (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-file_hcl_and_json (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-data-dir_empty (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-data-dir_non-directory (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-file_hcl_and_json (0.09s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-data-dir_empty (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-data-dir_non-directory (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-datacenter (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-datacenter_empty (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-dev (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-disable-host-node-id (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-disable-keyring-file (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-dns-port (0.08s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-domain (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-alt-domain (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-alt-domain_can't_be_prefixed_by_DC (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-datacenter_empty (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-dev (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-disable-host-node-id (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-disable-keyring-file (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-dns-port (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-domain (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-alt-domain (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-alt-domain_can't_be_prefixed_by_DC (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-alt-domain_can't_be_prefixed_by_service (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-alt-domain_can_be_prefixed_by_non-keywords (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-alt-domain_can't_be_prefixed_by_DC#01 (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-enable-script-checks (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-encrypt (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-format_disabled,_skip_unknown_files (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-format=json (0.09s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-format=hcl (0.07s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-http-port (0.34s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-join (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-alt-domain_can_be_prefixed_by_non-keywords (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-alt-domain_can't_be_prefixed_by_DC#01 (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-enable-script-checks (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-encrypt (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-format_disabled,_skip_unknown_files (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-format=json (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-config-format=hcl (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-http-port (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-join (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-join-wan (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-log-level (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-log-json (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:-log-rotate-max-files (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:-log-rotate-max-files (0.07s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-node (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-node-id (0.07s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-node-meta (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-non-voting-server (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-log-level (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-log-json (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:-log-rotate-max-files (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:-log-rotate-max-files (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-node (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-node-id (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-node-meta (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-non-voting-server (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-pid-file (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:-primary-gateway (0.08s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:-primary-gateway (0.07s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-protocol (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-raft-protocol (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-recursor (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-rejoin (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-interval (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-interval-wan (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-join (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-join-wan (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-max (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-max-wan (0.07s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-serf-lan-bind (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-serf-lan-port (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-serf-wan-bind (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-serf-wan-port (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:-primary-gateway (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:-primary-gateway (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-protocol (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-raft-protocol (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-recursor (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-rejoin (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-interval (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-interval-wan (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-join (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-join-wan (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-max (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-retry-max-wan (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-serf-lan-bind (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-serf-lan-port (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-serf-wan-bind (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-serf-wan-port (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-server (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-server-port (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-syslog (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-ui (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-ui-dir (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-ui-content-path (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_any_v4 (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_any_v4 (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_any_v6 (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_any_v6 (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_any_and_advertise_set_should_not_detect (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_any_and_advertise_set_should_not_detect (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr_and_ports_==_0 (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_addr_and_ports_==_0 (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr_and_ports_<_0 (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_addr_and_ports_<_0 (0.08s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr_and_ports_>_0 (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_addr_and_ports_>_0 (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr,_addresses_and_ports_==_0 (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-server-port (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-syslog (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-ui (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-ui-dir (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/-ui-content-path (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_any_v4 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_any_v4 (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_any_v6 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_any_v6 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_any_and_advertise_set_should_not_detect (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_any_and_advertise_set_should_not_detect (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr_and_ports_==_0 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_addr_and_ports_==_0 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr_and_ports_<_0 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_addr_and_ports_<_0 (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr_and_ports_>_0 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_addr_and_ports_>_0 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr,_addresses_and_ports_==_0 (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_addr,_addresses_and_ports_==_0 (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr,_addresses_and_ports_<_0 (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_addr,_addresses_and_ports_<_0 (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr,_addresses_and_ports (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr,_addresses_and_ports_<_0 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_addr,_addresses_and_ports_<_0 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_addr,_addresses_and_ports (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_addr,_addresses_and_ports (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_template_and_ports (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_template_and_ports (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client,_address_template_and_ports (0.07s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client,_address_template_and_ports (0.07s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_lan_template (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_lan_template (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_wan_template (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_wan_template (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_lan_with_ports (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_lan_with_ports (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_wan_with_ports (0.07s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_wan_with_ports (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:allow_disabling_serf_wan_port (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:allow_disabling_serf_wan_port (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:serf_bind_address_lan_template (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:serf_bind_address_lan_template (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:serf_bind_address_wan_template (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:serf_bind_address_wan_template (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:dns_recursor_templates_with_deduplication (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:dns_recursor_templates_with_deduplication (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_template_and_ports (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_template_and_ports (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client,_address_template_and_ports (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client,_address_template_and_ports (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_lan_template (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_lan_template (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_wan_template (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_wan_template (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_lan_with_ports (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_lan_with_ports (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_wan_with_ports (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_wan_with_ports (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:allow_disabling_serf_wan_port (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:allow_disabling_serf_wan_port (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:serf_bind_address_lan_template (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:serf_bind_address_lan_template (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:serf_bind_address_wan_template (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:serf_bind_address_wan_template (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:dns_recursor_templates_with_deduplication (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:dns_recursor_templates_with_deduplication (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:start_join_address_template (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:start_join_address_template (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:start_join_wan_address_template (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:start_join_address_template (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:start_join_wan_address_template (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:start_join_wan_address_template (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:retry_join_address_template (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:retry_join_address_template (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:retry_join_wan_address_template (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:retry_join_wan_address_template (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:min/max_ports_for_dynamic_exposed_listeners (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:min/max_ports_for_dynamic_exposed_listeners (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/defaults_for_dynamic_exposed_listeners (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:precedence:_merge_order (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:retry_join_address_template (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:retry_join_address_template (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:retry_join_wan_address_template (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:retry_join_wan_address_template (0.08s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:min/max_ports_for_dynamic_exposed_listeners (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:min/max_ports_for_dynamic_exposed_listeners (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/defaults_for_dynamic_exposed_listeners (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:precedence:_merge_order (0.04s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:precedence:_merge_order (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:precedence:_flag_before_file (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:precedence:_flag_before_file (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:raft_performance_scaling (0.07s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:raft_performance_scaling (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/Serf_Allowed_CIDRS_LAN,_multiple_values_from_flags (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:precedence:_flag_before_file (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:precedence:_flag_before_file (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:raft_performance_scaling (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:raft_performance_scaling (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/Serf_Allowed_CIDRS_LAN,_multiple_values_from_flags (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:Serf_Allowed_CIDRS_LAN/WAN,_multiple_values_from_HCL/JSON (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:Serf_Allowed_CIDRS_LAN/WAN,_multiple_values_from_HCL/JSON (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/Serf_Allowed_CIDRS_WAN,_multiple_values_from_flags (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:invalid_input (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:invalid_input (0.00s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:datacenter_is_lower-cased (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:datacenter_is_lower-cased (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:acl_datacenter_is_lower-cased (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:acl_datacenter_is_lower-cased (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:Serf_Allowed_CIDRS_LAN/WAN,_multiple_values_from_HCL/JSON (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/Serf_Allowed_CIDRS_WAN,_multiple_values_from_flags (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:invalid_input (0.00s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:invalid_input (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:datacenter_is_lower-cased (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:datacenter_is_lower-cased (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:acl_datacenter_is_lower-cased (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:acl_datacenter_is_lower-cased (0.05s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:acl_replication_token_enables_acl_replication (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:acl_replication_token_enables_acl_replication (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:acl_enforce_version_8_is_deprecated (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:acl_replication_token_enables_acl_replication (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:acl_enforce_version_8_is_deprecated (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:acl_enforce_version_8_is_deprecated (0.04s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_fails_v4 (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_fails_v4 (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_none_v4 (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_none_v4 (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_multiple_v4 (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_fails_v4 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_none_v4 (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_none_v4 (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_multiple_v4 (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_multiple_v4 (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_fails_v6 (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_fails_v6 (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_fails_v6 (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_none_v6 (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_none_v6 (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_multiple_v6 (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_none_v6 (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_none_v6 (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_address_detect_multiple_v6 (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_address_detect_multiple_v6 (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/ae_interval_invalid_==_0 (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/ae_interval_invalid_<_0 (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:acl_datacenter_invalid (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:acl_datacenter_invalid (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:autopilot.max_trailing_logs_invalid (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:autopilot.max_trailing_logs_invalid (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/ae_interval_invalid_==_0 (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/ae_interval_invalid_<_0 (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:acl_datacenter_invalid (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:acl_datacenter_invalid (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:autopilot.max_trailing_logs_invalid (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:autopilot.max_trailing_logs_invalid (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_cannot_be_empty (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_cannot_be_empty (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_cannot_be_empty (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_does_not_allow_multiple_addresses (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_does_not_allow_multiple_addresses (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_cannot_be_a_unix_socket (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_cannot_be_a_unix_socket (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_does_not_allow_multiple_addresses (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bind_addr_cannot_be_a_unix_socket (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bind_addr_cannot_be_a_unix_socket (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap_without_server (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap_without_server (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap_without_server (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_without_server (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_without_server (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_invalid (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_without_server (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_invalid (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_invalid (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_and_dev_mode (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_and_dev_mode (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_and_bootstrap (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_and_bootstrap (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_and_dev_mode (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_and_dev_mode (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_and_bootstrap (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_and_bootstrap (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect=1_equals_bootstrap (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect=1_equals_bootstrap (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect=2_warning (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect=2_warning (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_>_2_but_even_warning (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_>_2_but_even_warning (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_mode_sets_LeaveOnTerm_and_SkipLeaveOnInt_correctly (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_mode_sets_LeaveOnTerm_and_SkipLeaveOnInt_correctly (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect=1_equals_bootstrap (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect=2_warning (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect=2_warning (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:bootstrap-expect_>_2_but_even_warning (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:bootstrap-expect_>_2_but_even_warning (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_mode_sets_LeaveOnTerm_and_SkipLeaveOnInt_correctly (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_mode_sets_LeaveOnTerm_and_SkipLeaveOnInt_correctly (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:client_does_not_allow_socket (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_does_not_allow_socket (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:client_does_not_allow_socket (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:datacenter_invalid (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:datacenter_invalid (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:dns_does_not_allow_socket (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:dns_does_not_allow_socket (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ui_and_ui_dir (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ui_and_ui_dir (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:dns_does_not_allow_socket (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:dns_does_not_allow_socket (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ui_and_ui_dir (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ui_and_ui_dir (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_addr_any (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_addr_any (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:advertise_addr_wan_any (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:advertise_addr_wan_any (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:recursors_any (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:recursors_any (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:dns_config.udp_answer_limit_invalid (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:dns_config.udp_answer_limit_invalid (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:dns_config.udp_answer_limit_invalid (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:dns_config.a_record_limit_invalid (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:dns_config.a_record_limit_invalid (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:performance.raft_multiplier_<_0 (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:performance.raft_multiplier_<_0 (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:performance.raft_multiplier_==_0 (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:performance.raft_multiplier_==_0 (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:dns_config.a_record_limit_invalid (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:performance.raft_multiplier_<_0 (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:performance.raft_multiplier_<_0 (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:performance.raft_multiplier_==_0 (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:performance.raft_multiplier_==_0 (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:performance.raft_multiplier_>_10 (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:performance.raft_multiplier_>_10 (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/node_name_invalid (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/node_name_invalid (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:node_meta_key_too_long (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:node_meta_key_too_long (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:node_meta_value_too_long (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:node_meta_key_too_long (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:node_meta_value_too_long (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:node_meta_value_too_long (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:node_meta_too_many_keys (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:node_meta_too_many_keys (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:unique_listeners_dns_vs_http (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:node_meta_too_many_keys (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:unique_listeners_dns_vs_http (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:unique_listeners_dns_vs_http (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:unique_listeners_dns_vs_https (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:unique_listeners_dns_vs_https (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:unique_listeners_dns_vs_https (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:unique_listeners_http_vs_https (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:unique_listeners_http_vs_https (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:unique_listeners_http_vs_https (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:unique_listeners_http_vs_https (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:unique_advertise_addresses_HTTP_vs_RPC (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:unique_advertise_addresses_HTTP_vs_RPC (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:unique_advertise_addresses_RPC_vs_Serf_LAN (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:unique_advertise_addresses_RPC_vs_Serf_LAN (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:unique_advertise_addresses_RPC_vs_Serf_LAN (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:unique_advertise_addresses_RPC_vs_Serf_LAN (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:unique_advertise_addresses_RPC_vs_Serf_WAN (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:unique_advertise_addresses_RPC_vs_Serf_WAN (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:http_use_cache_defaults_to_true (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:http_use_cache_defaults_to_true (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:http_use_cache_is_enabled_when_true (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:http_use_cache_is_enabled_when_true (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:http_use_cache_is_disabled_when_false (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:http_use_cache_is_disabled_when_false (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:sidecar_service_can't_have_ID (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:sidecar_service_can't_have_ID (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:unique_advertise_addresses_RPC_vs_Serf_WAN (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:http_use_cache_defaults_to_true (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:http_use_cache_defaults_to_true (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:http_use_cache_is_enabled_when_true (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:http_use_cache_is_enabled_when_true (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:http_use_cache_is_disabled_when_false (0.06s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:http_use_cache_is_disabled_when_false (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:sidecar_service_can't_have_ID (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:sidecar_service_can't_have_ID (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:sidecar_service_can't_have_nested_sidecar (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:sidecar_service_can't_have_nested_sidecar (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:sidecar_service_can't_have_nested_sidecar (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:telemetry.prefix_filter_cannot_be_empty (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:telemetry.prefix_filter_cannot_be_empty (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:telemetry.prefix_filter_cannot_be_empty (0.04s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:telemetry.prefix_filter_must_start_with_+_or_- (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:telemetry.prefix_filter_must_start_with_+_or_- (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:telemetry.prefix_filter_must_start_with_+_or_- (0.04s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:encrypt_has_invalid_key (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:encrypt_has_invalid_key (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:multiple_check_files (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:multiple_check_files (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:multiple_check_files (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:grpc_check (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:grpc_check (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:alias_check_with_no_node (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:grpc_check (0.06s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:alias_check_with_no_node (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:alias_check_with_no_node (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:multiple_service_files (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:multiple_service_files (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:service_with_wrong_meta:_too_long_key (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:service_with_wrong_meta:_too_long_key (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:multiple_service_files (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:multiple_service_files (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:service_with_wrong_meta:_too_long_key (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:service_with_wrong_meta:_too_long_key (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:service_with_wrong_meta:_too_long_value (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:service_with_wrong_meta:_too_long_value (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:service_with_wrong_meta:_too_many_meta (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:service_with_wrong_meta:_too_many_meta (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:translated_keys (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:translated_keys (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ignore_snapshot_agent_sub-object (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:service_with_wrong_meta:_too_many_meta (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:service_with_wrong_meta:_too_many_meta (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:translated_keys (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:translated_keys (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ignore_snapshot_agent_sub-object (0.07s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ignore_snapshot_agent_sub-object (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:service.connectsidecar_service_with_checks_and_upstreams (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:service.connectsidecar_service_with_checks_and_upstreams (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:services.connect.sidecar_service_with_checks_and_upstreams (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:services.connect.sidecar_service_with_checks_and_upstreams (0.07s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:service.connectsidecar_service_with_checks_and_upstreams (0.06s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:service.connectsidecar_service_with_checks_and_upstreams (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:services.connect.sidecar_service_with_checks_and_upstreams (0.06s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:services.connect.sidecar_service_with_checks_and_upstreams (0.04s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:verify_server_hostname_implies_verify_outgoing (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:verify_server_hostname_implies_verify_outgoing (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_works_implies_connect (0.07s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_works_implies_connect (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_works_with_verify_incoming (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_works_with_verify_incoming (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_works_with_verify_incoming_rpc (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_works_with_verify_incoming_rpc (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_warns_without_verify_incoming_or_verify_incoming_rpc (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_warns_without_verify_incoming_or_verify_incoming_rpc (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_errors_in_client_mode (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:verify_server_hostname_implies_verify_outgoing (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_works_implies_connect (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_works_implies_connect (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_works_with_verify_incoming (0.07s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_works_with_verify_incoming (0.06s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_works_with_verify_incoming_rpc (0.06s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_works_with_verify_incoming_rpc (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_warns_without_verify_incoming_or_verify_incoming_rpc (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_warns_without_verify_incoming_or_verify_incoming_rpc (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.allow_tls_errors_in_client_mode (0.04s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.allow_tls_errors_in_client_mode (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_encrypt.tls_errors_in_server_mode (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.tls_errors_in_server_mode (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:test_connect_vault_provider_configuration (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:test_connect_vault_provider_configuration (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:Connect_AWS_CA_provider_configuration (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:Connect_AWS_CA_provider_configuration (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:Connect_AWS_CA_provider_TTL_validation (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:Connect_AWS_CA_provider_TTL_validation (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_encrypt.tls_errors_in_server_mode (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:test_connect_vault_provider_configuration (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:test_connect_vault_provider_configuration (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:Connect_AWS_CA_provider_configuration (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:Connect_AWS_CA_provider_configuration (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:Connect_AWS_CA_provider_TTL_validation (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:Connect_AWS_CA_provider_TTL_validation (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:Connect_AWS_CA_provider_EC_key_length_validation (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:Connect_AWS_CA_provider_EC_key_length_validation (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_requires_connect.enabled (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_requires_connect.enabled (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_cannot_use_-join-wan (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:Connect_AWS_CA_provider_EC_key_length_validation (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_requires_connect.enabled (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_requires_connect.enabled (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_cannot_use_-join-wan (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_cannot_use_-join-wan (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_cannot_use_-retry-join-wan (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_cannot_use_-retry-join-wan (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_cannot_use_-retry-join-wan (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_requires_server_mode (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_requires_server_mode (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_requires_server_mode (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_requires_no_slashes_in_node_names (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_requires_no_slashes_in_node_names (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_requires_no_slashes_in_node_names (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:primary_gateways_requires_server_mode (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:primary_gateways_requires_server_mode (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:primary_gateways_only_works_in_a_secondary_datacenter (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:primary_gateways_only_works_in_a_secondary_datacenter (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:primary_gateways_requires_server_mode (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:primary_gateways_only_works_in_a_secondary_datacenter (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:primary_gateways_only_works_in_a_secondary_datacenter (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:connect.enable_mesh_gateway_wan_federation_in_secondary_with_primary_gateways_configured (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_in_secondary_with_primary_gateways_configured (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:connect.enable_mesh_gateway_wan_federation_in_secondary_with_primary_gateways_configured (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_doesn't_parse (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_doesn't_parse (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_unknown_kind (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_doesn't_parse (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_unknown_kind (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_unknown_kind (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_invalid_service-defaults (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_invalid_service-defaults (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_proxy-defaults_(snake-case) (0.05s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_proxy-defaults_(snake-case) (0.07s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_proxy-defaults_(camel-case) (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_invalid_service-defaults (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_invalid_service-defaults (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_proxy-defaults_(snake-case) (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_proxy-defaults_(snake-case) (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_proxy-defaults_(camel-case) (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_proxy-defaults_(camel-case) (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_service-defaults_(snake-case) (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_service-defaults_(snake-case) (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_service-defaults_(snake-case) (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_service-defaults_(snake-case) (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_service-defaults_(camel-case) (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_service-defaults_(camel-case) (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_service-defaults_(camel-case) (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:ConfigEntry_bootstrap_service-router_(snake-case) (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_service-router_(snake-case) (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/default_limits (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:ConfigEntry_bootstrap_service-router_(snake-case) (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/default_limits (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_and_auto_encrypt_error (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_and_auto_encrypt_error (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_not_allowed_for_servers (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_not_allowed_for_servers (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_not_allowed_for_servers (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_not_allowed_for_servers (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_tls_not_enabled (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_tls_not_enabled (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_server_tls_not_enabled (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_server_tls_not_enabled (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_no_intro_token (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_no_intro_token (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_server_tls_not_enabled (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_server_tls_not_enabled (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_no_intro_token (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_no_intro_token (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_no_server_addresses (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_no_server_addresses (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_client (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_client (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_client_not_allowed (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_client_not_allowed (0.01s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_invalid_config (0.01s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_client (0.06s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_client (0.05s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_client_not_allowed (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_client_not_allowed (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_invalid_config (0.02s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_invalid_config (0.03s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_invalid_config_2 (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_invalid_config_2 (0.02s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_require_token_replication_in_secondary (0.04s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_invalid_config_2 (0.03s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_require_token_replication_in_secondary (0.01s) --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_require_token_replication_in_secondary (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_invalid_claim_assertion (0.03s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_invalid_claim_assertion (0.04s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_ok (0.06s) - --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_ok (0.07s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_invalid_claim_assertion (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_invalid_claim_assertion (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/json:auto_config_authorizer_ok (0.02s) + --- PASS: TestBuilder_BuildAndValide_ConfigFlagsAndEdgecases/hcl:auto_config_authorizer_ok (0.02s) === RUN TestNewBuilder_InvalidConfigFormat --- PASS: TestNewBuilder_InvalidConfigFormat (0.00s) === RUN TestFullConfig @@ -57699,7 +57825,7 @@ === RUN TestNonZero/struct_with_nil_field === RUN TestNonZero/struct_with_zero_value_field === RUN TestNonZero/struct_with_empty_array ---- PASS: TestNonZero (0.01s) +--- PASS: TestNonZero (0.00s) --- PASS: TestNonZero/nil (0.00s) --- PASS: TestNonZero/zero_bool (0.00s) --- PASS: TestNonZero/zero_string (0.00s) @@ -57727,13 +57853,13 @@ === RUN TestConfigDecodeBytes === PAUSE TestConfigDecodeBytes === RUN TestSanitize ---- PASS: TestSanitize (0.00s) +--- PASS: TestSanitize (0.01s) === RUN TestRuntime_apiAddresses --- PASS: TestRuntime_apiAddresses (0.00s) === RUN TestRuntime_APIConfigHTTPS --- PASS: TestRuntime_APIConfigHTTPS (0.00s) === RUN TestRuntime_APIConfigHTTP ---- PASS: TestRuntime_APIConfigHTTP (0.01s) +--- PASS: TestRuntime_APIConfigHTTP (0.00s) === RUN TestRuntime_APIConfigUNIX --- PASS: TestRuntime_APIConfigUNIX (0.00s) === RUN TestRuntime_APIConfigANYAddrV4 @@ -57753,30 +57879,30 @@ === RUN Test_UIPathBuilder --- PASS: Test_UIPathBuilder (0.00s) === RUN TestConnectCAConfiguration +=== RUN TestConnectCAConfiguration/cluster-id-override +=== RUN TestConnectCAConfiguration/cluster-id-non-uuid === RUN TestConnectCAConfiguration/provider-override === RUN TestConnectCAConfiguration/other-config === RUN TestConnectCAConfiguration/defaults -=== RUN TestConnectCAConfiguration/cluster-id-override -=== RUN TestConnectCAConfiguration/cluster-id-non-uuid --- PASS: TestConnectCAConfiguration (0.00s) + --- PASS: TestConnectCAConfiguration/cluster-id-override (0.00s) + --- PASS: TestConnectCAConfiguration/cluster-id-non-uuid (0.00s) --- PASS: TestConnectCAConfiguration/provider-override (0.00s) --- PASS: TestConnectCAConfiguration/other-config (0.00s) --- PASS: TestConnectCAConfiguration/defaults (0.00s) - --- PASS: TestConnectCAConfiguration/cluster-id-override (0.00s) - --- PASS: TestConnectCAConfiguration/cluster-id-non-uuid (0.00s) === RUN TestSegments segment_oss_test.go:13: DM-skipped --- SKIP: TestSegments (0.00s) === CONT TestBuildAndValidate_HTTPMaxConnsPerClientExceedsRLimit === CONT TestConfigDecodeBytes --- PASS: TestConfigDecodeBytes (0.00s) ---- PASS: TestBuildAndValidate_HTTPMaxConnsPerClientExceedsRLimit (0.01s) +--- PASS: TestBuildAndValidate_HTTPMaxConnsPerClientExceedsRLimit (0.02s) PASS -ok github.com/hashicorp/consul/agent/config 12.340s +ok github.com/hashicorp/consul/agent/config 8.151s === RUN TestCollectHostInfo ---- PASS: TestCollectHostInfo (0.01s) +--- PASS: TestCollectHostInfo (0.00s) PASS -ok github.com/hashicorp/consul/agent/debug 0.099s +ok github.com/hashicorp/consul/agent/debug 0.013s ? github.com/hashicorp/consul/agent/dns [no test files] ? github.com/hashicorp/consul/agent/exec [no test files] === RUN TestAgentAntiEntropy_Services @@ -57824,977 +57950,985 @@ === PAUSE TestAliasNotifications_local === CONT TestAgentAntiEntropy_Services [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestAgent_ServiceTokens +=== CONT TestAgentAntiEntropy_NodeInfo === CONT TestAgentAntiEntropy_Checks +=== CONT TestAgent_AliasCheck_ServiceNotification === CONT TestAliasNotifications_local -=== CONT TestAgentAntiEntropy_NodeInfo -=== CONT TestAgentAntiEntropy_Check_DeferSync -=== CONT TestState_Notify -=== CONT TestAgent_UpdateCheck_DiscardOutput -=== CONT TestAgentAntiEntropy_Checks_ACLDeny -=== CONT TestAgentAntiEntropy_RemovingServiceAndCheck ---- PASS: TestState_Notify (0.00s) -=== CONT TestAgent_sendCoordinate === CONT TestAgent_AliasCheck -=== CONT TestAgent_AliasCheck_ServiceNotification +=== CONT TestAgentAntiEntropy_Checks_ACLDeny +=== CONT TestAgent_ServiceTokens === CONT TestAgent_AddCheckFailure === CONT TestAgent_CheckTokens === CONT TestAgentAntiEntropy_EnableTagOverride +=== CONT TestAgent_UpdateCheck_DiscardOutput +=== CONT TestAgentAntiEntropy_Check_DeferSync +=== CONT TestAgentAntiEntropy_Services_WithChecks +=== CONT TestAgent_ServiceWatchCh +=== CONT TestAgentAntiEntropy_Services_ACLDeny [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range ---- PASS: TestAgent_AliasCheck_ServiceNotification (0.04s) -=== CONT TestAgentAntiEntropy_Services_ACLDeny ---- PASS: TestAgent_CheckTokens (0.04s) -=== CONT TestAgentAntiEntropy_Services_WithChecks ---- PASS: TestAgent_ServiceTokens (0.05s) +--- PASS: TestAgent_AliasCheck_ServiceNotification (0.03s) +=== CONT TestAgentAntiEntropy_RemovingServiceAndCheck +--- PASS: TestAgent_ServiceTokens (0.04s) === CONT TestAgentAntiEntropy_Services_ConnectProxy ---- PASS: TestAgent_AliasCheck (0.07s) -=== CONT TestAgent_ServiceWatchCh ---- PASS: TestAgent_AddCheckFailure (0.09s) +--- PASS: TestAgent_AliasCheck (0.04s) +=== CONT TestState_Notify +--- PASS: TestState_Notify (0.00s) +=== CONT TestAgent_sendCoordinate +--- PASS: TestAgent_AddCheckFailure (0.04s) +--- PASS: TestAgent_CheckTokens (0.04s) === CONT TestAgent_sendCoordinate state_test.go:1985: 128 5 100ms -2021-10-31T23:11:46.608Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:46.727Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:46.729 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:40e4a748-2192-161a-0510-9bf59fe950b5 Address:127.0.0.1:24314}]" -11:46.729 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24314 [Follower]" leader= -11:46.730 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4da0ee22-9946-a287-489f-5ec62eee08b3.dc1 127.0.0.1 -11:46.731 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4da0ee22-9946-a287-489f-5ec62eee08b3 127.0.0.1 -2021-10-31T23:11:46.731Z [INFO] agent.router: Initializing LAN area manager -11:46.731 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24309 network=udp -11:46.731 [INFO] TestAgent.server: Adding LAN server: server="Node-4da0ee22-9946-a287-489f-5ec62eee08b3 (Addr: tcp/127.0.0.1:24314) (DC: dc1)" -11:46.731 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4da0ee22-9946-a287-489f-5ec62eee08b3.dc1 area=wan -11:46.743 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24309 network=tcp -11:46.745 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24310 network=tcp -11:46.745 [INFO] TestAgent: started state syncer -11:46.745 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24315 network=tcp -11:46.774 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:46.774 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24314 [Candidate]" term=2 -11:46.775 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:46.775 [DEBUG] TestAgent.server.raft: vote granted: from=40e4a748-2192-161a-0510-9bf59fe950b5 term=2 tally=1 -11:46.775 [INFO] TestAgent.server.raft: election won: tally=1 -11:46.775 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24314 [Leader]" -11:46.775 [INFO] TestAgent.server: cluster leadership acquired -11:46.789 [INFO] TestAgent.server: New leader elected: payload=Node-4da0ee22-9946-a287-489f-5ec62eee08b3 -11:46.789 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24314 -11:46.818 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:47.058 [INFO] TestAgent: Synced node info -11:47.058 [DEBUG] TestAgent: Node info in sync -11:47.133 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:47.133 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:47.133 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:47.133 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:47.133 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:47.134 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4da0ee22-9946-a287-489f-5ec62eee08b3 -11:47.147 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4da0ee22-9946-a287-489f-5ec62eee08b3 -11:47.162 [INFO] TestAgent: Synced node info -11:47.203 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:47.203 [INFO] TestAgent: Synced node info -11:47.203 [INFO] TestAgent: Requesting shutdown -11:47.204 [INFO] TestAgent.server: shutting down server -11:47.204 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:47.204 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:47.204 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:47.204 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:47.204 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:47.204 [INFO] TestAgent.server: federation state anti-entropy synced -11:47.204 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:47.204 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:47.204 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:47.204 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:47.204 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:47.204Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:47.204Z [INFO] agent.router.manager: shutting down -11:47.204 [INFO] TestAgent: consul server down -11:47.204 [INFO] TestAgent: shutdown complete -11:47.204 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24309 network=tcp -11:47.204 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24309 network=udp -11:47.204 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24310 network=tcp -11:47.704 [INFO] TestAgent: Waiting for endpoints to shut down -11:47.704 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_NodeInfo (1.27s) -2021-10-31T23:11:46.863Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:47.079Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:47.082 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:54733646-0abd-7f89-d988-9b46c6821e6b Address:127.0.0.1:24349}]" -11:47.082 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-54733646-0abd-7f89-d988-9b46c6821e6b.dc1 127.0.0.1 -11:47.083 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-54733646-0abd-7f89-d988-9b46c6821e6b 127.0.0.1 -2021-10-31T23:11:47.083Z [INFO] agent.router: Initializing LAN area manager -11:47.083 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24344 network=udp -11:47.083 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24349 [Follower]" leader= -11:47.086 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-54733646-0abd-7f89-d988-9b46c6821e6b.dc1 area=wan -11:47.086 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24344 network=tcp -11:47.087 [INFO] TestAgent.server: Adding LAN server: server="Node-54733646-0abd-7f89-d988-9b46c6821e6b (Addr: tcp/127.0.0.1:24349) (DC: dc1)" -11:47.087 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24345 network=tcp -11:47.087 [INFO] TestAgent: started state syncer -11:47.087 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24350 network=tcp -11:47.128 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:47.128 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24349 [Candidate]" term=2 -11:47.129 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:47.129 [DEBUG] TestAgent.server.raft: vote granted: from=54733646-0abd-7f89-d988-9b46c6821e6b term=2 tally=1 -11:47.129 [INFO] TestAgent.server.raft: election won: tally=1 -11:47.129 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24349 [Leader]" -11:47.131 [INFO] TestAgent.server: cluster leadership acquired -11:47.131 [INFO] TestAgent.server: New leader elected: payload=Node-54733646-0abd-7f89-d988-9b46c6821e6b -11:47.131 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24349 -11:47.194 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:47.196 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:47.196 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:47.196 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:47.196 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:47.196 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:47.196 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-54733646-0abd-7f89-d988-9b46c6821e6b -11:47.196 [INFO] TestAgent.server: member joined, marking health alive: member=Node-54733646-0abd-7f89-d988-9b46c6821e6b -11:47.201 [INFO] TestAgent.server: federation state anti-entropy synced -11:47.202 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:47.203 [INFO] TestAgent: Synced node info -11:47.203 [INFO] TestAgent: Deregistered service: service=deleted-check-service -11:47.203 [INFO] TestAgent: Deregistered check: check=lb -11:47.203 [INFO] TestAgent: Requesting shutdown -11:47.203 [INFO] TestAgent.server: shutting down server -11:47.203 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:47.203 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:47.203 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:47.203 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:47.203 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:47.203 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:47.203 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:47.203 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:47.204 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:47.204 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:47.204 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:47.204 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:47.204Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:47.204Z [INFO] agent.router.manager: shutting down -11:47.204 [INFO] TestAgent: consul server down -11:47.204 [INFO] TestAgent: shutdown complete -11:47.204 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24344 network=tcp -11:47.204 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24344 network=udp -11:47.204 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24345 network=tcp -11:47.705 [INFO] TestAgent: Waiting for endpoints to shut down -11:47.705 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_RemovingServiceAndCheck (1.27s) -2021-10-31T23:11:46.594Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:46.645Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:46.730 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:45fedfe5-b647-a7ed-8b2f-c137e2a34cfc Address:127.0.0.1:24307}]" -11:46.731 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-45fedfe5-b647-a7ed-8b2f-c137e2a34cfc.dc1 127.0.0.1 -11:46.732 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-45fedfe5-b647-a7ed-8b2f-c137e2a34cfc 127.0.0.1 -2021-10-31T23:11:46.732Z [INFO] agent.router: Initializing LAN area manager -11:46.732 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=udp -11:46.732 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24307 [Follower]" leader= -11:46.732 [INFO] TestAgent.server: Adding LAN server: server="Node-45fedfe5-b647-a7ed-8b2f-c137e2a34cfc (Addr: tcp/127.0.0.1:24307) (DC: dc1)" -11:46.732 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-45fedfe5-b647-a7ed-8b2f-c137e2a34cfc.dc1 area=wan -11:46.733 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=tcp -11:46.733 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24303 network=tcp -11:46.733 [INFO] TestAgent: started state syncer -11:46.733 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24308 network=tcp -11:46.789 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:46.789 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24307 [Candidate]" term=2 -11:46.789 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:46.789 [DEBUG] TestAgent.server.raft: vote granted: from=45fedfe5-b647-a7ed-8b2f-c137e2a34cfc term=2 tally=1 -11:46.789 [INFO] TestAgent.server.raft: election won: tally=1 -11:46.789 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24307 [Leader]" -11:46.809 [INFO] TestAgent.server: cluster leadership acquired -11:46.857 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24307 -11:46.857 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:46.858 [INFO] TestAgent.server: New leader elected: payload=Node-45fedfe5-b647-a7ed-8b2f-c137e2a34cfc -11:47.034 [INFO] TestAgent: Synced node info -11:47.080 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:47.080 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:47.080 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:47.080 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:47.080 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:47.080 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-45fedfe5-b647-a7ed-8b2f-c137e2a34cfc -11:47.080 [INFO] TestAgent.server: member joined, marking health alive: member=Node-45fedfe5-b647-a7ed-8b2f-c137e2a34cfc -11:47.081 [INFO] TestAgent.server: federation state anti-entropy synced -11:47.120 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:47.120 [INFO] TestAgent: Synced node info -11:47.120 [DEBUG] TestAgent: Check in sync: check=mysql -11:47.121 [INFO] TestAgent: Synced check: check=redis -11:47.121 [INFO] TestAgent: Synced check: check=web -11:47.138 [INFO] TestAgent: Synced check: check=cache -11:47.174 [INFO] TestAgent: Deregistered check: check=lb -11:47.204 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:47.205 [INFO] TestAgent: Synced node info -11:47.205 [DEBUG] TestAgent: Check in sync: check=mysql -11:47.216 [INFO] TestAgent: Deregistered check: check=redis -11:47.216 [DEBUG] TestAgent: Check in sync: check=web -11:47.216 [DEBUG] TestAgent: Check in sync: check=cache -11:47.216 [INFO] TestAgent: Requesting shutdown -11:47.216 [INFO] TestAgent.server: shutting down server -11:47.217 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:47.217 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:47.217 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:47.217 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:47.217 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:47.217 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:47.217 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:47.217 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:47.217 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:47.218Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:47.218Z [INFO] agent.router.manager: shutting down -11:47.226 [INFO] TestAgent: consul server down -11:47.226 [INFO] TestAgent: shutdown complete -11:47.226 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=tcp -11:47.226 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=udp -11:47.226 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24303 network=tcp -11:47.228 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:47.228 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:47.729 [INFO] TestAgent: Waiting for endpoints to shut down -11:47.729 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_Checks (1.29s) -2021-10-31T23:11:46.600Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:46.678Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:46.690 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:382dbf68-9881-5e51-ed7e-a953cbde953c Address:127.0.0.1:24300}]" -11:46.693 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= -11:46.749 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:46.750 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 -11:46.750 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=382dbf68-9881-5e51-ed7e-a953cbde953c fallback=127.0.0.1:24300 error="Could not find address for server id 382dbf68-9881-5e51-ed7e-a953cbde953c" -11:46.750 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:46.750 [DEBUG] TestAgent.server.raft: vote granted: from=382dbf68-9881-5e51-ed7e-a953cbde953c term=2 tally=1 -11:46.750 [INFO] TestAgent.server.raft: election won: tally=1 -11:46.750 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" -11:46.790 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-382dbf68-9881-5e51-ed7e-a953cbde953c.dc1 127.0.0.1 -11:46.790 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-382dbf68-9881-5e51-ed7e-a953cbde953c 127.0.0.1 -2021-10-31T23:11:46.790Z [INFO] agent.router: Initializing LAN area manager -11:46.791 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp -11:46.791 [INFO] TestAgent.server: Adding LAN server: server="Node-382dbf68-9881-5e51-ed7e-a953cbde953c (Addr: tcp/127.0.0.1:24300) (DC: dc1)" -11:46.791 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-382dbf68-9881-5e51-ed7e-a953cbde953c.dc1 area=wan -11:46.791 [INFO] TestAgent.server: cluster leadership acquired -11:46.791 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24300 -11:46.792 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp -11:46.793 [INFO] TestAgent.server: New leader elected: payload=Node-382dbf68-9881-5e51-ed7e-a953cbde953c -11:46.823 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:46.856 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp -11:46.856 [INFO] TestAgent: started state syncer -11:46.858 [INFO] TestAgent: Synced node info -11:46.858 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp -11:47.051 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:47.051 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:47.051 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:47.051 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:47.051 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:47.051 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-382dbf68-9881-5e51-ed7e-a953cbde953c -11:47.051 [INFO] TestAgent.server: member joined, marking health alive: member=Node-382dbf68-9881-5e51-ed7e-a953cbde953c -11:47.078 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-382dbf68-9881-5e51-ed7e-a953cbde953c -11:47.079 [INFO] TestAgent.server: federation state anti-entropy synced -11:47.205 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:47.205 [INFO] TestAgent: Synced node info -11:47.211 [INFO] TestAgent: Synced service: service=cache -11:47.212 [INFO] TestAgent: Deregistered service: service=lb -11:47.212 [DEBUG] TestAgent: Service in sync: service=mysql -11:47.212 [INFO] TestAgent: Synced service: service=redis -11:47.223 [INFO] TestAgent: Synced service: service=web -11:47.252 [INFO] TestAgent: Synced service: service=api -11:47.253 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:47.253 [INFO] TestAgent: Synced node info -11:47.253 [DEBUG] TestAgent: Service in sync: service=mysql -11:47.253 [DEBUG] TestAgent: Service in sync: service=redis -11:47.253 [DEBUG] TestAgent: Service in sync: service=web -11:47.254 [INFO] TestAgent: Deregistered service: service=api -11:47.254 [DEBUG] TestAgent: Service in sync: service=cache -11:47.254 [INFO] TestAgent: Requesting shutdown -11:47.255 [INFO] TestAgent.server: shutting down server -11:47.255 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:47.255 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:47.255 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:47.255 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:47.255 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:47.255 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:47.255 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:47.255 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:47.255 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:47.255 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:47.255 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:47.255Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:47.255Z [INFO] agent.router.manager: shutting down -11:47.255 [INFO] TestAgent: consul server down -11:47.255 [INFO] TestAgent: shutdown complete -11:47.255 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp -11:47.255 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp -11:47.255 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp -11:47.757 [INFO] TestAgent: Waiting for endpoints to shut down -11:47.757 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_Services (1.32s) -2021-10-31T23:11:46.671Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:46.771Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:46.772 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0cd83b23-a2b7-1aa8-f5f3-feaeb8275e3d Address:127.0.0.1:24335}]" -11:46.773 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0cd83b23-a2b7-1aa8-f5f3-feaeb8275e3d.dc1 127.0.0.1 -11:46.773 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0cd83b23-a2b7-1aa8-f5f3-feaeb8275e3d 127.0.0.1 -2021-10-31T23:11:46.773Z [INFO] agent.router: Initializing LAN area manager -11:46.774 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24330 network=udp -11:46.774 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24335 [Follower]" leader= -11:46.774 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0cd83b23-a2b7-1aa8-f5f3-feaeb8275e3d.dc1 area=wan -11:46.774 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24330 network=tcp -11:46.774 [INFO] TestAgent.server: Adding LAN server: server="Node-0cd83b23-a2b7-1aa8-f5f3-feaeb8275e3d (Addr: tcp/127.0.0.1:24335) (DC: dc1)" -11:46.774 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24331 network=tcp -11:46.774 [INFO] TestAgent: started state syncer -11:46.774 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24336 network=tcp -11:47.088 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:47.126 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24335 [Candidate]" term=2 -11:47.126 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:47.126 [DEBUG] TestAgent.server.raft: vote granted: from=0cd83b23-a2b7-1aa8-f5f3-feaeb8275e3d term=2 tally=1 -11:47.126 [INFO] TestAgent.server.raft: election won: tally=1 -11:47.126 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24335 [Leader]" -11:47.126 [INFO] TestAgent.server: cluster leadership acquired -11:47.127 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24335 -11:47.128 [INFO] TestAgent.server: New leader elected: payload=Node-0cd83b23-a2b7-1aa8-f5f3-feaeb8275e3d -11:47.128 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:47.174 [INFO] TestAgent: Synced node info -11:47.175 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:47.175 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:47.175 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:47.175 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:47.175 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:47.175 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0cd83b23-a2b7-1aa8-f5f3-feaeb8275e3d -11:47.175 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0cd83b23-a2b7-1aa8-f5f3-feaeb8275e3d -11:47.214 [INFO] TestAgent.server: federation state anti-entropy synced -11:47.251 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:47.251 [DEBUG] TestAgent: Node info in sync -11:47.254 [INFO] TestAgent: Synced check: check=web -11:47.254 [INFO] TestAgent: Requesting shutdown -11:47.254 [INFO] TestAgent.server: shutting down server -11:47.254 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:47.254 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:47.254 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:47.254 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:47.254 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:47.255 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:47.255 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:47.255 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:47.255 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:47.255 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:47.255 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:47.255Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:47.255Z [INFO] agent.router.manager: shutting down -11:47.255 [INFO] TestAgent: consul server down -11:47.255 [INFO] TestAgent: shutdown complete -11:47.255 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24330 network=tcp -11:47.255 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24330 network=udp -11:47.255 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24331 network=tcp -11:47.758 [INFO] TestAgent: Waiting for endpoints to shut down -11:47.758 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_UpdateCheck_DiscardOutput (1.32s) -2021-10-31T23:11:46.624Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:46.750Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:46.752 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d773e16a-f352-b94d-d120-2681aa517ab7 Address:127.0.0.1:24321}]" -11:46.794 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24321 [Follower]" leader= -11:46.862 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d773e16a-f352-b94d-d120-2681aa517ab7.dc1 127.0.0.1 -11:46.864 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d773e16a-f352-b94d-d120-2681aa517ab7 127.0.0.1 -2021-10-31T23:11:46.864Z [INFO] agent.router: Initializing LAN area manager -11:46.864 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24316 network=udp -11:46.869 [INFO] TestAgent.server: Adding LAN server: server="Node-d773e16a-f352-b94d-d120-2681aa517ab7 (Addr: tcp/127.0.0.1:24321) (DC: dc1)" -11:46.869 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d773e16a-f352-b94d-d120-2681aa517ab7.dc1 area=wan -11:46.953 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:46.953 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24321 [Candidate]" term=2 -11:46.954 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24316 network=tcp -11:46.955 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:46.956 [DEBUG] TestAgent.server.raft: vote granted: from=d773e16a-f352-b94d-d120-2681aa517ab7 term=2 tally=1 -11:46.956 [INFO] TestAgent.server.raft: election won: tally=1 -11:46.956 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24321 [Leader]" -11:46.955 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24317 network=tcp -11:46.957 [INFO] TestAgent: started state syncer -11:46.957 [INFO] TestAgent.server: cluster leadership acquired -11:46.957 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24321 -11:46.960 [INFO] TestAgent.server: New leader elected: payload=Node-d773e16a-f352-b94d-d120-2681aa517ab7 -11:46.960 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24322 network=tcp -11:46.986 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:47.034 [INFO] TestAgent: Synced node info -11:47.127 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:47.127 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:47.127 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:47.127 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:47.127 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:47.128 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d773e16a-f352-b94d-d120-2681aa517ab7 -11:47.128 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d773e16a-f352-b94d-d120-2681aa517ab7 -11:47.163 [INFO] TestAgent.server: federation state anti-entropy synced -11:47.289 [INFO] TestAgent: Requesting shutdown -11:47.290 [INFO] TestAgent.server: shutting down server -11:47.290 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:47.290 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:47.290 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:47.290 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:47.290 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:47.290 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:47.290 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:47.290 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:47.290 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:47.290 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:47.290 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:47.290Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:47.290Z [INFO] agent.router.manager: shutting down -11:47.290 [INFO] TestAgent: consul server down -11:47.290 [INFO] TestAgent: shutdown complete -11:47.290 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24316 network=tcp -11:47.290 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24316 network=udp -11:47.290 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24317 network=tcp -11:47.792 [INFO] TestAgent: Waiting for endpoints to shut down -11:47.792 [INFO] TestAgent: Endpoints down ---- PASS: TestAliasNotifications_local (1.36s) -2021-10-31T23:11:46.857Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:47.052Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:47.119 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b912114e-f8fc-b091-329e-1365ab3ef318 Address:127.0.0.1:24356}]" -11:47.120 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b912114e-f8fc-b091-329e-1365ab3ef318.dc1 127.0.0.1 -11:47.121 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b912114e-f8fc-b091-329e-1365ab3ef318 127.0.0.1 -2021-10-31T23:11:47.121Z [INFO] agent.router: Initializing LAN area manager -11:47.121 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24351 network=udp -11:47.121 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24356 [Follower]" leader= -11:47.126 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b912114e-f8fc-b091-329e-1365ab3ef318.dc1 area=wan -11:47.126 [INFO] TestAgent.server: Adding LAN server: server="Node-b912114e-f8fc-b091-329e-1365ab3ef318 (Addr: tcp/127.0.0.1:24356) (DC: dc1)" -11:47.128 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24351 network=tcp -11:47.129 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24352 network=tcp -11:47.129 [INFO] TestAgent: started state syncer -11:47.174 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24357 network=tcp -11:47.174 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:47.174 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24356 [Candidate]" term=2 -11:47.174 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:47.174 [DEBUG] TestAgent.server.raft: vote granted: from=b912114e-f8fc-b091-329e-1365ab3ef318 term=2 tally=1 -11:47.174 [INFO] TestAgent.server.raft: election won: tally=1 -11:47.174 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24356 [Leader]" -11:47.174 [INFO] TestAgent.server: cluster leadership acquired -11:47.204 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24356 -11:47.205 [INFO] TestAgent.server: New leader elected: payload=Node-b912114e-f8fc-b091-329e-1365ab3ef318 -11:47.237 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:47.239 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:47.239 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:47.239 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:47.239 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:47.239 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:47.239 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b912114e-f8fc-b091-329e-1365ab3ef318 -11:47.239 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b912114e-f8fc-b091-329e-1365ab3ef318 -11:47.250 [INFO] TestAgent.server: federation state anti-entropy synced -11:47.383 [INFO] TestAgent: Requesting shutdown -11:47.383 [INFO] TestAgent.server: shutting down server -11:47.383 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:47.383 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:47.383 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:47.383 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:47.383 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:47.383 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:47.383 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:47.383 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:47.383 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:47.383 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:47.383 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:47.383 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:47.383Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:47.383Z [INFO] agent.router.manager: shutting down -11:47.383 [INFO] TestAgent: consul server down -11:47.383 [INFO] TestAgent: shutdown complete -11:47.383 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24351 network=tcp -11:47.383 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24351 network=udp -11:47.383 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24352 network=tcp -11:47.890 [INFO] TestAgent: Waiting for endpoints to shut down -11:47.890 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_sendCoordinate (1.45s) -2021-10-31T23:11:46.790Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:46.790Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:47.204Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:47.204Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:47.251 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fd967a2a-e00f-6f14-7949-ef2aa2708f26 Address:127.0.0.1:24342}]" -11:47.252 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fd967a2a-e00f-6f14-7949-ef2aa2708f26.dc1 127.0.0.1 -11:47.253 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fd967a2a-e00f-6f14-7949-ef2aa2708f26 127.0.0.1 -2021-10-31T23:11:47.253Z [INFO] agent.router: Initializing LAN area manager -11:47.254 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24337 network=udp -11:47.254 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24337 network=tcp -11:47.254 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24342 [Follower]" leader= -11:47.254 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fd967a2a-e00f-6f14-7949-ef2aa2708f26.dc1 area=wan -11:47.255 [INFO] TestAgent.server: Adding LAN server: server="Node-fd967a2a-e00f-6f14-7949-ef2aa2708f26 (Addr: tcp/127.0.0.1:24342) (DC: dc1)" -11:47.255 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24338 network=tcp -11:47.255 [INFO] TestAgent: started state syncer -11:47.255 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24343 network=tcp -11:47.304 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -11:47.304 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:47.304 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24342 [Candidate]" term=2 -11:47.304 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:47.304 [DEBUG] TestAgent.server.raft: vote granted: from=fd967a2a-e00f-6f14-7949-ef2aa2708f26 term=2 tally=1 -11:47.304 [INFO] TestAgent.server.raft: election won: tally=1 -11:47.304 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24342 [Leader]" -11:47.305 [INFO] TestAgent.server: cluster leadership acquired -11:47.305 [INFO] TestAgent.server: initializing acls -11:47.305 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:47.305 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:47.305 [INFO] TestAgent.server: New leader elected: payload=Node-fd967a2a-e00f-6f14-7949-ef2aa2708f26 -11:47.305 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:47.306 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:47.306 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:47.306 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:47.306 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-fd967a2a-e00f-6f14-7949-ef2aa2708f26 -11:47.306 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-fd967a2a-e00f-6f14-7949-ef2aa2708f26.dc1 -11:47.307 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:47.308 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:47.308 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:47.308 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:47.308 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:47.308 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:47.308 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fd967a2a-e00f-6f14-7949-ef2aa2708f26 -11:47.308 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fd967a2a-e00f-6f14-7949-ef2aa2708f26 -11:47.309 [INFO] TestAgent.server: Updating LAN server: server="Node-fd967a2a-e00f-6f14-7949-ef2aa2708f26 (Addr: tcp/127.0.0.1:24342) (DC: dc1)" -11:47.309 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fd967a2a-e00f-6f14-7949-ef2aa2708f26 -11:47.309 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-fd967a2a-e00f-6f14-7949-ef2aa2708f26.dc1 area=wan -11:47.309 [INFO] TestAgent.server: federation state anti-entropy synced -11:47.420 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fd967a2a-e00f-6f14-7949-ef2aa2708f26 -11:47.421 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-fd967a2a-e00f-6f14-7949-ef2aa2708f26 -11:47.422 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -11:47.422 [WARN] TestAgent: Node info update blocked by ACLs: node=fd967a2a-e00f-6f14-7949-ef2aa2708f26 accessorID=00000000-0000-0000-0000-000000000002 -11:47.422 [INFO] TestAgent: Synced service: service=mysql -11:47.423 [INFO] TestAgent: Synced service: service=api -11:47.423 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -11:47.423 [WARN] TestAgent: Node info update blocked by ACLs: node=fd967a2a-e00f-6f14-7949-ef2aa2708f26 accessorID=00000000-0000-0000-0000-000000000002 -11:47.423 [INFO] TestAgent: Synced service: service=mysql -11:47.423 [INFO] TestAgent: Synced service: service=api -11:47.423 [WARN] TestAgent: Check registration blocked by ACLs: check=mysql-check accessorID= -11:47.424 [INFO] TestAgent: Synced check: check=api-check -11:47.424 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=api-check -11:47.424 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -11:47.424 [WARN] TestAgent: Node info update blocked by ACLs: node=fd967a2a-e00f-6f14-7949-ef2aa2708f26 accessorID=00000000-0000-0000-0000-000000000002 -11:47.424 [INFO] TestAgent: Synced service: service=mysql -11:47.424 [INFO] TestAgent: Synced service: service=api -11:47.425 [WARN] TestAgent: Check registration blocked by ACLs: check=mysql-check accessorID= -11:47.425 [INFO] TestAgent: Deregistered check: check=api-check -11:47.425 [INFO] TestAgent: Requesting shutdown -11:47.425 [INFO] TestAgent.server: shutting down server -11:47.425 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:47.425 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:47.425 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:47.425 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:47.425 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:47.425 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:47.425 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:47.425 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:47.425 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:47.425 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:47.425 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:47.425 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:47.425 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:47.425 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:47.425 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:47.426 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:47.426Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:47.426Z [INFO] agent.router.manager: shutting down -11:47.426 [INFO] TestAgent: consul server down -11:47.426 [INFO] TestAgent: shutdown complete -11:47.426 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24337 network=tcp -11:47.426 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24337 network=udp -11:47.426 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24338 network=tcp -11:47.929 [INFO] TestAgent: Waiting for endpoints to shut down -11:47.929 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_Checks_ACLDeny (1.49s) -2021-10-31T23:11:47.078Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:47.254Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:47.263 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8e62cf0f-b00d-99f7-0e54-ad886a5641ce Address:127.0.0.1:24377}]" -11:47.263 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24377 [Follower]" leader= -11:47.266 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8e62cf0f-b00d-99f7-0e54-ad886a5641ce.dc1 127.0.0.1 -11:47.267 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8e62cf0f-b00d-99f7-0e54-ad886a5641ce 127.0.0.1 -2021-10-31T23:11:47.267Z [INFO] agent.router: Initializing LAN area manager -11:47.271 [INFO] TestAgent.server: Adding LAN server: server="Node-8e62cf0f-b00d-99f7-0e54-ad886a5641ce (Addr: tcp/127.0.0.1:24377) (DC: dc1)" -11:47.271 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24372 network=udp -11:47.271 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8e62cf0f-b00d-99f7-0e54-ad886a5641ce.dc1 area=wan -11:47.271 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24372 network=tcp -11:47.273 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24373 network=tcp -11:47.274 [INFO] TestAgent: started state syncer -11:47.274 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24378 network=tcp -11:47.327 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:47.327 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24377 [Candidate]" term=2 -11:47.328 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:47.328 [DEBUG] TestAgent.server.raft: vote granted: from=8e62cf0f-b00d-99f7-0e54-ad886a5641ce term=2 tally=1 -11:47.328 [INFO] TestAgent.server.raft: election won: tally=1 -11:47.328 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24377 [Leader]" -11:47.328 [INFO] TestAgent.server: cluster leadership acquired -11:47.328 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24377 -11:47.329 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:47.329 [INFO] TestAgent.server: New leader elected: payload=Node-8e62cf0f-b00d-99f7-0e54-ad886a5641ce -11:47.331 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:47.331 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:47.331 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:47.331 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:47.331 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:47.331 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8e62cf0f-b00d-99f7-0e54-ad886a5641ce -11:47.331 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8e62cf0f-b00d-99f7-0e54-ad886a5641ce -11:47.331 [INFO] TestAgent.server: federation state anti-entropy synced -11:47.507 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:47.508 [INFO] TestAgent: Synced node info -11:47.508 [INFO] TestAgent: Synced service: service=mysql -11:47.508 [DEBUG] TestAgent: Check in sync: check=mysql -11:47.508 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:47.508 [DEBUG] TestAgent: Node info in sync -11:47.509 [INFO] TestAgent: Synced service: service=mysql -11:47.509 [INFO] TestAgent: Synced service: service=redis -11:47.509 [DEBUG] TestAgent: Check in sync: check=mysql -11:47.509 [DEBUG] TestAgent: Check in sync: check=redis:1 -11:47.509 [DEBUG] TestAgent: Check in sync: check=redis:2 -11:47.509 [INFO] TestAgent: Requesting shutdown -11:47.509 [INFO] TestAgent.server: shutting down server -11:47.509 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:47.509 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:47.509 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:47.509 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:47.509 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:47.509 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:47.509 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:47.509 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:47.509 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:47.510 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -11:47.510 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:47.510 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2021-10-31T23:11:47.510Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:47.510Z [INFO] agent.router.manager: shutting down -11:47.510 [INFO] TestAgent: consul server down -11:47.510 [INFO] TestAgent: shutdown complete -11:47.510 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24372 network=tcp -11:47.510 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24372 network=udp -11:47.510 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24373 network=tcp -11:48.010 [INFO] TestAgent: Waiting for endpoints to shut down -11:48.010 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_Services_WithChecks (1.53s) -2021-10-31T23:11:47.049Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:47.199Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:47.201 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9b5c9d0f-3baf-c9ac-3974-853637d93657 Address:127.0.0.1:24363}]" -11:47.201 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24363 [Follower]" leader= -11:47.201 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9b5c9d0f-3baf-c9ac-3974-853637d93657.dc1 127.0.0.1 -11:47.202 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9b5c9d0f-3baf-c9ac-3974-853637d93657 127.0.0.1 -2021-10-31T23:11:47.202Z [INFO] agent.router: Initializing LAN area manager -11:47.202 [INFO] TestAgent.server: Adding LAN server: server="Node-9b5c9d0f-3baf-c9ac-3974-853637d93657 (Addr: tcp/127.0.0.1:24363) (DC: dc1)" -11:47.202 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24358 network=udp -11:47.203 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9b5c9d0f-3baf-c9ac-3974-853637d93657.dc1 area=wan -11:47.203 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24358 network=tcp -11:47.203 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24359 network=tcp -11:47.203 [INFO] TestAgent: started state syncer -11:47.203 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24364 network=tcp -11:47.277 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:47.277 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24363 [Candidate]" term=2 -11:47.278 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:47.278 [DEBUG] TestAgent.server.raft: vote granted: from=9b5c9d0f-3baf-c9ac-3974-853637d93657 term=2 tally=1 -11:47.278 [INFO] TestAgent.server.raft: election won: tally=1 -11:47.278 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24363 [Leader]" -11:47.278 [INFO] TestAgent.server: cluster leadership acquired -11:47.278 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24363 -11:47.278 [INFO] TestAgent.server: New leader elected: payload=Node-9b5c9d0f-3baf-c9ac-3974-853637d93657 -11:47.279 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:47.280 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:47.280 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:47.280 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:47.280 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:47.280 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:47.280 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9b5c9d0f-3baf-c9ac-3974-853637d93657 -11:47.280 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9b5c9d0f-3baf-c9ac-3974-853637d93657 -11:47.281 [INFO] TestAgent.server: federation state anti-entropy synced -11:47.526 [INFO] TestAgent: Synced node info -11:47.527 [INFO] TestAgent: Synced service: service=svc_id1 -11:47.527 [INFO] TestAgent: Synced service: service=svc_id2 -11:47.528 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:47.528 [INFO] TestAgent: Synced node info -11:47.528 [INFO] TestAgent: Synced service: service=svc_id1 -11:47.528 [INFO] TestAgent: Synced service: service=svc_id2 -11:47.529 [INFO] TestAgent: Requesting shutdown -11:47.529 [INFO] TestAgent.server: shutting down server -11:47.529 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:47.529 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:47.529 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:47.529 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:47.529 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:47.529 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:47.529 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:47.529 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:47.529 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:47.529 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:47.529 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:47.529 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:47.529Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:47.529Z [INFO] agent.router.manager: shutting down -11:47.529 [INFO] TestAgent: consul server down -11:47.529 [INFO] TestAgent: shutdown complete -11:47.529 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24358 network=tcp -11:47.529 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24358 network=udp -11:47.529 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24359 network=tcp -11:48.031 [INFO] TestAgent: Waiting for endpoints to shut down -11:48.031 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_EnableTagOverride (1.59s) -2021-10-31T23:11:47.171Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:47.205Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:47.251 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:002c1274-aafc-4104-8270-c7967985cba0 Address:127.0.0.1:24391}]" -11:47.252 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-002c1274-aafc-4104-8270-c7967985cba0.dc1 127.0.0.1 -11:47.253 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-002c1274-aafc-4104-8270-c7967985cba0 127.0.0.1 -2021-10-31T23:11:47.253Z [INFO] agent.router: Initializing LAN area manager -11:47.253 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24386 network=udp -11:47.253 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24391 [Follower]" leader= -11:47.255 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24386 network=tcp -11:47.258 [INFO] TestAgent.server: Adding LAN server: server="Node-002c1274-aafc-4104-8270-c7967985cba0 (Addr: tcp/127.0.0.1:24391) (DC: dc1)" -11:47.267 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24387 network=tcp -11:47.267 [INFO] TestAgent: started state syncer -11:47.267 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24392 network=tcp -11:47.271 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-002c1274-aafc-4104-8270-c7967985cba0.dc1 area=wan -11:47.305 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:47.305 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24391 [Candidate]" term=2 -11:47.306 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:47.306 [DEBUG] TestAgent.server.raft: vote granted: from=002c1274-aafc-4104-8270-c7967985cba0 term=2 tally=1 -11:47.306 [INFO] TestAgent.server.raft: election won: tally=1 -11:47.306 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24391 [Leader]" -11:47.306 [INFO] TestAgent.server: cluster leadership acquired -11:47.306 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24391 -11:47.307 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:47.309 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:47.309 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:47.309 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:47.309 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:47.309 [INFO] TestAgent.server: New leader elected: payload=Node-002c1274-aafc-4104-8270-c7967985cba0 -11:47.309 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:47.309 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-002c1274-aafc-4104-8270-c7967985cba0 -11:47.309 [INFO] TestAgent.server: member joined, marking health alive: member=Node-002c1274-aafc-4104-8270-c7967985cba0 -11:47.310 [INFO] TestAgent.server: federation state anti-entropy synced -11:47.533 [INFO] TestAgent: Requesting shutdown -11:47.533 [INFO] TestAgent.server: shutting down server -11:47.533 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:47.533 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:47.533 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:47.533 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:47.533 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:47.533 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:47.533 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:47.533 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:47.533 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:47.533 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:47.533 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:47.533 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:47.533Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:47.533Z [INFO] agent.router.manager: shutting down -11:47.533 [INFO] TestAgent: consul server down -11:47.533 [INFO] TestAgent: shutdown complete -11:47.533 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24386 network=tcp -11:47.533 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24386 network=udp -11:47.533 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24387 network=tcp -11:48.033 [INFO] TestAgent: Waiting for endpoints to shut down -11:48.033 [INFO] TestAgent: Endpoints down ---- PASS: TestAgent_ServiceWatchCh (1.53s) -2021-10-31T23:11:46.836Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:46.836Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:47.266Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. -2021-10-31T23:11:47.266Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:47.274 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:57cd1e43-1cee-d30b-3860-87434169c744 Address:127.0.0.1:24370}]" -11:47.274 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24370 [Follower]" leader= -11:47.275 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-57cd1e43-1cee-d30b-3860-87434169c744.dc1 127.0.0.1 -11:47.276 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-57cd1e43-1cee-d30b-3860-87434169c744 127.0.0.1 -2021-10-31T23:11:47.276Z [INFO] agent.router: Initializing LAN area manager -11:47.277 [INFO] TestAgent.server: Adding LAN server: server="Node-57cd1e43-1cee-d30b-3860-87434169c744 (Addr: tcp/127.0.0.1:24370) (DC: dc1)" -11:47.277 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-57cd1e43-1cee-d30b-3860-87434169c744.dc1 area=wan -11:47.277 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24365 network=tcp -11:47.277 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24365 network=udp -11:47.278 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24366 network=tcp -11:47.278 [INFO] TestAgent: started state syncer -11:47.278 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24371 network=tcp -11:47.322 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:47.322 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24370 [Candidate]" term=2 -11:47.322 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:47.322 [DEBUG] TestAgent.server.raft: vote granted: from=57cd1e43-1cee-d30b-3860-87434169c744 term=2 tally=1 -11:47.323 [INFO] TestAgent.server.raft: election won: tally=1 -11:47.323 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24370 [Leader]" -11:47.323 [INFO] TestAgent.server: cluster leadership acquired -11:47.323 [INFO] TestAgent.server: initializing acls -11:47.323 [INFO] TestAgent.server: New leader elected: payload=Node-57cd1e43-1cee-d30b-3860-87434169c744 -11:47.323 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:47.323 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:47.323 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:47.324 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:47.324 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:47.324 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:47.324 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-57cd1e43-1cee-d30b-3860-87434169c744 -11:47.324 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-57cd1e43-1cee-d30b-3860-87434169c744.dc1 -11:47.325 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:47.325 [INFO] TestAgent.server: Updating LAN server: server="Node-57cd1e43-1cee-d30b-3860-87434169c744 (Addr: tcp/127.0.0.1:24370) (DC: dc1)" -11:47.325 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-57cd1e43-1cee-d30b-3860-87434169c744.dc1 area=wan -11:47.327 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:47.327 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:47.327 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:47.327 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:47.327 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:47.327 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-57cd1e43-1cee-d30b-3860-87434169c744 -11:47.327 [INFO] TestAgent.server: member joined, marking health alive: member=Node-57cd1e43-1cee-d30b-3860-87434169c744 -11:47.327 [INFO] TestAgent.server: federation state anti-entropy synced -11:47.328 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-57cd1e43-1cee-d30b-3860-87434169c744 -11:47.330 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -11:47.330 [WARN] TestAgent: Node info update blocked by ACLs: node=57cd1e43-1cee-d30b-3860-87434169c744 accessorID=00000000-0000-0000-0000-000000000002 -11:47.563 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-57cd1e43-1cee-d30b-3860-87434169c744 -11:47.563 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-57cd1e43-1cee-d30b-3860-87434169c744 -11:47.564 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -11:47.564 [WARN] TestAgent: Node info update blocked by ACLs: node=57cd1e43-1cee-d30b-3860-87434169c744 accessorID=00000000-0000-0000-0000-000000000002 -11:47.565 [WARN] TestAgent: Service registration blocked by ACLs: service=mysql accessorID= -11:47.565 [INFO] TestAgent: Synced service: service=api -11:47.565 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth -11:47.565 [WARN] TestAgent: Node info update blocked by ACLs: node=57cd1e43-1cee-d30b-3860-87434169c744 accessorID=00000000-0000-0000-0000-000000000002 -11:47.565 [WARN] TestAgent: Service registration blocked by ACLs: service=mysql accessorID= -11:47.566 [INFO] TestAgent: Deregistered service: service=api -11:47.566 [INFO] TestAgent: Requesting shutdown -11:47.566 [INFO] TestAgent.server: shutting down server -11:47.566 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:47.566 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:47.566 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:47.566 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:47.566 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:47.566 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:47.566 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:47.566 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:47.566 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:47.566 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:47.566 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:47.566 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:47.566 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:47.566 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:47.566 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:47.566Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:47.566Z [INFO] agent.router.manager: shutting down -11:47.566 [INFO] TestAgent: consul server down -11:47.566 [INFO] TestAgent: shutdown complete -11:47.566 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24365 network=tcp -11:47.567 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24365 network=udp -11:47.567 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24366 network=tcp -11:48.068 [INFO] TestAgent: Waiting for endpoints to shut down -11:48.068 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_Services_ACLDeny (1.59s) -2021-10-31T23:11:47.124Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:47.254Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:47.267 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2389b91f-4c3e-c49d-3b62-63d8dbf94ed5 Address:127.0.0.1:24384}]" -11:47.272 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24384 [Follower]" leader= -11:47.274 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2389b91f-4c3e-c49d-3b62-63d8dbf94ed5.dc1 127.0.0.1 -11:47.275 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2389b91f-4c3e-c49d-3b62-63d8dbf94ed5 127.0.0.1 -2021-10-31T23:11:47.275Z [INFO] agent.router: Initializing LAN area manager -11:47.276 [INFO] TestAgent.server: Adding LAN server: server="Node-2389b91f-4c3e-c49d-3b62-63d8dbf94ed5 (Addr: tcp/127.0.0.1:24384) (DC: dc1)" -11:47.276 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2389b91f-4c3e-c49d-3b62-63d8dbf94ed5.dc1 area=wan -11:47.276 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24379 network=udp -11:47.276 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24379 network=tcp -11:47.279 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24380 network=tcp -11:47.279 [INFO] TestAgent: started state syncer -11:47.279 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24385 network=tcp -11:47.316 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:47.316 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24384 [Candidate]" term=2 -11:47.316 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:47.316 [DEBUG] TestAgent.server.raft: vote granted: from=2389b91f-4c3e-c49d-3b62-63d8dbf94ed5 term=2 tally=1 -11:47.316 [INFO] TestAgent.server.raft: election won: tally=1 -11:47.316 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24384 [Leader]" -11:47.316 [INFO] TestAgent.server: cluster leadership acquired -11:47.316 [INFO] TestAgent.server: New leader elected: payload=Node-2389b91f-4c3e-c49d-3b62-63d8dbf94ed5 -11:47.316 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24384 -11:47.317 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:47.319 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:47.319 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:47.319 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:47.319 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:47.319 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:47.319 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2389b91f-4c3e-c49d-3b62-63d8dbf94ed5 -11:47.319 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2389b91f-4c3e-c49d-3b62-63d8dbf94ed5 -11:47.320 [INFO] TestAgent.server: federation state anti-entropy synced -11:47.699 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:47.699 [INFO] TestAgent: Synced node info -11:47.699 [DEBUG] TestAgent: Service in sync: service=mysql-proxy -11:47.699 [INFO] TestAgent: Synced service: service=redis-proxy -11:47.700 [INFO] TestAgent: Synced service: service=web-proxy -11:47.700 [INFO] TestAgent: Synced service: service=cache-proxy -11:47.700 [INFO] TestAgent: Deregistered service: service=lb-proxy -11:47.700 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:47.700 [DEBUG] TestAgent: Node info in sync -11:47.700 [INFO] TestAgent: Deregistered service: service=cache-proxy -11:47.701 [DEBUG] TestAgent: Service in sync: service=mysql-proxy -11:47.701 [DEBUG] TestAgent: Service in sync: service=redis-proxy -11:47.701 [DEBUG] TestAgent: Service in sync: service=web-proxy -11:47.701 [INFO] TestAgent: Requesting shutdown -11:47.701 [INFO] TestAgent.server: shutting down server -11:47.701 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:47.701 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:47.701 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:47.701 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:47.701 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:47.701 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:47.701 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:47.701 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -11:47.701 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:47.701 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:47.701 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:47.701 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:47.701Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:47.702Z [INFO] agent.router.manager: shutting down -11:47.702 [INFO] TestAgent: consul server down -11:47.702 [INFO] TestAgent: shutdown complete -11:47.702 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24379 network=tcp -11:47.702 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24379 network=udp -11:47.702 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24380 network=tcp -11:48.205 [INFO] TestAgent: Waiting for endpoints to shut down -11:48.205 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_Services_ConnectProxy (1.72s) -2021-10-31T23:11:46.678Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:46.994Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:47.084 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:be6abbed-0bae-fd7f-c828-08cb073cb3da Address:127.0.0.1:24328}]" -11:47.084 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-be6abbed-0bae-fd7f-c828-08cb073cb3da.dc1 127.0.0.1 -11:47.085 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-be6abbed-0bae-fd7f-c828-08cb073cb3da 127.0.0.1 -2021-10-31T23:11:47.085Z [INFO] agent.router: Initializing LAN area manager -11:47.085 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24323 network=udp -11:47.087 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24328 [Follower]" leader= -11:47.087 [INFO] TestAgent.server: Adding LAN server: server="Node-be6abbed-0bae-fd7f-c828-08cb073cb3da (Addr: tcp/127.0.0.1:24328) (DC: dc1)" -11:47.087 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-be6abbed-0bae-fd7f-c828-08cb073cb3da.dc1 area=wan -11:47.087 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24323 network=tcp -11:47.088 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24324 network=tcp -11:47.088 [INFO] TestAgent: started state syncer -11:47.088 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24329 network=tcp -11:47.142 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:47.176 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24328 [Candidate]" term=2 -11:47.227 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:47.227 [DEBUG] TestAgent.server.raft: vote granted: from=be6abbed-0bae-fd7f-c828-08cb073cb3da term=2 tally=1 -11:47.227 [INFO] TestAgent.server.raft: election won: tally=1 -11:47.227 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24328 [Leader]" -11:47.227 [INFO] TestAgent.server: cluster leadership acquired -11:47.227 [INFO] TestAgent.server: New leader elected: payload=Node-be6abbed-0bae-fd7f-c828-08cb073cb3da -11:47.234 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24328 -11:47.239 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:47.253 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:47.253 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:47.253 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:47.253 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:47.253 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:47.253 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-be6abbed-0bae-fd7f-c828-08cb073cb3da -11:47.253 [INFO] TestAgent.server: member joined, marking health alive: member=Node-be6abbed-0bae-fd7f-c828-08cb073cb3da -11:47.257 [INFO] TestAgent.server: federation state anti-entropy synced -11:47.437 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:47.437 [INFO] TestAgent: Synced node info -11:47.437 [INFO] TestAgent: Synced check: check=web -11:47.569 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:47.569 [DEBUG] TestAgent: Node info in sync -11:47.569 [DEBUG] TestAgent: Check in sync: check=web -11:47.569 [DEBUG] TestAgent: Node info in sync -11:47.569 [DEBUG] TestAgent: Check in sync: check=web -11:49.994 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:49.994 [DEBUG] TestAgent: Node info in sync -11:49.994 [INFO] TestAgent: Synced check: check=web -11:49.994 [DEBUG] TestAgent: Node info in sync -11:49.994 [DEBUG] TestAgent: Check in sync: check=web -11:49.998 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:49.998 [INFO] TestAgent: Synced node info -11:49.999 [INFO] TestAgent: Synced check: check=web -11:49.999 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -11:49.999 [INFO] TestAgent: Synced node info -11:49.999 [DEBUG] TestAgent: Check in sync: check=web -11:50.680 [DEBUG] TestAgent: Node info in sync -11:50.680 [INFO] TestAgent: Synced check: check=web -11:50.688 [INFO] TestAgent: Requesting shutdown -11:50.688 [INFO] TestAgent.server: shutting down server -11:50.688 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:50.688 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:50.688 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:50.688 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:50.688 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:50.689 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:50.689 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:50.689 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:50.689 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:50.689 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:50.689 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:50.689Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:50.689Z [INFO] agent.router.manager: shutting down -11:50.689 [INFO] TestAgent: consul server down -11:50.689 [INFO] TestAgent: shutdown complete -11:50.689 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24323 network=tcp -11:50.689 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24323 network=udp -11:50.689 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24324 network=tcp -11:51.189 [INFO] TestAgent: Waiting for endpoints to shut down -11:51.189 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentAntiEntropy_Check_DeferSync (4.75s) +2022-12-04T05:47:55.335Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:55.369Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:55.383 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c28d63e4-2958-e4c2-f698-a78301b4fbce Address:127.0.0.1:22258}]" +47:55.383 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22258 [Follower]" leader= +47:55.422 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c28d63e4-2958-e4c2-f698-a78301b4fbce.dc1 127.0.0.1 +47:55.423 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c28d63e4-2958-e4c2-f698-a78301b4fbce 127.0.0.1 +2022-12-04T05:47:55.423Z [INFO] agent.router: Initializing LAN area manager +47:55.423 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=udp +47:55.439 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c28d63e4-2958-e4c2-f698-a78301b4fbce.dc1 area=wan +47:55.439 [INFO] TestAgent.server: Adding LAN server: server="Node-c28d63e4-2958-e4c2-f698-a78301b4fbce (Addr: tcp/127.0.0.1:22258) (DC: dc1)" +47:55.440 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=tcp +47:55.441 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22254 network=tcp +47:55.441 [INFO] TestAgent: started state syncer +47:55.441 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22259 network=tcp +47:55.452 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:55.452 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22258 [Candidate]" term=2 +47:55.453 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:55.453 [DEBUG] TestAgent.server.raft: vote granted: from=c28d63e4-2958-e4c2-f698-a78301b4fbce term=2 tally=1 +47:55.453 [INFO] TestAgent.server.raft: election won: tally=1 +47:55.453 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22258 [Leader]" +47:55.453 [INFO] TestAgent.server: cluster leadership acquired +47:55.453 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22258 +47:55.453 [INFO] TestAgent.server: New leader elected: payload=Node-c28d63e4-2958-e4c2-f698-a78301b4fbce +47:55.480 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:55.502 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:55.502 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:55.502 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:55.502 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:55.502 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:55.502 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c28d63e4-2958-e4c2-f698-a78301b4fbce +47:55.502 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c28d63e4-2958-e4c2-f698-a78301b4fbce +47:55.553 [INFO] TestAgent.server: federation state anti-entropy synced +47:55.607 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.607 [INFO] TestAgent: Synced node info +47:55.607 [INFO] TestAgent: Synced service: service=cache +47:55.607 [INFO] TestAgent: Deregistered service: service=lb +47:55.607 [DEBUG] TestAgent: Service in sync: service=mysql +47:55.608 [INFO] TestAgent: Synced service: service=redis +47:55.608 [INFO] TestAgent: Synced service: service=web +47:55.608 [INFO] TestAgent: Synced service: service=api +47:55.608 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.609 [INFO] TestAgent: Synced node info +47:55.609 [DEBUG] TestAgent: Service in sync: service=redis +47:55.609 [DEBUG] TestAgent: Service in sync: service=web +47:55.609 [INFO] TestAgent: Deregistered service: service=api +47:55.609 [DEBUG] TestAgent: Service in sync: service=cache +47:55.609 [DEBUG] TestAgent: Service in sync: service=mysql +47:55.609 [INFO] TestAgent: Requesting shutdown +47:55.609 [INFO] TestAgent.server: shutting down server +47:55.609 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:55.609 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:55.609 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:55.609 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:55.609 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:55.609 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:55.609 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:55.609 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +47:55.609 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:55.609 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:55.609 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:55.609 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:55.609Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:55.609Z [INFO] agent.router.manager: shutting down +47:55.609 [INFO] TestAgent: consul server down +47:55.609 [INFO] TestAgent: shutdown complete +47:55.610 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=tcp +47:55.610 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=udp +47:55.610 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22254 network=tcp +47:56.112 [INFO] TestAgent: Waiting for endpoints to shut down +47:56.112 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_Services (0.85s) +2022-12-04T05:47:55.363Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:55.475Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:55.489 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:81a334f4-829e-703b-dfc6-6559ba4805bf Address:127.0.0.1:22272}]" +47:55.489 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22272 [Follower]" leader= +47:55.498 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-81a334f4-829e-703b-dfc6-6559ba4805bf.dc1 127.0.0.1 +47:55.555 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:55.556 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22272 [Candidate]" term=2 +47:55.557 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=81a334f4-829e-703b-dfc6-6559ba4805bf fallback=127.0.0.1:22272 error="Could not find address for server id 81a334f4-829e-703b-dfc6-6559ba4805bf" +47:55.560 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-81a334f4-829e-703b-dfc6-6559ba4805bf 127.0.0.1 +2022-12-04T05:47:55.561Z [INFO] agent.router: Initializing LAN area manager +47:55.572 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:55.572 [DEBUG] TestAgent.server.raft: vote granted: from=81a334f4-829e-703b-dfc6-6559ba4805bf term=2 tally=1 +47:55.572 [INFO] TestAgent.server.raft: election won: tally=1 +47:55.572 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22272 [Leader]" +47:55.578 [INFO] TestAgent.server: cluster leadership acquired +47:55.580 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22267 network=udp +47:55.580 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22267 network=tcp +47:55.581 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22268 network=tcp +47:55.581 [INFO] TestAgent: started state syncer +47:55.582 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22273 network=tcp +47:55.583 [INFO] TestAgent: Synced node info +47:55.583 [DEBUG] TestAgent: Node info in sync +47:55.585 [INFO] TestAgent.server: Adding LAN server: server="Node-81a334f4-829e-703b-dfc6-6559ba4805bf (Addr: tcp/127.0.0.1:22272) (DC: dc1)" +47:55.585 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-81a334f4-829e-703b-dfc6-6559ba4805bf.dc1 area=wan +47:55.586 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22272 +47:55.586 [INFO] TestAgent.server: New leader elected: payload=Node-81a334f4-829e-703b-dfc6-6559ba4805bf +47:55.587 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:55.590 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:55.590 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:55.590 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:55.590 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:55.590 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:55.590 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-81a334f4-829e-703b-dfc6-6559ba4805bf +47:55.590 [INFO] TestAgent.server: member joined, marking health alive: member=Node-81a334f4-829e-703b-dfc6-6559ba4805bf +47:55.591 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-81a334f4-829e-703b-dfc6-6559ba4805bf +47:55.605 [INFO] TestAgent.server: federation state anti-entropy synced +47:55.607 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.608 [INFO] TestAgent: Synced node info +47:55.608 [DEBUG] TestAgent: Check in sync: check=mysql +47:55.608 [INFO] TestAgent: Synced check: check=redis +47:55.609 [INFO] TestAgent: Synced check: check=web +47:55.609 [INFO] TestAgent: Synced check: check=cache +47:55.609 [INFO] TestAgent: Deregistered check: check=lb +47:55.609 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.610 [INFO] TestAgent: Synced node info +47:55.610 [DEBUG] TestAgent: Check in sync: check=mysql +47:55.610 [INFO] TestAgent: Deregistered check: check=redis +47:55.610 [DEBUG] TestAgent: Check in sync: check=web +47:55.610 [DEBUG] TestAgent: Check in sync: check=cache +47:55.610 [INFO] TestAgent: Requesting shutdown +47:55.610 [INFO] TestAgent.server: shutting down server +47:55.610 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:55.610 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:55.610 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:55.610 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:55.610 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:55.610 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:55.610 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:55.610 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:55.610 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:55.610 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:55.610 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:55.610Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:55.610Z [INFO] agent.router.manager: shutting down +47:55.611 [INFO] TestAgent: consul server down +47:55.611 [INFO] TestAgent: shutdown complete +47:55.611 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22267 network=tcp +47:55.611 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22267 network=udp +47:55.611 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22268 network=tcp +47:56.112 [INFO] TestAgent: Waiting for endpoints to shut down +47:56.112 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_Checks (0.85s) +2022-12-04T05:47:55.441Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:55.473Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:55.475 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3141dc53-a819-aa80-bc32-740cf83de094 Address:127.0.0.1:22293}]" +47:55.476 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3141dc53-a819-aa80-bc32-740cf83de094.dc1 127.0.0.1 +47:55.476 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22293 [Follower]" leader= +47:55.477 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3141dc53-a819-aa80-bc32-740cf83de094 127.0.0.1 +2022-12-04T05:47:55.477Z [INFO] agent.router: Initializing LAN area manager +47:55.479 [INFO] TestAgent.server: Adding LAN server: server="Node-3141dc53-a819-aa80-bc32-740cf83de094 (Addr: tcp/127.0.0.1:22293) (DC: dc1)" +47:55.480 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3141dc53-a819-aa80-bc32-740cf83de094.dc1 area=wan +47:55.480 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22288 network=udp +47:55.480 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22288 network=tcp +47:55.487 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22289 network=tcp +47:55.487 [INFO] TestAgent: started state syncer +47:55.487 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22294 network=tcp +47:55.552 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:55.552 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22293 [Candidate]" term=2 +47:55.553 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:55.553 [DEBUG] TestAgent.server.raft: vote granted: from=3141dc53-a819-aa80-bc32-740cf83de094 term=2 tally=1 +47:55.553 [INFO] TestAgent.server.raft: election won: tally=1 +47:55.553 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22293 [Leader]" +47:55.553 [INFO] TestAgent.server: cluster leadership acquired +47:55.553 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22293 +47:55.555 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:55.569 [INFO] TestAgent.server: New leader elected: payload=Node-3141dc53-a819-aa80-bc32-740cf83de094 +47:55.570 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:55.570 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:55.570 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:55.570 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:55.570 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:55.571 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3141dc53-a819-aa80-bc32-740cf83de094 +47:55.571 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3141dc53-a819-aa80-bc32-740cf83de094 +47:55.572 [INFO] TestAgent.server: federation state anti-entropy synced +47:55.619 [INFO] TestAgent: Synced node info +47:55.619 [INFO] TestAgent: Synced service: service=svc_id1 +47:55.619 [INFO] TestAgent: Synced service: service=svc_id2 +47:55.621 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.621 [INFO] TestAgent: Synced node info +47:55.621 [INFO] TestAgent: Synced service: service=svc_id1 +47:55.621 [INFO] TestAgent: Synced service: service=svc_id2 +47:55.621 [INFO] TestAgent: Requesting shutdown +47:55.622 [INFO] TestAgent.server: shutting down server +47:55.622 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:55.622 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:55.622 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:55.622 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:55.622 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:55.622 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:55.622 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:55.622 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +47:55.622 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:55.622 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:55.622 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:55.622 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:55.622Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:55.622Z [INFO] agent.router.manager: shutting down +47:55.622 [INFO] TestAgent: consul server down +47:55.622 [INFO] TestAgent: shutdown complete +47:55.622 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22288 network=tcp +47:55.622 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22288 network=udp +47:55.622 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22289 network=tcp +47:56.124 [INFO] TestAgent: Waiting for endpoints to shut down +47:56.124 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_EnableTagOverride (0.86s) +2022-12-04T05:47:55.517Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:55.577Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:55.585 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f94e2663-6cd4-56d7-1d90-8fbb8d99ff92 Address:127.0.0.1:22314}]" +47:55.586 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f94e2663-6cd4-56d7-1d90-8fbb8d99ff92.dc1 127.0.0.1 +47:55.587 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f94e2663-6cd4-56d7-1d90-8fbb8d99ff92 127.0.0.1 +2022-12-04T05:47:55.587Z [INFO] agent.router: Initializing LAN area manager +47:55.587 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22309 network=udp +47:55.588 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22314 [Follower]" leader= +47:55.601 [INFO] TestAgent.server: Adding LAN server: server="Node-f94e2663-6cd4-56d7-1d90-8fbb8d99ff92 (Addr: tcp/127.0.0.1:22314) (DC: dc1)" +47:55.601 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22309 network=tcp +47:55.602 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f94e2663-6cd4-56d7-1d90-8fbb8d99ff92.dc1 area=wan +47:55.604 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22310 network=tcp +47:55.630 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:55.630 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22314 [Candidate]" term=2 +47:55.630 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:55.630 [DEBUG] TestAgent.server.raft: vote granted: from=f94e2663-6cd4-56d7-1d90-8fbb8d99ff92 term=2 tally=1 +47:55.630 [INFO] TestAgent.server.raft: election won: tally=1 +47:55.630 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22314 [Leader]" +47:55.630 [INFO] TestAgent.server: cluster leadership acquired +47:55.630 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22314 +47:55.630 [INFO] TestAgent.server: New leader elected: payload=Node-f94e2663-6cd4-56d7-1d90-8fbb8d99ff92 +47:55.631 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:55.632 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:55.632 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:55.632 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:55.632 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:55.632 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:55.632 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f94e2663-6cd4-56d7-1d90-8fbb8d99ff92 +47:55.632 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f94e2663-6cd4-56d7-1d90-8fbb8d99ff92 +47:55.632 [INFO] TestAgent.server: federation state anti-entropy synced +47:55.637 [INFO] TestAgent: started state syncer +47:55.637 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22315 network=tcp +47:55.637 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.638 [INFO] TestAgent: Synced node info +47:55.638 [INFO] TestAgent: Synced service: service=mysql +47:55.638 [DEBUG] TestAgent: Check in sync: check=mysql +47:55.638 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.638 [INFO] TestAgent: Synced node info +47:55.639 [INFO] TestAgent: Synced service: service=mysql +47:55.639 [DEBUG] TestAgent: Check in sync: check=mysql +47:55.639 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.639 [DEBUG] TestAgent: Node info in sync +47:55.639 [INFO] TestAgent: Synced service: service=mysql +47:55.645 [INFO] TestAgent: Synced service: service=redis +47:55.645 [DEBUG] TestAgent: Check in sync: check=mysql +47:55.645 [DEBUG] TestAgent: Check in sync: check=redis:1 +47:55.645 [DEBUG] TestAgent: Check in sync: check=redis:2 +47:55.645 [INFO] TestAgent: Requesting shutdown +47:55.645 [INFO] TestAgent.server: shutting down server +47:55.645 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:55.645 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:55.645 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:55.645 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:55.645 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:55.646 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:55.646 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:55.646 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:55.646 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:55.646 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:55.646 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:55.646Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:55.646Z [INFO] agent.router.manager: shutting down +47:55.646 [INFO] TestAgent: consul server down +47:55.646 [INFO] TestAgent: shutdown complete +47:55.646 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22309 network=tcp +47:55.646 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22309 network=udp +47:55.646 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22310 network=tcp +47:56.146 [INFO] TestAgent: Waiting for endpoints to shut down +47:56.146 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_Services_WithChecks (0.89s) +2022-12-04T05:47:55.455Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:55.487Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:55.519 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7325da48-4a8a-f1ee-0309-9447fa8fc3dc Address:127.0.0.1:22349}]" +47:55.520 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22349 [Follower]" leader= +47:55.553 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7325da48-4a8a-f1ee-0309-9447fa8fc3dc.dc1 127.0.0.1 +47:55.554 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7325da48-4a8a-f1ee-0309-9447fa8fc3dc 127.0.0.1 +2022-12-04T05:47:55.554Z [INFO] agent.router: Initializing LAN area manager +47:55.556 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22344 network=udp +47:55.569 [INFO] TestAgent.server: Adding LAN server: server="Node-7325da48-4a8a-f1ee-0309-9447fa8fc3dc (Addr: tcp/127.0.0.1:22349) (DC: dc1)" +47:55.569 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7325da48-4a8a-f1ee-0309-9447fa8fc3dc.dc1 area=wan +47:55.578 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:55.578 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22349 [Candidate]" term=2 +47:55.581 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22344 network=tcp +47:55.582 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22345 network=tcp +47:55.582 [INFO] TestAgent: started state syncer +47:55.583 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22350 network=tcp +47:55.585 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:55.585 [DEBUG] TestAgent.server.raft: vote granted: from=7325da48-4a8a-f1ee-0309-9447fa8fc3dc term=2 tally=1 +47:55.585 [INFO] TestAgent.server.raft: election won: tally=1 +47:55.586 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22349 [Leader]" +47:55.586 [INFO] TestAgent.server: cluster leadership acquired +47:55.587 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22349 +47:55.587 [INFO] TestAgent.server: New leader elected: payload=Node-7325da48-4a8a-f1ee-0309-9447fa8fc3dc +47:55.588 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:55.604 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:55.604 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:55.604 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:55.604 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:55.604 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:55.604 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7325da48-4a8a-f1ee-0309-9447fa8fc3dc +47:55.604 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7325da48-4a8a-f1ee-0309-9447fa8fc3dc +47:55.605 [INFO] TestAgent.server: federation state anti-entropy synced +47:55.720 [INFO] TestAgent: Requesting shutdown +47:55.720 [INFO] TestAgent.server: shutting down server +47:55.720 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:55.721 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:55.721 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:55.721 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:55.721 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:55.721 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +47:55.721 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:55.721 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:55.721 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:55.721 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:55.721 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:55.721 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:55.721Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:55.721Z [INFO] agent.router.manager: shutting down +47:55.721 [INFO] TestAgent: consul server down +47:55.721 [INFO] TestAgent: shutdown complete +47:55.721 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22344 network=tcp +47:55.721 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22344 network=udp +47:55.721 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22345 network=tcp +47:56.224 [INFO] TestAgent: Waiting for endpoints to shut down +47:56.224 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_sendCoordinate (0.92s) +2022-12-04T05:47:55.461Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:55.576Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:55.577 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1f0beda3-c35f-4657-5b0d-41307e04c576 Address:127.0.0.1:22300}]" +47:55.578 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22300 [Follower]" leader= +47:55.584 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1f0beda3-c35f-4657-5b0d-41307e04c576.dc1 127.0.0.1 +47:55.585 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1f0beda3-c35f-4657-5b0d-41307e04c576 127.0.0.1 +2022-12-04T05:47:55.585Z [INFO] agent.router: Initializing LAN area manager +47:55.586 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22295 network=udp +47:55.586 [INFO] TestAgent.server: Adding LAN server: server="Node-1f0beda3-c35f-4657-5b0d-41307e04c576 (Addr: tcp/127.0.0.1:22300) (DC: dc1)" +47:55.586 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1f0beda3-c35f-4657-5b0d-41307e04c576.dc1 area=wan +47:55.586 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22295 network=tcp +47:55.587 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22296 network=tcp +47:55.587 [INFO] TestAgent: started state syncer +47:55.588 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22301 network=tcp +47:55.619 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:55.619 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22300 [Candidate]" term=2 +47:55.619 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:55.619 [DEBUG] TestAgent.server.raft: vote granted: from=1f0beda3-c35f-4657-5b0d-41307e04c576 term=2 tally=1 +47:55.619 [INFO] TestAgent.server.raft: election won: tally=1 +47:55.619 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22300 [Leader]" +47:55.619 [INFO] TestAgent.server: cluster leadership acquired +47:55.619 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22300 +47:55.620 [INFO] TestAgent.server: New leader elected: payload=Node-1f0beda3-c35f-4657-5b0d-41307e04c576 +47:55.620 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:55.622 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:55.622 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:55.622 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:55.622 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:55.622 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:55.622 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1f0beda3-c35f-4657-5b0d-41307e04c576 +47:55.622 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1f0beda3-c35f-4657-5b0d-41307e04c576 +47:55.622 [INFO] TestAgent.server: federation state anti-entropy synced +47:55.724 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.724 [INFO] TestAgent: Synced node info +47:55.739 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.739 [DEBUG] TestAgent: Node info in sync +47:55.739 [INFO] TestAgent: Synced check: check=web +47:55.739 [INFO] TestAgent: Requesting shutdown +47:55.739 [INFO] TestAgent.server: shutting down server +47:55.739 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:55.739 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:55.739 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:55.739 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:55.739 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:55.739 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:55.739 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:55.739 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:55.739 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:55.739 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:55.739 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:55.739Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:55.739Z [INFO] agent.router.manager: shutting down +47:55.740 [INFO] TestAgent: consul server down +47:55.740 [INFO] TestAgent: shutdown complete +47:55.740 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22295 network=tcp +47:55.740 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22295 network=udp +47:55.740 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22296 network=tcp +47:56.240 [INFO] TestAgent: Waiting for endpoints to shut down +47:56.240 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_UpdateCheck_DiscardOutput (0.98s) +2022-12-04T05:47:55.335Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:55.365Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:55.381 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:40e4a748-2192-161a-0510-9bf59fe950b5 Address:127.0.0.1:22265}]" +47:55.382 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22265 [Follower]" leader= +47:55.389 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8eb973a5-1801-de51-e23c-fb8a66fd76b0.dc1 127.0.0.1 +47:55.422 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8eb973a5-1801-de51-e23c-fb8a66fd76b0 127.0.0.1 +2022-12-04T05:47:55.422Z [INFO] agent.router: Initializing LAN area manager +47:55.422 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=udp +47:55.423 [INFO] TestAgent.server: Adding LAN server: server="Node-8eb973a5-1801-de51-e23c-fb8a66fd76b0 (Addr: tcp/127.0.0.1:22265) (DC: dc1)" +47:55.423 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8eb973a5-1801-de51-e23c-fb8a66fd76b0.dc1 area=wan +47:55.423 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=tcp +47:55.424 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22261 network=tcp +47:55.424 [INFO] TestAgent: started state syncer +47:55.425 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22266 network=tcp +47:55.452 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:55.452 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22265 [Candidate]" term=2 +47:55.453 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:55.453 [DEBUG] TestAgent.server.raft: vote granted: from=40e4a748-2192-161a-0510-9bf59fe950b5 term=2 tally=1 +47:55.453 [INFO] TestAgent.server.raft: election won: tally=1 +47:55.453 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22265 [Leader]" +47:55.453 [INFO] TestAgent.server: cluster leadership acquired +47:55.453 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22265 +47:55.453 [INFO] TestAgent.server: New leader elected: payload=Node-8eb973a5-1801-de51-e23c-fb8a66fd76b0 +47:55.454 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:55.489 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:55.489 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:55.489 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:55.489 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:55.489 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:55.489 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8eb973a5-1801-de51-e23c-fb8a66fd76b0 +47:55.489 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8eb973a5-1801-de51-e23c-fb8a66fd76b0 +47:55.491 [INFO] TestAgent.server: federation state anti-entropy synced +47:55.584 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.585 [INFO] TestAgent: Synced node info +47:55.830 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.830 [INFO] TestAgent: Synced node info +47:55.831 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.831 [INFO] TestAgent: Synced node info +47:55.831 [INFO] TestAgent: Requesting shutdown +47:55.831 [INFO] TestAgent.server: shutting down server +47:55.831 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:55.831 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:55.831 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:55.831 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:55.831 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:55.831 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:55.831 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:55.831 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:55.831 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:55.831 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:55.831 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:55.831Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:55.831Z [INFO] agent.router.manager: shutting down +47:55.831 [INFO] TestAgent: consul server down +47:55.831 [INFO] TestAgent: shutdown complete +47:55.831 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=tcp +47:55.831 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=udp +47:55.831 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22261 network=tcp +47:56.332 [INFO] TestAgent: Waiting for endpoints to shut down +47:56.332 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_NodeInfo (1.07s) +2022-12-04T05:47:55.551Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:55.583Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:55.639 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fca08083-3c57-a09f-4fa4-e186950e446d Address:127.0.0.1:22321}]" +47:55.639 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22321 [Follower]" leader= +47:55.649 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fca08083-3c57-a09f-4fa4-e186950e446d.dc1 127.0.0.1 +47:55.650 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fca08083-3c57-a09f-4fa4-e186950e446d 127.0.0.1 +2022-12-04T05:47:55.650Z [INFO] agent.router: Initializing LAN area manager +47:55.650 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22316 network=udp +47:55.650 [INFO] TestAgent.server: Adding LAN server: server="Node-fca08083-3c57-a09f-4fa4-e186950e446d (Addr: tcp/127.0.0.1:22321) (DC: dc1)" +47:55.650 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fca08083-3c57-a09f-4fa4-e186950e446d.dc1 area=wan +47:55.650 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22316 network=tcp +47:55.651 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22317 network=tcp +47:55.651 [INFO] TestAgent: started state syncer +47:55.651 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22322 network=tcp +47:55.712 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:55.712 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22321 [Candidate]" term=2 +47:55.712 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:55.712 [DEBUG] TestAgent.server.raft: vote granted: from=fca08083-3c57-a09f-4fa4-e186950e446d term=2 tally=1 +47:55.712 [INFO] TestAgent.server.raft: election won: tally=1 +47:55.712 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22321 [Leader]" +47:55.712 [INFO] TestAgent.server: cluster leadership acquired +47:55.712 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22321 +47:55.713 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:55.715 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:55.715 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:55.715 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:55.715 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:55.715 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:55.715 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fca08083-3c57-a09f-4fa4-e186950e446d +47:55.715 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fca08083-3c57-a09f-4fa4-e186950e446d +47:55.715 [INFO] TestAgent.server: New leader elected: payload=Node-fca08083-3c57-a09f-4fa4-e186950e446d +47:55.716 [INFO] TestAgent.server: federation state anti-entropy synced +47:55.850 [INFO] TestAgent: Requesting shutdown +47:55.850 [INFO] TestAgent.server: shutting down server +47:55.850 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:55.850 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:55.850 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:55.850 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:55.850 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:55.850 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:55.850 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:55.850 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +47:55.850 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:55.850 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:55.850 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:55.850 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:55.851Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:55.851Z [INFO] agent.router.manager: shutting down +47:55.851 [INFO] TestAgent: consul server down +47:55.851 [INFO] TestAgent: shutdown complete +47:55.851 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22316 network=tcp +47:55.851 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22316 network=udp +47:55.851 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22317 network=tcp +47:56.351 [INFO] TestAgent: Waiting for endpoints to shut down +47:56.351 [INFO] TestAgent: Endpoints down +--- PASS: TestAgent_ServiceWatchCh (1.09s) +2022-12-04T05:47:55.490Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:47:55.490Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:55.601Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:47:55.601Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:55.638 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:caefe9f2-68f4-083a-06c9-664f5e3fe9b6 Address:127.0.0.1:22328}]" +47:55.638 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22328 [Follower]" leader= +47:55.639 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-caefe9f2-68f4-083a-06c9-664f5e3fe9b6.dc1 127.0.0.1 +47:55.648 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-caefe9f2-68f4-083a-06c9-664f5e3fe9b6 127.0.0.1 +2022-12-04T05:47:55.648Z [INFO] agent.router: Initializing LAN area manager +47:55.648 [INFO] TestAgent.server: Adding LAN server: server="Node-caefe9f2-68f4-083a-06c9-664f5e3fe9b6 (Addr: tcp/127.0.0.1:22328) (DC: dc1)" +47:55.648 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-caefe9f2-68f4-083a-06c9-664f5e3fe9b6.dc1 area=wan +47:55.649 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22323 network=udp +47:55.649 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22323 network=tcp +47:55.650 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22324 network=tcp +47:55.650 [INFO] TestAgent: started state syncer +47:55.650 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22329 network=tcp +47:55.684 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:55.684 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22328 [Candidate]" term=2 +47:55.684 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:55.684 [DEBUG] TestAgent.server.raft: vote granted: from=caefe9f2-68f4-083a-06c9-664f5e3fe9b6 term=2 tally=1 +47:55.684 [INFO] TestAgent.server.raft: election won: tally=1 +47:55.684 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22328 [Leader]" +47:55.684 [INFO] TestAgent.server: cluster leadership acquired +47:55.684 [INFO] TestAgent.server: initializing acls +47:55.685 [INFO] TestAgent.server: Created ACL 'global-management' policy +47:55.685 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +47:55.685 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +47:55.685 [INFO] TestAgent.server: Created ACL anonymous token from configuration +47:55.685 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +47:55.685 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +47:55.686 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-caefe9f2-68f4-083a-06c9-664f5e3fe9b6 +47:55.686 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-caefe9f2-68f4-083a-06c9-664f5e3fe9b6.dc1 +47:55.686 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:55.687 [INFO] TestAgent.server: New leader elected: payload=Node-caefe9f2-68f4-083a-06c9-664f5e3fe9b6 +47:55.687 [INFO] TestAgent.server: Updating LAN server: server="Node-caefe9f2-68f4-083a-06c9-664f5e3fe9b6 (Addr: tcp/127.0.0.1:22328) (DC: dc1)" +47:55.687 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-caefe9f2-68f4-083a-06c9-664f5e3fe9b6.dc1 area=wan +47:55.688 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:55.688 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:55.688 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:55.688 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:55.688 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:55.688 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-caefe9f2-68f4-083a-06c9-664f5e3fe9b6 +47:55.688 [INFO] TestAgent.server: member joined, marking health alive: member=Node-caefe9f2-68f4-083a-06c9-664f5e3fe9b6 +47:55.688 [INFO] TestAgent.server: federation state anti-entropy synced +47:55.689 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-caefe9f2-68f4-083a-06c9-664f5e3fe9b6 +47:55.868 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-caefe9f2-68f4-083a-06c9-664f5e3fe9b6 +47:55.868 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-caefe9f2-68f4-083a-06c9-664f5e3fe9b6 +47:55.869 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +47:55.869 [WARN] TestAgent: Node info update blocked by ACLs: node=caefe9f2-68f4-083a-06c9-664f5e3fe9b6 accessorID=00000000-0000-0000-0000-000000000002 +47:55.870 [INFO] TestAgent: Synced service: service=api +47:55.870 [WARN] TestAgent: Service registration blocked by ACLs: service=mysql accessorID= +47:55.870 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +47:55.870 [WARN] TestAgent: Node info update blocked by ACLs: node=caefe9f2-68f4-083a-06c9-664f5e3fe9b6 accessorID=00000000-0000-0000-0000-000000000002 +47:55.870 [WARN] TestAgent: Service registration blocked by ACLs: service=mysql accessorID= +47:55.871 [INFO] TestAgent: Deregistered service: service=api +47:55.871 [INFO] TestAgent: Requesting shutdown +47:55.871 [INFO] TestAgent.server: shutting down server +47:55.871 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:55.871 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:55.871 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +47:55.871 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +47:55.871 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:55.871 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:55.871 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:55.871 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +47:55.871 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:55.871 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:55.871 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +47:55.871 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:55.871 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:55.871 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +47:55.871 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:55.871 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:55.871Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:55.871Z [INFO] agent.router.manager: shutting down +47:55.871 [INFO] TestAgent: consul server down +47:55.871 [INFO] TestAgent: shutdown complete +47:55.871 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22323 network=tcp +47:55.871 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22323 network=udp +47:55.871 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22324 network=tcp +47:56.372 [INFO] TestAgent: Waiting for endpoints to shut down +47:56.372 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_Services_ACLDeny (1.11s) +2022-12-04T05:47:55.441Z [WARN] agent: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:47:55.442Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:55.587Z [WARN] agent.auto_config: The 'acl_datacenter' field is deprecated. Use the 'primary_datacenter' field instead. +2022-12-04T05:47:55.587Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:55.593 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:26298c64-c2b5-a0e8-6a98-0a8dc173ba77 Address:127.0.0.1:22279}]" +47:55.595 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-26298c64-c2b5-a0e8-6a98-0a8dc173ba77.dc1 127.0.0.1 +47:55.596 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-26298c64-c2b5-a0e8-6a98-0a8dc173ba77 127.0.0.1 +2022-12-04T05:47:55.596Z [INFO] agent.router: Initializing LAN area manager +47:55.604 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22279 [Follower]" leader= +47:55.605 [INFO] TestAgent.server: Adding LAN server: server="Node-26298c64-c2b5-a0e8-6a98-0a8dc173ba77 (Addr: tcp/127.0.0.1:22279) (DC: dc1)" +47:55.605 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-26298c64-c2b5-a0e8-6a98-0a8dc173ba77.dc1 area=wan +47:55.637 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22274 network=tcp +47:55.639 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22274 network=udp +47:55.648 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22275 network=tcp +47:55.649 [INFO] TestAgent: started state syncer +47:55.649 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22280 network=tcp +47:55.656 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +47:55.673 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:55.673 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22279 [Candidate]" term=2 +47:55.673 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:55.673 [DEBUG] TestAgent.server.raft: vote granted: from=26298c64-c2b5-a0e8-6a98-0a8dc173ba77 term=2 tally=1 +47:55.673 [INFO] TestAgent.server.raft: election won: tally=1 +47:55.673 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22279 [Leader]" +47:55.673 [INFO] TestAgent.server: cluster leadership acquired +47:55.674 [INFO] TestAgent.server: initializing acls +47:55.674 [INFO] TestAgent.server: Created ACL 'global-management' policy +47:55.674 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +47:55.674 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +47:55.675 [INFO] TestAgent.server: Created ACL anonymous token from configuration +47:55.675 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +47:55.675 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +47:55.675 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-26298c64-c2b5-a0e8-6a98-0a8dc173ba77 +47:55.675 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-26298c64-c2b5-a0e8-6a98-0a8dc173ba77.dc1 +47:55.675 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:55.676 [INFO] TestAgent.server: New leader elected: payload=Node-26298c64-c2b5-a0e8-6a98-0a8dc173ba77 +47:55.676 [INFO] TestAgent.server: Updating LAN server: server="Node-26298c64-c2b5-a0e8-6a98-0a8dc173ba77 (Addr: tcp/127.0.0.1:22279) (DC: dc1)" +47:55.676 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-26298c64-c2b5-a0e8-6a98-0a8dc173ba77.dc1 area=wan +47:55.677 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:55.677 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:55.677 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:55.677 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:55.677 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:55.677 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-26298c64-c2b5-a0e8-6a98-0a8dc173ba77 +47:55.677 [INFO] TestAgent.server: member joined, marking health alive: member=Node-26298c64-c2b5-a0e8-6a98-0a8dc173ba77 +47:55.677 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-26298c64-c2b5-a0e8-6a98-0a8dc173ba77 +47:55.678 [INFO] TestAgent.server: federation state anti-entropy synced +47:55.772 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +47:55.772 [WARN] TestAgent: Node info update blocked by ACLs: node=26298c64-c2b5-a0e8-6a98-0a8dc173ba77 accessorID=00000000-0000-0000-0000-000000000002 +47:55.894 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-26298c64-c2b5-a0e8-6a98-0a8dc173ba77 +47:55.894 [DEBUG] TestAgent.acl: dropping node from result due to ACLs: node=Node-26298c64-c2b5-a0e8-6a98-0a8dc173ba77 +47:55.895 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +47:55.895 [WARN] TestAgent: Node info update blocked by ACLs: node=26298c64-c2b5-a0e8-6a98-0a8dc173ba77 accessorID=00000000-0000-0000-0000-000000000002 +47:55.895 [INFO] TestAgent: Synced service: service=mysql +47:55.896 [INFO] TestAgent: Synced service: service=api +47:55.896 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +47:55.896 [WARN] TestAgent: Node info update blocked by ACLs: node=26298c64-c2b5-a0e8-6a98-0a8dc173ba77 accessorID=00000000-0000-0000-0000-000000000002 +47:55.897 [INFO] TestAgent: Synced service: service=mysql +47:55.897 [INFO] TestAgent: Synced service: service=api +47:55.897 [WARN] TestAgent: Check registration blocked by ACLs: check=mysql-check accessorID= +47:55.898 [INFO] TestAgent: Synced check: check=api-check +47:55.898 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=api-check +47:55.898 [DEBUG] TestAgent.acl: dropping check from result due to ACLs: check=serfHealth +47:55.898 [WARN] TestAgent: Node info update blocked by ACLs: node=26298c64-c2b5-a0e8-6a98-0a8dc173ba77 accessorID=00000000-0000-0000-0000-000000000002 +47:55.898 [INFO] TestAgent: Synced service: service=api +47:55.898 [INFO] TestAgent: Synced service: service=mysql +47:55.898 [WARN] TestAgent: Check registration blocked by ACLs: check=mysql-check accessorID= +47:55.899 [INFO] TestAgent: Deregistered check: check=api-check +47:55.899 [INFO] TestAgent: Requesting shutdown +47:55.899 [INFO] TestAgent.server: shutting down server +47:55.899 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:55.899 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:55.899 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +47:55.899 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +47:55.899 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:55.899 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:55.899 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:55.899 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:55.899 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:55.899 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:55.899 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:55.899 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +47:55.899 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +47:55.899 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:55.899 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:55.899Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:55.899Z [INFO] agent.router.manager: shutting down +47:55.899 [INFO] TestAgent: consul server down +47:55.899 [INFO] TestAgent: shutdown complete +47:55.899 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22274 network=tcp +47:55.899 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22274 network=udp +47:55.899 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22275 network=tcp +47:56.400 [INFO] TestAgent: Waiting for endpoints to shut down +47:56.400 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_Checks_ACLDeny (1.14s) +2022-12-04T05:47:55.551Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:55.589Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:55.593 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:903ea40c-d1e4-83fb-b4ec-ddce17256fc8 Address:127.0.0.1:22335}]" +47:55.596 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-903ea40c-d1e4-83fb-b4ec-ddce17256fc8.dc1 127.0.0.1 +47:55.597 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-903ea40c-d1e4-83fb-b4ec-ddce17256fc8 127.0.0.1 +2022-12-04T05:47:55.597Z [INFO] agent.router: Initializing LAN area manager +47:55.605 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22335 [Follower]" leader= +47:55.605 [INFO] TestAgent.server: Adding LAN server: server="Node-903ea40c-d1e4-83fb-b4ec-ddce17256fc8 (Addr: tcp/127.0.0.1:22335) (DC: dc1)" +47:55.605 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-903ea40c-d1e4-83fb-b4ec-ddce17256fc8.dc1 area=wan +47:55.637 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22330 network=udp +47:55.638 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22330 network=tcp +47:55.638 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22331 network=tcp +47:55.638 [INFO] TestAgent: started state syncer +47:55.638 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22336 network=tcp +47:55.648 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:55.648 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22335 [Candidate]" term=2 +47:55.649 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:55.649 [DEBUG] TestAgent.server.raft: vote granted: from=903ea40c-d1e4-83fb-b4ec-ddce17256fc8 term=2 tally=1 +47:55.649 [INFO] TestAgent.server.raft: election won: tally=1 +47:55.649 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22335 [Leader]" +47:55.649 [INFO] TestAgent.server: cluster leadership acquired +47:55.649 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22335 +47:55.650 [INFO] TestAgent.server: New leader elected: payload=Node-903ea40c-d1e4-83fb-b4ec-ddce17256fc8 +47:55.650 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:55.653 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:55.653 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:55.653 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:55.653 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:55.653 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:55.653 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-903ea40c-d1e4-83fb-b4ec-ddce17256fc8 +47:55.653 [INFO] TestAgent.server: member joined, marking health alive: member=Node-903ea40c-d1e4-83fb-b4ec-ddce17256fc8 +47:55.654 [INFO] TestAgent.server: federation state anti-entropy synced +47:55.664 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.664 [INFO] TestAgent: Synced node info +47:55.930 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.930 [INFO] TestAgent: Synced node info +47:55.930 [INFO] TestAgent: Deregistered service: service=deleted-check-service +47:55.931 [INFO] TestAgent: Deregistered check: check=lb +47:55.931 [INFO] TestAgent: Requesting shutdown +47:55.931 [INFO] TestAgent.server: shutting down server +47:55.931 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:55.931 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:55.931 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:55.931 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:55.931 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:55.931 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:55.931 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:55.931 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:55.931 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:55.931 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:55.931 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:55.931Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:55.931Z [INFO] agent.router.manager: shutting down +47:55.931 [INFO] TestAgent: consul server down +47:55.931 [INFO] TestAgent: shutdown complete +47:55.931 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22330 network=tcp +47:55.931 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22330 network=udp +47:55.931 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22331 network=tcp +47:56.432 [INFO] TestAgent: Waiting for endpoints to shut down +47:56.432 [INFO] TestAgent: Endpoints down +2022-12-04T05:47:55.405Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:55.474Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:55.476 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5e0c6d1d-8ee7-5326-355b-aeb96b34ee87 Address:127.0.0.1:22286}]" +47:55.477 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22286 [Follower]" leader= +47:55.477 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5e0c6d1d-8ee7-5326-355b-aeb96b34ee87.dc1 127.0.0.1 +47:55.490 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5e0c6d1d-8ee7-5326-355b-aeb96b34ee87 127.0.0.1 +2022-12-04T05:47:55.492Z [INFO] agent.router: Initializing LAN area manager +47:55.493 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22281 network=udp +47:55.493 [INFO] TestAgent.server: Adding LAN server: server="Node-5e0c6d1d-8ee7-5326-355b-aeb96b34ee87 (Addr: tcp/127.0.0.1:22286) (DC: dc1)" +47:55.493 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5e0c6d1d-8ee7-5326-355b-aeb96b34ee87.dc1 area=wan +47:55.493 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22281 network=tcp +47:55.497 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22282 network=tcp +47:55.497 [INFO] TestAgent: started state syncer +47:55.497 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22287 network=tcp +47:55.549 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:55.549 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22286 [Candidate]" term=2 +47:55.549 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:55.549 [DEBUG] TestAgent.server.raft: vote granted: from=5e0c6d1d-8ee7-5326-355b-aeb96b34ee87 term=2 tally=1 +47:55.549 [INFO] TestAgent.server.raft: election won: tally=1 +47:55.549 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22286 [Leader]" +47:55.549 [INFO] TestAgent.server: cluster leadership acquired +47:55.555 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22286 +47:55.569 [INFO] TestAgent.server: New leader elected: payload=Node-5e0c6d1d-8ee7-5326-355b-aeb96b34ee87 +47:55.569 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:55.571 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:55.571 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:55.571 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:55.571 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:55.571 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:55.571 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5e0c6d1d-8ee7-5326-355b-aeb96b34ee87 +47:55.571 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5e0c6d1d-8ee7-5326-355b-aeb96b34ee87 +47:55.571 [INFO] TestAgent.server: federation state anti-entropy synced +47:55.848 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.849 [INFO] TestAgent: Synced node info +47:55.928 [INFO] TestAgent: Requesting shutdown +47:55.928 [INFO] TestAgent.server: shutting down server +47:55.928 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:55.928 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:55.928 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:55.928 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:55.928 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:55.928 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:55.928 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:55.928 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:55.928 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:55.928 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:55.929 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:55.929Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:55.929Z [INFO] agent.router.manager: shutting down +47:55.929 [INFO] TestAgent: consul server down +47:55.929 [INFO] TestAgent: shutdown complete +47:55.929 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22281 network=tcp +47:55.929 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22281 network=udp +47:55.929 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22282 network=tcp +47:56.432 [INFO] TestAgent: Waiting for endpoints to shut down +47:56.432 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_RemovingServiceAndCheck (1.14s) +--- PASS: TestAliasNotifications_local (1.17s) +2022-12-04T05:47:55.565Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:55.613Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:55.615 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1243d2d4-f747-ecba-2ee0-77cf3ee2b4c1 Address:127.0.0.1:22342}]" +47:55.615 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22342 [Follower]" leader= +47:55.615 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1243d2d4-f747-ecba-2ee0-77cf3ee2b4c1.dc1 127.0.0.1 +47:55.615 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1243d2d4-f747-ecba-2ee0-77cf3ee2b4c1 127.0.0.1 +2022-12-04T05:47:55.615Z [INFO] agent.router: Initializing LAN area manager +47:55.616 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1243d2d4-f747-ecba-2ee0-77cf3ee2b4c1.dc1 area=wan +47:55.616 [INFO] TestAgent.server: Adding LAN server: server="Node-1243d2d4-f747-ecba-2ee0-77cf3ee2b4c1 (Addr: tcp/127.0.0.1:22342) (DC: dc1)" +47:55.616 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22337 network=udp +47:55.616 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22337 network=tcp +47:55.616 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22338 network=tcp +47:55.616 [INFO] TestAgent: started state syncer +47:55.616 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22343 network=tcp +47:55.678 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:55.678 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22342 [Candidate]" term=2 +47:55.678 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:55.678 [DEBUG] TestAgent.server.raft: vote granted: from=1243d2d4-f747-ecba-2ee0-77cf3ee2b4c1 term=2 tally=1 +47:55.678 [INFO] TestAgent.server.raft: election won: tally=1 +47:55.678 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22342 [Leader]" +47:55.678 [INFO] TestAgent.server: cluster leadership acquired +47:55.678 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22342 +47:55.679 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:55.679 [INFO] TestAgent.server: New leader elected: payload=Node-1243d2d4-f747-ecba-2ee0-77cf3ee2b4c1 +47:55.681 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:55.681 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:55.681 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:55.681 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:55.681 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:55.681 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1243d2d4-f747-ecba-2ee0-77cf3ee2b4c1 +47:55.681 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1243d2d4-f747-ecba-2ee0-77cf3ee2b4c1 +47:55.681 [INFO] TestAgent.server: federation state anti-entropy synced +47:55.880 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.880 [INFO] TestAgent: Synced node info +47:56.002 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:56.002 [INFO] TestAgent: Synced node info +47:56.002 [INFO] TestAgent: Synced service: service=web-proxy +47:56.002 [INFO] TestAgent: Synced service: service=cache-proxy +47:56.003 [INFO] TestAgent: Deregistered service: service=lb-proxy +47:56.003 [DEBUG] TestAgent: Service in sync: service=mysql-proxy +47:56.003 [INFO] TestAgent: Synced service: service=redis-proxy +47:56.003 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:56.003 [DEBUG] TestAgent: Node info in sync +47:56.003 [DEBUG] TestAgent: Service in sync: service=mysql-proxy +47:56.003 [DEBUG] TestAgent: Service in sync: service=redis-proxy +47:56.003 [DEBUG] TestAgent: Service in sync: service=web-proxy +47:56.004 [INFO] TestAgent: Deregistered service: service=cache-proxy +47:56.004 [INFO] TestAgent: Requesting shutdown +47:56.004 [INFO] TestAgent.server: shutting down server +47:56.004 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:56.004 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:56.004 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:56.004 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:56.004 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:56.004 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:56.004 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:56.004 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:56.004 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:56.004 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:56.004 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:56.004Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:56.004Z [INFO] agent.router.manager: shutting down +47:56.005 [INFO] TestAgent: consul server down +47:56.005 [INFO] TestAgent: shutdown complete +47:56.005 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22337 network=tcp +47:56.005 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22337 network=udp +47:56.005 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22338 network=tcp +47:56.507 [INFO] TestAgent: Waiting for endpoints to shut down +47:56.507 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_Services_ConnectProxy (1.21s) +2022-12-04T05:47:55.475Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:47:55.577Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +47:55.590 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e8417040-1c35-8a9b-2a77-42233dd21aa5 Address:127.0.0.1:22307}]" +47:55.597 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e8417040-1c35-8a9b-2a77-42233dd21aa5.dc1 127.0.0.1 +47:55.598 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e8417040-1c35-8a9b-2a77-42233dd21aa5 127.0.0.1 +2022-12-04T05:47:55.598Z [INFO] agent.router: Initializing LAN area manager +47:55.604 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22307 [Follower]" leader= +47:55.606 [INFO] TestAgent.server: Adding LAN server: server="Node-e8417040-1c35-8a9b-2a77-42233dd21aa5 (Addr: tcp/127.0.0.1:22307) (DC: dc1)" +47:55.606 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e8417040-1c35-8a9b-2a77-42233dd21aa5.dc1 area=wan +47:55.637 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22302 network=tcp +47:55.637 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22302 network=udp +47:55.638 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22303 network=tcp +47:55.638 [INFO] TestAgent: started state syncer +47:55.638 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22308 network=tcp +47:55.654 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +47:55.654 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22307 [Candidate]" term=2 +47:55.654 [DEBUG] TestAgent.server.raft: votes: needed=1 +47:55.654 [DEBUG] TestAgent.server.raft: vote granted: from=e8417040-1c35-8a9b-2a77-42233dd21aa5 term=2 tally=1 +47:55.654 [INFO] TestAgent.server.raft: election won: tally=1 +47:55.654 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22307 [Leader]" +47:55.654 [INFO] TestAgent.server: cluster leadership acquired +47:55.655 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22307 +47:55.656 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +47:55.656 [INFO] TestAgent.server: New leader elected: payload=Node-e8417040-1c35-8a9b-2a77-42233dd21aa5 +47:55.665 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +47:55.665 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +47:55.665 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +47:55.665 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +47:55.665 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +47:55.665 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e8417040-1c35-8a9b-2a77-42233dd21aa5 +47:55.665 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e8417040-1c35-8a9b-2a77-42233dd21aa5 +47:55.666 [INFO] TestAgent.server: federation state anti-entropy synced +47:55.964 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:55.965 [INFO] TestAgent: Synced node info +47:55.965 [INFO] TestAgent: Synced check: check=web +47:56.008 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:56.008 [DEBUG] TestAgent: Node info in sync +47:56.008 [DEBUG] TestAgent: Check in sync: check=web +47:58.523 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:58.523 [DEBUG] TestAgent: Node info in sync +47:58.524 [INFO] TestAgent: Synced check: check=web +47:58.524 [DEBUG] TestAgent: Node info in sync +47:58.524 [DEBUG] TestAgent: Check in sync: check=web +47:58.533 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:58.533 [INFO] TestAgent: Synced node info +47:58.533 [INFO] TestAgent: Synced check: check=web +47:58.534 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +47:58.534 [INFO] TestAgent: Synced node info +47:58.534 [DEBUG] TestAgent: Check in sync: check=web +47:59.264 [DEBUG] TestAgent: Node info in sync +47:59.265 [INFO] TestAgent: Synced check: check=web +47:59.280 [INFO] TestAgent: Requesting shutdown +47:59.280 [INFO] TestAgent.server: shutting down server +47:59.280 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +47:59.280 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +47:59.280 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +47:59.280 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +47:59.280 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +47:59.280 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +47:59.280 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +47:59.280 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +47:59.280 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +47:59.280 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +47:59.280 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:47:59.281Z [INFO] agent.router.manager: shutting down +2022-12-04T05:47:59.281Z [INFO] agent.router.manager: shutting down +47:59.281 [INFO] TestAgent: consul server down +47:59.281 [INFO] TestAgent: shutdown complete +47:59.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22302 network=tcp +47:59.281 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22302 network=udp +47:59.281 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22303 network=tcp +47:59.784 [INFO] TestAgent: Waiting for endpoints to shut down +47:59.784 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentAntiEntropy_Check_DeferSync (4.52s) PASS -ok github.com/hashicorp/consul/agent/local 4.907s +ok github.com/hashicorp/consul/agent/local 4.606s === RUN TestBuild === RUN TestBuild/no_version === RUN TestBuild/bad_version @@ -58818,7 +58952,7 @@ === RUN TestIsConsulServer_Optional --- PASS: TestIsConsulServer_Optional (0.00s) PASS -ok github.com/hashicorp/consul/agent/metadata 0.109s +ok github.com/hashicorp/consul/agent/metadata 0.033s ? github.com/hashicorp/consul/agent/mock [no test files] === RUN TestPeekForTLS_not_TLS === RUN TestPeekForTLS_not_TLS/tcp_rpc_type_byte_0 @@ -58847,17 +58981,17 @@ === RUN TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_5 === RUN TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_6 === RUN TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_7 ---- PASS: TestPeekForTLS_actual_TLS (0.08s) - --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_0 (0.04s) +--- PASS: TestPeekForTLS_actual_TLS (0.04s) + --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_0 (0.02s) --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_1 (0.00s) - --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_2 (0.01s) + --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_2 (0.00s) --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_3 (0.00s) - --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_4 (0.01s) - --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_5 (0.01s) - --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_6 (0.01s) + --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_4 (0.00s) + --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_5 (0.00s) + --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_6 (0.00s) --- PASS: TestPeekForTLS_actual_TLS/tcp_rpc_type_byte_7 (0.00s) PASS -ok github.com/hashicorp/consul/agent/pool 0.147s +ok github.com/hashicorp/consul/agent/pool 0.176s === RUN TestManagerInternal_cycleServer --- PASS: TestManagerInternal_cycleServer (0.00s) === RUN TestManagerInternal_getServerList @@ -58891,11 +59025,11 @@ === RUN TestRouter_Routing --- PASS: TestRouter_Routing (0.00s) === RUN TestRouter_Routing_Offline -2021-10-31T23:11:45.859Z [DEBUG] TestRouter_Routing_Offline.router.manager: pinging server failed: server="node3.dc1 (Addr: tcp/127.0.0.4:8300) (DC: dc1)" error= -2021-10-31T23:11:45.859Z [DEBUG] TestRouter_Routing_Offline.router.manager: pinging server failed: server="node2.dc1 (Addr: tcp/127.0.0.3:8300) (DC: dc1)" error= -2021-10-31T23:11:45.859Z [DEBUG] TestRouter_Routing_Offline.router.manager: pinging server failed: server="node4.dc1 (Addr: tcp/127.0.0.5:8300) (DC: dc1)" error= -2021-10-31T23:11:45.859Z [DEBUG] TestRouter_Routing_Offline.router.manager: pinging server failed: server="node1.dc1 (Addr: tcp/127.0.0.2:8300) (DC: dc1)" error= -2021-10-31T23:11:45.859Z [DEBUG] TestRouter_Routing_Offline.router.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.544Z [DEBUG] TestRouter_Routing_Offline.router.manager: pinging server failed: server="node4.dc1 (Addr: tcp/127.0.0.5:8300) (DC: dc1)" error= +2022-12-04T05:47:54.544Z [DEBUG] TestRouter_Routing_Offline.router.manager: pinging server failed: server="node1.dc1 (Addr: tcp/127.0.0.2:8300) (DC: dc1)" error= +2022-12-04T05:47:54.544Z [DEBUG] TestRouter_Routing_Offline.router.manager: pinging server failed: server="node2.dc1 (Addr: tcp/127.0.0.3:8300) (DC: dc1)" error= +2022-12-04T05:47:54.544Z [DEBUG] TestRouter_Routing_Offline.router.manager: pinging server failed: server="node3.dc1 (Addr: tcp/127.0.0.4:8300) (DC: dc1)" error= +2022-12-04T05:47:54.544Z [DEBUG] TestRouter_Routing_Offline.router.manager: No healthy servers during rebalance, aborting --- PASS: TestRouter_Routing_Offline (0.00s) === RUN TestRouter_GetDatacenters --- PASS: TestRouter_GetDatacenters (0.00s) @@ -58906,10568 +59040,10551 @@ === RUN TestRouter_GetDatacenterMaps --- PASS: TestRouter_GetDatacenterMaps (0.00s) === RUN TestRouter_FindLANServer -2021-10-31T23:11:45.860Z [INFO] TestRouter_FindLANServer.router: Initializing LAN area manager +2022-12-04T05:47:54.545Z [INFO] TestRouter_FindLANServer.router: Initializing LAN area manager --- PASS: TestRouter_FindLANServer (0.00s) === RUN TestServers_AddServer --- PASS: TestServers_AddServer (0.00s) === RUN TestServers_IsOffline -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: pinging server failed: server="s1 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.546Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: Rebalanced servers, new active server: number_of_servers=1 active_server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.545Z [DEBUG] TestServers_IsOffline.manager: No healthy servers during rebalance, aborting --- PASS: TestServers_IsOffline (0.00s) === RUN TestServers_FindServer -2021-10-31T23:11:45.860Z [WARN] TestServers_FindServer.manager: No servers available -2021-10-31T23:11:45.860Z [DEBUG] TestServers_FindServer.manager: cycled away from server: server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_FindServer.manager: cycled away from server: server="s2 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [WARN] TestServers_FindServer.manager: No servers available +2022-12-04T05:47:54.547Z [DEBUG] TestServers_FindServer.manager: cycled away from server: server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_FindServer.manager: cycled away from server: server="s2 (Addr: /) (DC: )" --- PASS: TestServers_FindServer (0.00s) === RUN TestServers_New --- PASS: TestServers_New (0.00s) === RUN TestServers_NotifyFailedServer -2021-10-31T23:11:45.860Z [DEBUG] TestServers_NotifyFailedServer.manager: cycled away from server: server="s1 (Addr: /) (DC: )" -2021-10-31T23:11:45.860Z [DEBUG] TestServers_NotifyFailedServer.manager: cycled away from server: server="s2 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_NotifyFailedServer.manager: cycled away from server: server="s1 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_NotifyFailedServer.manager: cycled away from server: server="s2 (Addr: /) (DC: )" --- PASS: TestServers_NotifyFailedServer (0.00s) === RUN TestServers_NumServers --- PASS: TestServers_NumServers (0.00s) === RUN TestServers_RebalanceServers -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.861Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s48 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.870Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s67 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s59 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s93 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s63 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s88 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s66 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.871Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s88 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.872Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s63 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s98 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.873Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.874Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s08 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s86 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s61 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s59 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.875Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.897Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.898Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.899Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s50 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s79 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.900Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s25 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.901Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.906Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s15 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s40 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s22 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.907Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s58 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s18 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.908Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.909Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s31 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s91 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s22 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s52 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s82 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s85 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s92 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s63 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s43 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s79 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s53 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s36 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s09 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.910Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.911Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s61 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s83 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.912Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s91 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.913Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.917Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.917Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.917Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.917Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.917Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.917Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.917Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.917Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.917Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.917Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.917Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.917Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.917Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.917Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.917Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s87 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.918Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.919Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.920Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.921Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.926Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s33 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.927Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.928Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.928Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.928Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.928Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.928Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.928Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.928Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.928Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.944Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s57 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s93 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.945Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.946Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s18 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s55 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s56 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s54 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.947Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s24 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s44 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s31 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.948Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s94 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s53 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.949Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.953Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.953Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.953Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s38 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.953Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s29 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.953Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s93 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s31 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s50 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.954Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s79 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s99 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s11 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.955Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.956Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.957Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.958Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s09 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s68 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.959Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s40 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s53 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.960Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.961Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s88 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.962Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.969Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s61 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s69 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.970Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s12 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.971Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.972Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s40 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s04 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.973Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s91 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s67 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.974Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s62 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s31 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.975Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s99 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s48 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s82 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.976Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s64 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.977Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.981Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.982Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s99 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.983Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s20 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s95 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s96 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.984Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s00 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.985Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s34 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s66 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s62 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.988Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.989Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.994Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.995Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s48 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s68 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s64 (Addr: /) (DC: )" error= -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:45.996Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:45.997Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s97 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s30 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s08 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s33 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s89 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.002Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s39 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s82 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s23 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.003Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s24 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.004Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.005Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.009Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.009Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.009Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.009Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.009Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.009Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.009Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.009Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.009Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.009Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.009Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.009Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s92 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.010Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s68 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s64 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s51 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s75 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s58 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s02 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s92 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s23 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s99 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.011Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s65 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s27 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s57 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s74 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.012Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s26 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.013Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s04 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s02 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s42 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s48 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s67 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s99 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.014Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.018Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.030Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s91 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.031Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s27 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s50 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s76 (Addr: /) (DC: )" error= -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.032Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.033Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.034Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.035Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" -2021-10-31T23:11:46.036Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.037Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" ---- PASS: TestServers_RebalanceServers (0.28s) +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s36 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s86 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s04 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.547Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s63 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s08 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s00 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s08 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.548Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s71 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.549Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s70 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s54 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s09 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s91 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s96 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.550Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.551Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.553Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s51 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s86 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.554Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s54 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s73 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.555Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s55 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s74 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.556Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.557Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s10 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s60 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s15 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s02 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s27 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.558Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s42 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.559Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.560Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.561Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s46 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s79 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.562Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.563Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s26 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.564Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s34 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.565Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s95 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s31 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s67 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s41 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.566Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s06 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s36 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.567Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s05 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s31 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s65 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s45 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s76 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s93 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s28 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s42 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s01 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s68 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s08 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s46 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.568Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s88 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s87 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s64 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s07 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.569Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s63 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s60 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s82 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.571Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s69 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.572Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s39 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s89 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.573Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s08 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.574Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s85 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s41 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.575Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s00 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s26 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.576Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s73 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s03 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.577Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s30 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s85 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s98 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s55 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.578Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.579Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s74 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.580Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s36 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s42 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s85 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.581Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s22 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s21 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s85 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.582Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s97 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s63 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.583Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s02 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s01 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.584Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s28 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s72 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s68 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s52 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s47 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s36 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s50 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.585Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.586Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s39 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.587Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.588Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s36 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s96 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s70 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s24 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.589Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s25 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s34 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s09 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.590Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s62 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.591Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.592Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.593Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.600Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s30 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s04 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s33 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.601Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s41 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s21 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.602Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s83 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.603Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s53 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s27 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.604Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s82 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s50 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: pinging server failed: server="s64 (Addr: /) (DC: )" error= +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: Rebalanced servers, new active server: number_of_servers=100 active_server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s63 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s98 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s78 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s65 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s22 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s45 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s66 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s71 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s83 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s47 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s80 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s69 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s52 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s54 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s57 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s79 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s75 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s32 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s42 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s89 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s31 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s85 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s74 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s72 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s37 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s86 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s29 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s95 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s60 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s84 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s59 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s43 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s36 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s13 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s40 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s21 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s91 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s27 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s35 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s41 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.605Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s49 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s20 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s26 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s68 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s39 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s44 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s48 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s81 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s28 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s24 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s38 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s67 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s88 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s92 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s76 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s51 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s70 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s23 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s56 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s00 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s25 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s93 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s46 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s58 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s77 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s87 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s17 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s96 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s55 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s62 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s99 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s34 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s94 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s33 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s90 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s97 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s53 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s30 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s61 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s73 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s82 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s50 (Addr: /) (DC: )" +2022-12-04T05:47:54.606Z [DEBUG] TestServers_RebalanceServers.manager: cycled away from server: server="s64 (Addr: /) (DC: )" +--- PASS: TestServers_RebalanceServers (0.06s) === RUN TestServers_RebalanceServers_AvoidFailed -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" -2021-10-31T23:11:46.142Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.611Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: pinging server failed: server="s2 (Addr: faux/s2) (DC: )" error= +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s1 (Addr: faux/s1) (DC: )" +2022-12-04T05:47:54.610Z [DEBUG] TestServers_RebalanceServers_AvoidFailed.manager: Rebalanced servers, new active server: number_of_servers=3 active_server="s3 (Addr: faux/s3) (DC: )" --- PASS: TestServers_RebalanceServers_AvoidFailed (0.00s) === RUN TestManager_RemoveServer -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: Rebalanced servers, new active server: number_of_servers=19 active_server="s18 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s02 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s11 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s14 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s07 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s13 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s19 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s06 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s12 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s01 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s03 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s04 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s16 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s17 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s10 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s09 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s08 (Addr: /) (DC: )" -2021-10-31T23:11:46.143Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: Rebalanced servers, new active server: number_of_servers=19 active_server="s02 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s01 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s14 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s18 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s19 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s06 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s09 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s04 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s12 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s05 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s07 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s03 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s15 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s11 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s16 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s08 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s10 (Addr: /) (DC: )" +2022-12-04T05:47:54.612Z [DEBUG] TestManager_RemoveServer.manager: cycled away from server: server="s08 (Addr: /) (DC: )" --- PASS: TestManager_RemoveServer (0.00s) PASS -ok github.com/hashicorp/consul/agent/router 0.387s +ok github.com/hashicorp/consul/agent/router 0.085s === RUN TestStructs_ACLCaches === RUN TestStructs_ACLCaches/New === RUN TestStructs_ACLCaches/New/Valid_Sizes @@ -69839,7 +69956,7 @@ === RUN TestServiceRouterConfigEntry/route_with_header_match_and_prefix_rewrite#01 === RUN TestServiceRouterConfigEntry/route_with_method_matches === RUN TestServiceRouterConfigEntry/route_with_method_matches_repeated ---- PASS: TestServiceRouterConfigEntry (0.01s) +--- PASS: TestServiceRouterConfigEntry (0.00s) --- PASS: TestServiceRouterConfigEntry/nil (0.00s) --- PASS: TestServiceRouterConfigEntry/no_name (0.00s) --- PASS: TestServiceRouterConfigEntry/empty (0.00s) @@ -69973,13 +70090,13 @@ --- PASS: TestIngressConfigEntry_Normalize/empty_protocol (0.00s) --- PASS: TestIngressConfigEntry_Normalize/lowercase_protocols (0.00s) === RUN TestIngressConfigEntry_ListRelatedServices -=== RUN TestIngressConfigEntry_ListRelatedServices/one_exact === RUN TestIngressConfigEntry_ListRelatedServices/one_wild === RUN TestIngressConfigEntry_ListRelatedServices/kitchen_sink +=== RUN TestIngressConfigEntry_ListRelatedServices/one_exact --- PASS: TestIngressConfigEntry_ListRelatedServices (0.00s) - --- PASS: TestIngressConfigEntry_ListRelatedServices/one_exact (0.00s) --- PASS: TestIngressConfigEntry_ListRelatedServices/one_wild (0.00s) --- PASS: TestIngressConfigEntry_ListRelatedServices/kitchen_sink (0.00s) + --- PASS: TestIngressConfigEntry_ListRelatedServices/one_exact (0.00s) === RUN TestIngressConfigEntry_Validate === RUN TestIngressConfigEntry_Validate/port_conflict === RUN TestIngressConfigEntry_Validate/http_features:_wildcard @@ -70183,7 +70300,7 @@ === RUN TestValidateMeshGatewayMode/local_(explicit) === RUN TestValidateMeshGatewayMode/remote_(constant) === RUN TestValidateMeshGatewayMode/remote_(explicit) ---- PASS: TestValidateMeshGatewayMode (0.02s) +--- PASS: TestValidateMeshGatewayMode (0.00s) --- PASS: TestValidateMeshGatewayMode/none_(constant) (0.00s) --- PASS: TestValidateMeshGatewayMode/none_(explicit) (0.00s) --- PASS: TestValidateMeshGatewayMode/_(constant) (0.00s) @@ -70264,7 +70381,7 @@ === RUN TestStructs_FilterFieldConfigurations/Node === RUN TestStructs_FilterFieldConfigurations/NodeService --- PASS: TestStructs_FilterFieldConfigurations (0.01s) - --- PASS: TestStructs_FilterFieldConfigurations/ServiceNode (0.01s) + --- PASS: TestStructs_FilterFieldConfigurations/ServiceNode (0.00s) --- PASS: TestStructs_FilterFieldConfigurations/HealthCheck (0.00s) --- PASS: TestStructs_FilterFieldConfigurations/CheckServiceNode (0.00s) --- PASS: TestStructs_FilterFieldConfigurations/NodeInfo (0.00s) @@ -70289,23 +70406,23 @@ === RUN TestStructs_ServiceNode_Conversions --- PASS: TestStructs_ServiceNode_Conversions (0.00s) === RUN TestStructs_NodeService_ValidateMeshGateway +=== RUN TestStructs_NodeService_ValidateMeshGateway/proxy-local-address +=== RUN TestStructs_NodeService_ValidateMeshGateway/proxy-local-port +=== RUN TestStructs_NodeService_ValidateMeshGateway/proxy-upstreams === RUN TestStructs_NodeService_ValidateMeshGateway/valid === RUN TestStructs_NodeService_ValidateMeshGateway/zero-port === RUN TestStructs_NodeService_ValidateMeshGateway/sidecar-service === RUN TestStructs_NodeService_ValidateMeshGateway/proxy-destination-name === RUN TestStructs_NodeService_ValidateMeshGateway/proxy-destination-id -=== RUN TestStructs_NodeService_ValidateMeshGateway/proxy-local-address -=== RUN TestStructs_NodeService_ValidateMeshGateway/proxy-local-port -=== RUN TestStructs_NodeService_ValidateMeshGateway/proxy-upstreams --- PASS: TestStructs_NodeService_ValidateMeshGateway (0.00s) + --- PASS: TestStructs_NodeService_ValidateMeshGateway/proxy-local-address (0.00s) + --- PASS: TestStructs_NodeService_ValidateMeshGateway/proxy-local-port (0.00s) + --- PASS: TestStructs_NodeService_ValidateMeshGateway/proxy-upstreams (0.00s) --- PASS: TestStructs_NodeService_ValidateMeshGateway/valid (0.00s) --- PASS: TestStructs_NodeService_ValidateMeshGateway/zero-port (0.00s) --- PASS: TestStructs_NodeService_ValidateMeshGateway/sidecar-service (0.00s) --- PASS: TestStructs_NodeService_ValidateMeshGateway/proxy-destination-name (0.00s) --- PASS: TestStructs_NodeService_ValidateMeshGateway/proxy-destination-id (0.00s) - --- PASS: TestStructs_NodeService_ValidateMeshGateway/proxy-local-address (0.00s) - --- PASS: TestStructs_NodeService_ValidateMeshGateway/proxy-local-port (0.00s) - --- PASS: TestStructs_NodeService_ValidateMeshGateway/proxy-upstreams (0.00s) === RUN TestStructs_NodeService_ValidateTerminatingGateway === RUN TestStructs_NodeService_ValidateTerminatingGateway/proxy-destination-name === RUN TestStructs_NodeService_ValidateTerminatingGateway/proxy-destination-id @@ -70339,21 +70456,21 @@ --- PASS: TestStructs_NodeService_ValidateIngressGateway/proxy-local-port (0.00s) --- PASS: TestStructs_NodeService_ValidateIngressGateway/proxy-upstreams (0.00s) === RUN TestStructs_NodeService_ValidateExposeConfig -=== RUN TestStructs_NodeService_ValidateExposeConfig/valid -=== RUN TestStructs_NodeService_ValidateExposeConfig/empty_path -=== RUN TestStructs_NodeService_ValidateExposeConfig/invalid_port_negative === RUN TestStructs_NodeService_ValidateExposeConfig/invalid_port_too_large === RUN TestStructs_NodeService_ValidateExposeConfig/duplicate_paths === RUN TestStructs_NodeService_ValidateExposeConfig/duplicate_ports === RUN TestStructs_NodeService_ValidateExposeConfig/protocol_not_supported +=== RUN TestStructs_NodeService_ValidateExposeConfig/valid +=== RUN TestStructs_NodeService_ValidateExposeConfig/empty_path +=== RUN TestStructs_NodeService_ValidateExposeConfig/invalid_port_negative --- PASS: TestStructs_NodeService_ValidateExposeConfig (0.00s) - --- PASS: TestStructs_NodeService_ValidateExposeConfig/valid (0.00s) - --- PASS: TestStructs_NodeService_ValidateExposeConfig/empty_path (0.00s) - --- PASS: TestStructs_NodeService_ValidateExposeConfig/invalid_port_negative (0.00s) --- PASS: TestStructs_NodeService_ValidateExposeConfig/invalid_port_too_large (0.00s) --- PASS: TestStructs_NodeService_ValidateExposeConfig/duplicate_paths (0.00s) --- PASS: TestStructs_NodeService_ValidateExposeConfig/duplicate_ports (0.00s) --- PASS: TestStructs_NodeService_ValidateExposeConfig/protocol_not_supported (0.00s) + --- PASS: TestStructs_NodeService_ValidateExposeConfig/valid (0.00s) + --- PASS: TestStructs_NodeService_ValidateExposeConfig/empty_path (0.00s) + --- PASS: TestStructs_NodeService_ValidateExposeConfig/invalid_port_negative (0.00s) === RUN TestStructs_NodeService_ValidateConnectProxy === RUN TestStructs_NodeService_ValidateConnectProxy/valid === RUN TestStructs_NodeService_ValidateConnectProxy/connect-proxy:_no_Proxy.DestinationServiceName @@ -70372,7 +70489,7 @@ === RUN TestStructs_NodeService_ValidateConnectProxy/connect-proxy:_Upstreams_duplicated_by_name === RUN TestStructs_NodeService_ValidateConnectProxy/connect-proxy:_Upstreams_duplicated_by_name_and_datacenter === RUN TestStructs_NodeService_ValidateConnectProxy/connect-proxy:_Upstreams_duplicated_by_name_and_namespace ---- PASS: TestStructs_NodeService_ValidateConnectProxy (0.01s) +--- PASS: TestStructs_NodeService_ValidateConnectProxy (0.00s) --- PASS: TestStructs_NodeService_ValidateConnectProxy/valid (0.00s) --- PASS: TestStructs_NodeService_ValidateConnectProxy/connect-proxy:_no_Proxy.DestinationServiceName (0.00s) --- PASS: TestStructs_NodeService_ValidateConnectProxy/connect-proxy:_whitespace_Proxy.DestinationServiceName (0.00s) @@ -70413,14 +70530,6 @@ === RUN TestStructs_DirEntry_Clone --- PASS: TestStructs_DirEntry_Clone (0.00s) === RUN TestStructs_ValidateServiceAndNodeMetadata -=== RUN TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed_via_an_allowlist_just_for_gateway_-_consul-wan-federation -=== RUN TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed_via_an_allowlist_just_for_gateway_-_consul-wan-federation/ValidateNodeMetadata -=== RUN TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed_via_an_allowlist_just_for_gateway_-_consul-wan-federation/ValidateServiceMetadata_-_typical -=== RUN TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed_via_an_allowlist_just_for_gateway_-_consul-wan-federation/ValidateServiceMetadata_-_mesh-gateway -=== RUN TestStructs_ValidateServiceAndNodeMetadata/should_succeed -=== RUN TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateNodeMetadata -=== RUN TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateServiceMetadata_-_typical -=== RUN TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateServiceMetadata_-_mesh-gateway === RUN TestStructs_ValidateServiceAndNodeMetadata/invalid_key === RUN TestStructs_ValidateServiceAndNodeMetadata/invalid_key/ValidateNodeMetadata === RUN TestStructs_ValidateServiceAndNodeMetadata/invalid_key/ValidateServiceMetadata_-_typical @@ -70437,15 +70546,15 @@ === RUN TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed/ValidateNodeMetadata === RUN TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed/ValidateServiceMetadata_-_typical === RUN TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed/ValidateServiceMetadata_-_mesh-gateway +=== RUN TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed_via_an_allowlist_just_for_gateway_-_consul-wan-federation +=== RUN TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed_via_an_allowlist_just_for_gateway_-_consul-wan-federation/ValidateNodeMetadata +=== RUN TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed_via_an_allowlist_just_for_gateway_-_consul-wan-federation/ValidateServiceMetadata_-_typical +=== RUN TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed_via_an_allowlist_just_for_gateway_-_consul-wan-federation/ValidateServiceMetadata_-_mesh-gateway +=== RUN TestStructs_ValidateServiceAndNodeMetadata/should_succeed +=== RUN TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateNodeMetadata +=== RUN TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateServiceMetadata_-_typical +=== RUN TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateServiceMetadata_-_mesh-gateway --- PASS: TestStructs_ValidateServiceAndNodeMetadata (0.00s) - --- PASS: TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed_via_an_allowlist_just_for_gateway_-_consul-wan-federation (0.00s) - --- PASS: TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed_via_an_allowlist_just_for_gateway_-_consul-wan-federation/ValidateNodeMetadata (0.00s) - --- PASS: TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed_via_an_allowlist_just_for_gateway_-_consul-wan-federation/ValidateServiceMetadata_-_typical (0.00s) - --- PASS: TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed_via_an_allowlist_just_for_gateway_-_consul-wan-federation/ValidateServiceMetadata_-_mesh-gateway (0.00s) - --- PASS: TestStructs_ValidateServiceAndNodeMetadata/should_succeed (0.00s) - --- PASS: TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateNodeMetadata (0.00s) - --- PASS: TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateServiceMetadata_-_typical (0.00s) - --- PASS: TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateServiceMetadata_-_mesh-gateway (0.00s) --- PASS: TestStructs_ValidateServiceAndNodeMetadata/invalid_key (0.00s) --- PASS: TestStructs_ValidateServiceAndNodeMetadata/invalid_key/ValidateNodeMetadata (0.00s) --- PASS: TestStructs_ValidateServiceAndNodeMetadata/invalid_key/ValidateServiceMetadata_-_typical (0.00s) @@ -70462,6 +70571,14 @@ --- PASS: TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed/ValidateNodeMetadata (0.00s) --- PASS: TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed/ValidateServiceMetadata_-_typical (0.00s) --- PASS: TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed/ValidateServiceMetadata_-_mesh-gateway (0.00s) + --- PASS: TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed_via_an_allowlist_just_for_gateway_-_consul-wan-federation (0.00s) + --- PASS: TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed_via_an_allowlist_just_for_gateway_-_consul-wan-federation/ValidateNodeMetadata (0.00s) + --- PASS: TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed_via_an_allowlist_just_for_gateway_-_consul-wan-federation/ValidateServiceMetadata_-_typical (0.00s) + --- PASS: TestStructs_ValidateServiceAndNodeMetadata/reserved_key_prefix_allowed_via_an_allowlist_just_for_gateway_-_consul-wan-federation/ValidateServiceMetadata_-_mesh-gateway (0.00s) + --- PASS: TestStructs_ValidateServiceAndNodeMetadata/should_succeed (0.00s) + --- PASS: TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateNodeMetadata (0.00s) + --- PASS: TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateServiceMetadata_-_typical (0.00s) + --- PASS: TestStructs_ValidateServiceAndNodeMetadata/should_succeed/ValidateServiceMetadata_-_mesh-gateway (0.00s) === RUN TestStructs_validateMetaPair --- PASS: TestStructs_validateMetaPair (0.00s) === RUN TestSpecificServiceRequest_CacheInfo @@ -70497,37 +70614,37 @@ --- PASS: TestServiceNode_JSON_OmitServiceTaggedAdddresses/nil (0.00s) --- PASS: TestServiceNode_JSON_OmitServiceTaggedAdddresses/empty (0.00s) === RUN TestNode_BestAddress -=== RUN TestNode_BestAddress/address === RUN TestNode_BestAddress/wan-address +=== RUN TestNode_BestAddress/address --- PASS: TestNode_BestAddress (0.00s) - --- PASS: TestNode_BestAddress/address (0.00s) --- PASS: TestNode_BestAddress/wan-address (0.00s) + --- PASS: TestNode_BestAddress/address (0.00s) === RUN TestNodeService_BestAddress +=== RUN TestNodeService_BestAddress/no-address +=== RUN TestNodeService_BestAddress/service-address === RUN TestNodeService_BestAddress/service-wan-address === RUN TestNodeService_BestAddress/service-wan-address-default-port === RUN TestNodeService_BestAddress/service-wan-address-node-lan -=== RUN TestNodeService_BestAddress/no-address -=== RUN TestNodeService_BestAddress/service-address --- PASS: TestNodeService_BestAddress (0.00s) + --- PASS: TestNodeService_BestAddress/no-address (0.00s) + --- PASS: TestNodeService_BestAddress/service-address (0.00s) --- PASS: TestNodeService_BestAddress/service-wan-address (0.00s) --- PASS: TestNodeService_BestAddress/service-wan-address-default-port (0.00s) --- PASS: TestNodeService_BestAddress/service-wan-address-node-lan (0.00s) - --- PASS: TestNodeService_BestAddress/no-address (0.00s) - --- PASS: TestNodeService_BestAddress/service-address (0.00s) === RUN TestCheckServiceNode_BestAddress -=== RUN TestCheckServiceNode_BestAddress/node-address -=== RUN TestCheckServiceNode_BestAddress/node-wan-address === RUN TestCheckServiceNode_BestAddress/service-address === RUN TestCheckServiceNode_BestAddress/service-wan-address === RUN TestCheckServiceNode_BestAddress/service-wan-address-default-port === RUN TestCheckServiceNode_BestAddress/service-wan-address-node-lan +=== RUN TestCheckServiceNode_BestAddress/node-address +=== RUN TestCheckServiceNode_BestAddress/node-wan-address --- PASS: TestCheckServiceNode_BestAddress (0.00s) - --- PASS: TestCheckServiceNode_BestAddress/node-address (0.00s) - --- PASS: TestCheckServiceNode_BestAddress/node-wan-address (0.00s) --- PASS: TestCheckServiceNode_BestAddress/service-address (0.00s) --- PASS: TestCheckServiceNode_BestAddress/service-wan-address (0.00s) --- PASS: TestCheckServiceNode_BestAddress/service-wan-address-default-port (0.00s) --- PASS: TestCheckServiceNode_BestAddress/service-wan-address-node-lan (0.00s) + --- PASS: TestCheckServiceNode_BestAddress/node-address (0.00s) + --- PASS: TestCheckServiceNode_BestAddress/node-wan-address (0.00s) === RUN TestNodeService_JSON_Marshal --- PASS: TestNodeService_JSON_Marshal (0.00s) === RUN TestServiceNode_JSON_Marshal @@ -70579,7 +70696,7 @@ === RUN TestGatewayService_IsSame --- PASS: TestGatewayService_IsSame (0.00s) PASS -ok github.com/hashicorp/consul/agent/structs 0.196s +ok github.com/hashicorp/consul/agent/structs 0.062s ? github.com/hashicorp/consul/agent/systemd [no test files] === RUN TestStore_RegularTokens === PAUSE TestStore_RegularTokens @@ -70634,675 +70751,675 @@ --- PASS: TestStore_RegularTokens/set_agent_-_config (0.00s) --- PASS: TestStore_RegularTokens/set_user_-_api (0.00s) PASS -ok github.com/hashicorp/consul/agent/token 0.032s +ok github.com/hashicorp/consul/agent/token 0.046s === RUN TestClustersFromSnapshot === RUN TestClustersFromSnapshot/envoy-1.14.5 === RUN TestClustersFromSnapshot/envoy-1.14.5/defaults -2021-10-31T23:11:56.018Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/defaults: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.690Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/defaults: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-local-app -2021-10-31T23:11:56.037Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-local-app: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.696Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-local-app: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-local-app-typed -2021-10-31T23:11:56.044Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-local-app-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.708Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-local-app-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-upstream -2021-10-31T23:11:56.049Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-upstream: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.711Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-upstream: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-default-chain -2021-10-31T23:11:56.054Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-default-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.717Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-default-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-typed -2021-10-31T23:11:56.060Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.720Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-ignores-tls -2021-10-31T23:11:56.062Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-ignores-tls: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.723Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-ignores-tls: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-timeouts -2021-10-31T23:11:56.066Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-timeouts: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.726Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-timeouts: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-limits-max-connections-only -2021-10-31T23:11:56.070Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-limits-max-connections-only: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.729Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-limits-max-connections-only: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-limits-set-to-zero -2021-10-31T23:11:56.074Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-limits-set-to-zero: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.732Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-limits-set-to-zero: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/custom-limits -2021-10-31T23:11:56.083Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-limits: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.735Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/custom-limits: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain -2021-10-31T23:11:56.091Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.738Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni -2021-10-31T23:11:56.097Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.741Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides -2021-10-31T23:11:56.101Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.744Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover -2021-10-31T23:11:56.110Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.747Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway -2021-10-31T23:11:56.114Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.749Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered -2021-10-31T23:11:56.120Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.765Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway -2021-10-31T23:11:56.123Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.769Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered -2021-10-31T23:11:56.126Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.774Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway -2021-10-31T23:11:56.130Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.783Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered -2021-10-31T23:11:56.132Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.786Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway -2021-10-31T23:11:56.137Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.798Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered -2021-10-31T23:11:56.141Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.801Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect -2021-10-31T23:11:56.155Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:56.155Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.804Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.804Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/expose-paths-local-app-paths === RUN TestClustersFromSnapshot/envoy-1.14.5/expose-paths-new-cluster-http2 === RUN TestClustersFromSnapshot/envoy-1.14.5/expose-paths-grpc-new-cluster-http1 === RUN TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway -2021-10-31T23:11:56.178Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:02.810Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-using-federation-states -2021-10-31T23:11:56.181Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-using-federation-states.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:02.813Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-using-federation-states.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-subsets -2021-10-31T23:11:56.185Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-subsets.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:02.817Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-subsets.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-ignore-extra-resolvers -2021-10-31T23:11:56.191Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-ignore-extra-resolvers.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:02.821Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-ignore-extra-resolvers.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-timeouts -2021-10-31T23:11:56.195Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-timeouts.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:02.833Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-timeouts.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-gateway -2021-10-31T23:11:56.204Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.841Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain -2021-10-31T23:11:56.222Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.853Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni -2021-10-31T23:11:56.240Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.855Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides -2021-10-31T23:11:56.243Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.857Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-failover -2021-10-31T23:11:56.251Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.859Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway -2021-10-31T23:11:56.259Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.870Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered -2021-10-31T23:11:56.267Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.872Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway -2021-10-31T23:11:56.275Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.878Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered -2021-10-31T23:11:56.283Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.888Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway -2021-10-31T23:11:56.291Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.891Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered -2021-10-31T23:11:56.299Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.893Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway -2021-10-31T23:11:56.304Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.895Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered -2021-10-31T23:11:56.307Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.907Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect -2021-10-31T23:11:56.310Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:56.310Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.921Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.921Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway -2021-10-31T23:11:56.346Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2021-10-31T23:11:56.346Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:02.926Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:02.926Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api === RUN TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets -2021-10-31T23:11:56.360Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2021-10-31T23:11:56.360Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db -2021-10-31T23:11:56.361Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2022-12-04T05:48:02.938Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2022-12-04T05:48:02.938Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:02.938Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache === RUN TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-hostname-service-subsets -2021-10-31T23:11:56.370Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-hostname-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2021-10-31T23:11:56.370Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db -2021-10-31T23:11:56.370Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2022-12-04T05:48:02.942Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:02.942Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2022-12-04T05:48:02.942Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-hostname-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api === RUN TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-ignore-extra-resolvers -2021-10-31T23:11:56.376Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-ignore-extra-resolvers.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2021-10-31T23:11:56.376Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-ignore-extra-resolvers.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:02.954Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-ignore-extra-resolvers.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2022-12-04T05:48:02.954Z [WARN] TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-ignore-extra-resolvers.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db === RUN TestClustersFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service -2021-10-31T23:11:56.382Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:56.382Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.958Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.958Z [DEBUG] TestClustersFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6 === RUN TestClustersFromSnapshot/envoy-1.13.6/defaults -2021-10-31T23:11:56.387Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/defaults: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.969Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/defaults: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-local-app -2021-10-31T23:11:56.391Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-local-app: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.983Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-local-app: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-local-app-typed -2021-10-31T23:11:56.397Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-local-app-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.988Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-local-app-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-upstream -2021-10-31T23:11:56.401Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-upstream: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.991Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-upstream: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-default-chain -2021-10-31T23:11:56.405Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-default-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.994Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-default-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-typed -2021-10-31T23:11:56.410Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.997Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-ignores-tls -2021-10-31T23:11:56.415Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-ignores-tls: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:02.999Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-ignores-tls: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-timeouts -2021-10-31T23:11:56.424Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-timeouts: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.002Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-timeouts: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-limits-max-connections-only -2021-10-31T23:11:56.436Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-limits-max-connections-only: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.005Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-limits-max-connections-only: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-limits-set-to-zero -2021-10-31T23:11:56.448Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-limits-set-to-zero: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.007Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-limits-set-to-zero: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/custom-limits -2021-10-31T23:11:56.460Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-limits: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.011Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/custom-limits: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain -2021-10-31T23:11:56.472Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.013Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni -2021-10-31T23:11:56.484Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.015Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides -2021-10-31T23:11:56.496Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.017Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover -2021-10-31T23:11:56.508Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.019Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway -2021-10-31T23:11:56.520Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.025Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered -2021-10-31T23:11:56.522Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.029Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway -2021-10-31T23:11:56.542Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.033Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered -2021-10-31T23:11:56.547Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.037Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway -2021-10-31T23:11:56.559Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.041Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered -2021-10-31T23:11:56.562Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.044Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway -2021-10-31T23:11:56.572Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.047Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered -2021-10-31T23:11:56.580Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.049Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect -2021-10-31T23:11:56.588Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:56.588Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.053Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.053Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/expose-paths-local-app-paths === RUN TestClustersFromSnapshot/envoy-1.13.6/expose-paths-new-cluster-http2 === RUN TestClustersFromSnapshot/envoy-1.13.6/expose-paths-grpc-new-cluster-http1 === RUN TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway -2021-10-31T23:11:56.623Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:03.061Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states -2021-10-31T23:11:56.631Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:03.064Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-subsets -2021-10-31T23:11:56.647Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-subsets.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:03.067Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-subsets.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-ignore-extra-resolvers -2021-10-31T23:11:56.656Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-ignore-extra-resolvers.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:03.071Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-ignore-extra-resolvers.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-timeouts -2021-10-31T23:11:56.663Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-timeouts.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:03.074Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-timeouts.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-gateway -2021-10-31T23:11:56.676Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.077Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain -2021-10-31T23:11:56.691Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.081Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni -2021-10-31T23:11:56.694Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.083Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides -2021-10-31T23:11:56.698Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.085Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover -2021-10-31T23:11:56.701Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.087Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway -2021-10-31T23:11:56.714Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.090Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway-triggered -2021-10-31T23:11:56.718Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.095Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway -2021-10-31T23:11:56.721Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.097Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered -2021-10-31T23:11:56.724Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.109Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway -2021-10-31T23:11:56.727Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.111Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway-triggered -2021-10-31T23:11:56.730Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.118Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway -2021-10-31T23:11:56.732Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.120Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered -2021-10-31T23:11:56.736Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.122Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect -2021-10-31T23:11:56.739Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:56.739Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.124Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.124Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway -2021-10-31T23:11:56.749Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2021-10-31T23:11:56.749Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:03.126Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:03.126Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api === RUN TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets -2021-10-31T23:11:56.756Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2021-10-31T23:11:56.756Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db -2021-10-31T23:11:56.756Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2022-12-04T05:48:03.129Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2022-12-04T05:48:03.129Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:03.129Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache === RUN TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-hostname-service-subsets -2021-10-31T23:11:56.761Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache -2021-10-31T23:11:56.761Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-hostname-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2021-10-31T23:11:56.762Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:03.133Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-hostname-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2022-12-04T05:48:03.133Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:03.133Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache === RUN TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-ignore-extra-resolvers -2021-10-31T23:11:56.767Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-ignore-extra-resolvers.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2021-10-31T23:11:56.767Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-ignore-extra-resolvers.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:03.142Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-ignore-extra-resolvers.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2022-12-04T05:48:03.142Z [WARN] TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-ignore-extra-resolvers.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db === RUN TestClustersFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service -2021-10-31T23:11:56.773Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:56.773Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.145Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.145Z [DEBUG] TestClustersFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7 === RUN TestClustersFromSnapshot/envoy-1.12.7/defaults -2021-10-31T23:11:56.777Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/defaults: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.147Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/defaults: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-local-app -2021-10-31T23:11:56.780Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-local-app: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.150Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-local-app: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-local-app-typed -2021-10-31T23:11:56.785Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-local-app-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.152Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-local-app-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-upstream -2021-10-31T23:11:56.793Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-upstream: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.155Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-upstream: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-default-chain -2021-10-31T23:11:56.800Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-default-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.158Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-default-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-typed -2021-10-31T23:11:56.803Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.160Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-ignores-tls -2021-10-31T23:11:56.806Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-ignores-tls: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.163Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-ignores-tls: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-timeouts -2021-10-31T23:11:56.809Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-timeouts: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.165Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-timeouts: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-limits-max-connections-only -2021-10-31T23:11:56.813Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-limits-max-connections-only: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.171Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-limits-max-connections-only: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-limits-set-to-zero -2021-10-31T23:11:56.817Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-limits-set-to-zero: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.173Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-limits-set-to-zero: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/custom-limits -2021-10-31T23:11:56.821Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-limits: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.176Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/custom-limits: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain -2021-10-31T23:11:56.825Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.178Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni -2021-10-31T23:11:56.833Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.181Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides -2021-10-31T23:11:56.840Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.183Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-failover -2021-10-31T23:11:56.844Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.185Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway -2021-10-31T23:11:56.847Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.187Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered -2021-10-31T23:11:56.857Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.189Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway -2021-10-31T23:11:56.860Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.191Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered -2021-10-31T23:11:56.868Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.196Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway -2021-10-31T23:11:56.876Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.198Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered -2021-10-31T23:11:56.888Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.200Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway -2021-10-31T23:11:56.891Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.202Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered -2021-10-31T23:11:56.904Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.204Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect -2021-10-31T23:11:56.912Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:56.912Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.206Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.206Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/expose-paths-local-app-paths === RUN TestClustersFromSnapshot/envoy-1.12.7/expose-paths-new-cluster-http2 === RUN TestClustersFromSnapshot/envoy-1.12.7/expose-paths-grpc-new-cluster-http1 === RUN TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway -2021-10-31T23:11:56.943Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:03.210Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states -2021-10-31T23:11:56.953Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:03.212Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-subsets -2021-10-31T23:11:56.969Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-subsets.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:03.214Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-subsets.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-ignore-extra-resolvers -2021-10-31T23:11:56.984Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-ignore-extra-resolvers.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:03.222Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-ignore-extra-resolvers.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-timeouts -2021-10-31T23:11:56.988Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-timeouts.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:03.225Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-timeouts.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-gateway -2021-10-31T23:11:56.997Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.227Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain -2021-10-31T23:11:57.009Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.231Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni -2021-10-31T23:11:57.015Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.232Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides -2021-10-31T23:11:57.022Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.235Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-and-failover -2021-10-31T23:11:57.029Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.236Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway -2021-10-31T23:11:57.033Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.238Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway-triggered -2021-10-31T23:11:57.036Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.240Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway -2021-10-31T23:11:57.040Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.242Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered -2021-10-31T23:11:57.043Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.244Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway -2021-10-31T23:11:57.046Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.245Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway-triggered -2021-10-31T23:11:57.054Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.247Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway -2021-10-31T23:11:57.059Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.250Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered -2021-10-31T23:11:57.062Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.252Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect -2021-10-31T23:11:57.070Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:57.070Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.254Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.254Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway -2021-10-31T23:11:57.076Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2021-10-31T23:11:57.076Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:03.255Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2022-12-04T05:48:03.255Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db === RUN TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets -2021-10-31T23:11:57.088Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2021-10-31T23:11:57.088Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db -2021-10-31T23:11:57.088Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2022-12-04T05:48:03.259Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2022-12-04T05:48:03.259Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:03.259Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache === RUN TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets -2021-10-31T23:11:57.096Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2021-10-31T23:11:57.096Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db -2021-10-31T23:11:57.096Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2022-12-04T05:48:03.262Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2022-12-04T05:48:03.262Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:03.262Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache === RUN TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-ignore-extra-resolvers -2021-10-31T23:11:57.104Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-ignore-extra-resolvers.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2021-10-31T23:11:57.104Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-ignore-extra-resolvers.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:03.265Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-ignore-extra-resolvers.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2022-12-04T05:48:03.265Z [WARN] TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-ignore-extra-resolvers.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db === RUN TestClustersFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service -2021-10-31T23:11:57.112Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:57.112Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.268Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.268Z [DEBUG] TestClustersFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2 === RUN TestClustersFromSnapshot/envoy-1.11.2/defaults -2021-10-31T23:11:57.119Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/defaults: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.270Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/defaults: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-local-app -2021-10-31T23:11:57.128Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-local-app: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.274Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-local-app: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-local-app-typed -2021-10-31T23:11:57.140Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-local-app-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.276Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-local-app-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-upstream -2021-10-31T23:11:57.146Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-upstream: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.278Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-upstream: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-default-chain -2021-10-31T23:11:57.172Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-default-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.280Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-default-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-typed -2021-10-31T23:11:57.180Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.282Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-typed: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-ignores-tls -2021-10-31T23:11:57.187Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-ignores-tls: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.284Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-ignores-tls: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-timeouts -2021-10-31T23:11:57.202Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-timeouts: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.286Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-timeouts: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-limits-max-connections-only -2021-10-31T23:11:57.212Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-limits-max-connections-only: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.289Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-limits-max-connections-only: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-limits-set-to-zero -2021-10-31T23:11:57.224Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-limits-set-to-zero: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.291Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-limits-set-to-zero: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/custom-limits -2021-10-31T23:11:57.232Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-limits: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.298Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/custom-limits: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain -2021-10-31T23:11:57.240Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.302Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni -2021-10-31T23:11:57.248Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.310Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides -2021-10-31T23:11:57.272Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.313Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover -2021-10-31T23:11:57.276Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.315Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway -2021-10-31T23:11:57.284Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.318Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered -2021-10-31T23:11:57.292Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.320Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway -2021-10-31T23:11:57.302Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.323Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered -2021-10-31T23:11:57.312Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.326Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway -2021-10-31T23:11:57.320Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.328Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered -2021-10-31T23:11:57.323Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.330Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway -2021-10-31T23:11:57.332Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.333Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered -2021-10-31T23:11:57.338Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.335Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect -2021-10-31T23:11:57.341Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:57.341Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.341Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.341Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/expose-paths-local-app-paths === RUN TestClustersFromSnapshot/envoy-1.11.2/expose-paths-new-cluster-http2 === RUN TestClustersFromSnapshot/envoy-1.11.2/expose-paths-grpc-new-cluster-http1 === RUN TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway -2021-10-31T23:11:57.383Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:03.345Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-using-federation-states -2021-10-31T23:11:57.391Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-using-federation-states.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:03.347Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-using-federation-states.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-subsets -2021-10-31T23:11:57.396Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-subsets.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:03.350Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-subsets.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-ignore-extra-resolvers -2021-10-31T23:11:57.400Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-ignore-extra-resolvers.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:03.352Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-ignore-extra-resolvers.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-timeouts -2021-10-31T23:11:57.412Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-timeouts.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway +2022-12-04T05:48:03.355Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-timeouts.mesh_gateway: upstream service does not contain any healthy instances: dc=dc6 service=mesh-gateway === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-gateway -2021-10-31T23:11:57.420Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.357Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain -2021-10-31T23:11:57.435Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.359Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni -2021-10-31T23:11:57.443Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.365Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides -2021-10-31T23:11:57.451Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.366Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides: generating cluster for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover -2021-10-31T23:11:57.460Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.368Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway -2021-10-31T23:11:57.467Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.369Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered -2021-10-31T23:11:57.487Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.370Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway -2021-10-31T23:11:57.492Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.372Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered -2021-10-31T23:11:57.504Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.373Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway -2021-10-31T23:11:57.510Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.375Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway-triggered -2021-10-31T23:11:57.531Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.376Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway -2021-10-31T23:11:57.539Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.378Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered -2021-10-31T23:11:57.547Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.379Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect -2021-10-31T23:11:57.555Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:57.555Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.381Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.381Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect: generating cluster for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway -2021-10-31T23:11:57.558Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db -2021-10-31T23:11:57.558Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2022-12-04T05:48:03.383Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2022-12-04T05:48:03.383Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db === RUN TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-no-services === RUN TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets -2021-10-31T23:11:57.575Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2021-10-31T23:11:57.575Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db -2021-10-31T23:11:57.575Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2022-12-04T05:48:03.390Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2022-12-04T05:48:03.390Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:03.391Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache === RUN TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-hostname-service-subsets -2021-10-31T23:11:57.588Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-hostname-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2021-10-31T23:11:57.588Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db -2021-10-31T23:11:57.588Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache +2022-12-04T05:48:03.395Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-hostname-service-subsets.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2022-12-04T05:48:03.396Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:03.396Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-hostname-service-subsets.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=cache === RUN TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-ignore-extra-resolvers -2021-10-31T23:11:57.600Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-ignore-extra-resolvers.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api -2021-10-31T23:11:57.600Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-ignore-extra-resolvers.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db +2022-12-04T05:48:03.400Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-ignore-extra-resolvers.terminating_gateway: service contains instances with more than one unique hostname; only "api.altdomain" be resolved by Envoy: dc=dc1 service=api +2022-12-04T05:48:03.400Z [WARN] TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-ignore-extra-resolvers.terminating_gateway: upstream service does not contain any healthy instances: dc=dc1 service=db === RUN TestClustersFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service -2021-10-31T23:11:57.604Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:57.604Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul ---- PASS: TestClustersFromSnapshot (1.62s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5 (0.40s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/defaults (0.05s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-local-app (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-local-app-typed (0.00s) +2022-12-04T05:48:03.404Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.405Z [DEBUG] TestClustersFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service: generating cluster for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +--- PASS: TestClustersFromSnapshot (0.73s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5 (0.29s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/defaults (0.02s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-local-app (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-local-app-typed (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-upstream (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-default-chain (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-typed (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-upstream-ignores-tls (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-timeouts (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-limits-max-connections-only (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-limits-set-to-zero (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-limits (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-limits-set-to-zero (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/custom-limits (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect (0.02s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/expose-paths-local-app-paths (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/expose-paths-new-cluster-http2 (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/expose-paths-grpc-new-cluster-http1 (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-using-federation-states (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-no-services (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-subsets (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-subsets (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-ignore-extra-resolvers (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/mesh-gateway-service-timeouts (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-gateway (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-gateway-no-services (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain (0.02s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-gateway-no-services (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-failover (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway (0.03s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect (0.02s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-no-services (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-hostname-service-subsets (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-ignore-extra-resolvers (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.14.5/terminating-gateway-ignore-extra-resolvers (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6 (0.39s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/defaults (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-local-app (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6 (0.19s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/defaults (0.02s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-local-app (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-local-app-typed (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-upstream (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-default-chain (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-typed (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-ignores-tls (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-timeouts (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-limits-max-connections-only (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-limits-set-to-zero (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-limits (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-upstream-ignores-tls (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-timeouts (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-limits-max-connections-only (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-limits-set-to-zero (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/custom-limits (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/expose-paths-local-app-paths (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/expose-paths-new-cluster-http2 (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/expose-paths-grpc-new-cluster-http1 (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/expose-paths-local-app-paths (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/expose-paths-new-cluster-http2 (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/expose-paths-grpc-new-cluster-http1 (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-no-services (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-subsets (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-ignore-extra-resolvers (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-timeouts (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-gateway (0.02s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-gateway-no-services (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-no-services (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-subsets (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-ignore-extra-resolvers (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/mesh-gateway-service-timeouts (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-gateway-no-services (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway-triggered (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-no-services (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-hostname-service-subsets (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-ignore-extra-resolvers (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.13.6/terminating-gateway-ignore-extra-resolvers (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7 (0.34s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7 (0.12s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/defaults (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-local-app (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-local-app-typed (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-upstream (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-default-chain (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-local-app-typed (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-upstream (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-default-chain (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-typed (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-upstream-ignores-tls (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-timeouts (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-limits-max-connections-only (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-limits-set-to-zero (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/custom-limits (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-failover (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/expose-paths-local-app-paths (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/expose-paths-new-cluster-http2 (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/expose-paths-local-app-paths (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/expose-paths-new-cluster-http2 (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/expose-paths-grpc-new-cluster-http1 (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-no-services (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-subsets (0.02s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-no-services (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-subsets (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-ignore-extra-resolvers (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-timeouts (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/mesh-gateway-service-timeouts (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-gateway-no-services (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-chain-and-failover (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway-triggered (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-no-services (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-ignore-extra-resolvers (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2 (0.49s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/defaults (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-local-app (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-local-app-typed (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-upstream (0.03s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-default-chain (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-no-services (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-hostname-service-subsets (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/terminating-gateway-ignore-extra-resolvers (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2 (0.14s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/defaults (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-local-app (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-local-app-typed (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-upstream (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-default-chain (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-typed (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-ignores-tls (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-timeouts (0.02s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-limits-max-connections-only (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-upstream-ignores-tls (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-timeouts (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-limits-max-connections-only (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-limits-set-to-zero (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-limits (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni (0.02s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/custom-limits (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.02s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/expose-paths-local-app-paths (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/expose-paths-new-cluster-http2 (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/expose-paths-grpc-new-cluster-http1 (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway (0.02s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/expose-paths-local-app-paths (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/expose-paths-new-cluster-http2 (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/expose-paths-grpc-new-cluster-http1 (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-using-federation-states (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-no-services (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-subsets (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-ignore-extra-resolvers (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-timeouts (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-gateway (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-gateway-no-services (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-ignore-extra-resolvers (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/mesh-gateway-service-timeouts (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-gateway-no-services (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni (0.01s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered (0.02s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway (0.02s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway-triggered (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway (0.00s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect (0.00s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-no-services (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets (0.01s) - --- PASS: TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-hostname-service-subsets (0.01s) + --- PASS: TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-hostname-service-subsets (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/terminating-gateway-ignore-extra-resolvers (0.00s) --- PASS: TestClustersFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service (0.00s) === RUN TestParseProxyConfig @@ -71317,7 +71434,7 @@ === RUN TestParseProxyConfig/local_connect_timeout_override,_string === RUN TestParseProxyConfig/local_connect_timeout_override,_float_ === RUN TestParseProxyConfig/local_connect_timeout_override,_int_ ---- PASS: TestParseProxyConfig (0.01s) +--- PASS: TestParseProxyConfig (0.00s) --- PASS: TestParseProxyConfig/defaults_-_nil (0.00s) --- PASS: TestParseProxyConfig/defaults_-_empty (0.00s) --- PASS: TestParseProxyConfig/defaults_-_other_stuff (0.00s) @@ -71378,322 +71495,322 @@ === RUN Test_endpointsFromSnapshot === RUN Test_endpointsFromSnapshot/envoy-1.14.5 === RUN Test_endpointsFromSnapshot/envoy-1.14.5/defaults -2021-10-31T23:11:57.615Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/defaults: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.414Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/defaults: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway === RUN Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-using-federation-states === RUN Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain -2021-10-31T23:11:57.658Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.429Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni -2021-10-31T23:11:57.663Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.431Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides -2021-10-31T23:11:57.667Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.433Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover -2021-10-31T23:11:57.675Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.435Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway -2021-10-31T23:11:57.683Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.437Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered -2021-10-31T23:11:57.685Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.439Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway -2021-10-31T23:11:57.691Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.442Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered -2021-10-31T23:11:57.699Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.444Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway -2021-10-31T23:11:57.707Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.445Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered -2021-10-31T23:11:57.716Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.447Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway -2021-10-31T23:11:57.720Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.449Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered -2021-10-31T23:11:57.742Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.450Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-default-chain-and-custom-cluster -2021-10-31T23:11:57.753Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-default-chain-and-custom-cluster: generating endpoints for: cluster=myservice +2022-12-04T05:48:03.453Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-default-chain-and-custom-cluster: generating endpoints for: cluster=myservice === RUN Test_endpointsFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect -2021-10-31T23:11:57.756Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:57.756Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.454Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.454Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-service-subsets === RUN Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-default-service-subset === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-gateway -2021-10-31T23:11:57.772Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.468Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain -2021-10-31T23:11:57.777Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.472Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni -2021-10-31T23:11:57.779Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.474Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides -2021-10-31T23:11:57.781Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.475Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-and-failover -2021-10-31T23:11:57.784Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.477Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway -2021-10-31T23:11:57.786Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.479Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered -2021-10-31T23:11:57.788Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.481Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway -2021-10-31T23:11:57.791Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.483Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered -2021-10-31T23:11:57.793Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.485Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway -2021-10-31T23:11:57.802Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.487Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered -2021-10-31T23:11:57.805Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.489Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway -2021-10-31T23:11:57.809Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.490Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered -2021-10-31T23:11:57.812Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.492Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect -2021-10-31T23:11:57.814Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:57.814Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.494Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.494Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.14.5/terminating-gateway === RUN Test_endpointsFromSnapshot/envoy-1.14.5/terminating-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets === RUN Test_endpointsFromSnapshot/envoy-1.14.5/terminating-gateway-default-service-subset === RUN Test_endpointsFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service -2021-10-31T23:11:57.830Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:57.830Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.504Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.504Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6 === RUN Test_endpointsFromSnapshot/envoy-1.13.6/defaults -2021-10-31T23:11:57.833Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/defaults: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.506Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/defaults: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway === RUN Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states === RUN Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain -2021-10-31T23:11:57.843Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.519Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni -2021-10-31T23:11:57.846Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.520Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides -2021-10-31T23:11:57.848Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.522Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover -2021-10-31T23:11:57.850Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.524Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway -2021-10-31T23:11:57.853Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.526Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered -2021-10-31T23:11:57.855Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.528Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway -2021-10-31T23:11:57.857Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.530Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered -2021-10-31T23:11:57.866Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.532Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway -2021-10-31T23:11:57.868Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.533Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered -2021-10-31T23:11:57.871Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.535Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway -2021-10-31T23:11:57.873Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.537Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered -2021-10-31T23:11:57.876Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.538Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-default-chain-and-custom-cluster -2021-10-31T23:11:57.878Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-default-chain-and-custom-cluster: generating endpoints for: cluster=myservice +2022-12-04T05:48:03.540Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-default-chain-and-custom-cluster: generating endpoints for: cluster=myservice === RUN Test_endpointsFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect -2021-10-31T23:11:57.880Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:57.880Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.542Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.542Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway-service-subsets === RUN Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway-default-service-subset === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-gateway -2021-10-31T23:11:57.893Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.554Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain -2021-10-31T23:11:57.898Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.556Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni -2021-10-31T23:11:57.905Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.557Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides -2021-10-31T23:11:57.908Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.559Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover -2021-10-31T23:11:57.909Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.560Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway -2021-10-31T23:11:57.919Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.562Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway-triggered -2021-10-31T23:11:57.923Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.563Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway -2021-10-31T23:11:57.925Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.564Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered -2021-10-31T23:11:57.928Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.565Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway -2021-10-31T23:11:57.931Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.567Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway-triggered -2021-10-31T23:11:57.933Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.568Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway -2021-10-31T23:11:57.936Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.570Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered -2021-10-31T23:11:57.938Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.571Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect -2021-10-31T23:11:57.940Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:57.940Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.572Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.572Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.13.6/terminating-gateway === RUN Test_endpointsFromSnapshot/envoy-1.13.6/terminating-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets === RUN Test_endpointsFromSnapshot/envoy-1.13.6/terminating-gateway-default-service-subset === RUN Test_endpointsFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service -2021-10-31T23:11:57.952Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:57.952Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.583Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.583Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7 === RUN Test_endpointsFromSnapshot/envoy-1.12.7/defaults -2021-10-31T23:11:57.954Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/defaults: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.586Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/defaults: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway === RUN Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states === RUN Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain -2021-10-31T23:11:57.979Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.595Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni -2021-10-31T23:11:57.987Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.597Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides -2021-10-31T23:11:57.990Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.599Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-failover -2021-10-31T23:11:57.992Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.601Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway -2021-10-31T23:11:57.994Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.603Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered -2021-10-31T23:11:57.996Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.605Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway -2021-10-31T23:11:58.002Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.606Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered -2021-10-31T23:11:58.007Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.608Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway -2021-10-31T23:11:58.009Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.610Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered -2021-10-31T23:11:58.015Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.612Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway -2021-10-31T23:11:58.017Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.614Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered -2021-10-31T23:11:58.023Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.615Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-default-chain-and-custom-cluster -2021-10-31T23:11:58.025Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-default-chain-and-custom-cluster: generating endpoints for: cluster=myservice +2022-12-04T05:48:03.617Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-default-chain-and-custom-cluster: generating endpoints for: cluster=myservice === RUN Test_endpointsFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect -2021-10-31T23:11:58.033Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:58.033Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.619Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.619Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-service-subsets === RUN Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-default-service-subset === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-gateway -2021-10-31T23:11:58.071Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.631Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain -2021-10-31T23:11:58.077Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.633Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni -2021-10-31T23:11:58.079Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.634Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides -2021-10-31T23:11:58.080Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.636Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-and-failover -2021-10-31T23:11:58.088Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.637Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway -2021-10-31T23:11:58.092Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.638Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway-triggered -2021-10-31T23:11:58.096Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.640Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway -2021-10-31T23:11:58.101Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.641Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered -2021-10-31T23:11:58.104Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.642Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway -2021-10-31T23:11:58.108Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.644Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway-triggered -2021-10-31T23:11:58.110Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.645Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway -2021-10-31T23:11:58.112Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.646Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered -2021-10-31T23:11:58.119Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.648Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect -2021-10-31T23:11:58.123Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:58.123Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.649Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.649Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway === RUN Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets === RUN Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway-default-service-subset === RUN Test_endpointsFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service -2021-10-31T23:11:58.141Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:58.142Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.660Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.660Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2 === RUN Test_endpointsFromSnapshot/envoy-1.11.2/defaults -2021-10-31T23:11:58.145Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/defaults: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.661Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/defaults: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway === RUN Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-using-federation-states === RUN Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain -2021-10-31T23:11:58.173Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.667Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni -2021-10-31T23:11:58.175Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.668Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides -2021-10-31T23:11:58.179Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.670Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover -2021-10-31T23:11:58.187Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.671Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway -2021-10-31T23:11:58.192Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.673Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered -2021-10-31T23:11:58.194Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.674Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway -2021-10-31T23:11:58.195Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.675Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered -2021-10-31T23:11:58.197Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.676Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway -2021-10-31T23:11:58.208Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.678Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered -2021-10-31T23:11:58.210Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.679Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway -2021-10-31T23:11:58.212Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.680Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered -2021-10-31T23:11:58.218Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.686Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-default-chain-and-custom-cluster -2021-10-31T23:11:58.228Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-default-chain-and-custom-cluster: generating endpoints for: cluster=myservice +2022-12-04T05:48:03.687Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-default-chain-and-custom-cluster: generating endpoints for: cluster=myservice === RUN Test_endpointsFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect -2021-10-31T23:11:58.230Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:58.230Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.689Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.689Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-service-subsets === RUN Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-default-service-subset === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-gateway -2021-10-31T23:11:58.274Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.696Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain -2021-10-31T23:11:58.278Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.699Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni -2021-10-31T23:11:58.283Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.700Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides -2021-10-31T23:11:58.285Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.701Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides: generating endpoints for: cluster=a236e964~db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover -2021-10-31T23:11:58.287Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.702Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway -2021-10-31T23:11:58.289Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.705Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered -2021-10-31T23:11:58.295Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.709Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway -2021-10-31T23:11:58.297Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.710Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered -2021-10-31T23:11:58.315Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.711Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway -2021-10-31T23:11:58.323Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.712Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway-triggered -2021-10-31T23:11:58.325Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.714Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway -2021-10-31T23:11:58.335Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.715Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered -2021-10-31T23:11:58.337Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.716Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect -2021-10-31T23:11:58.351Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:58.351Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.718Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v1.db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.718Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect: generating endpoints for: cluster=v2.db.default.dc2.internal.11111111-2222-3333-4444-555555555555.consul === RUN Test_endpointsFromSnapshot/envoy-1.11.2/terminating-gateway === RUN Test_endpointsFromSnapshot/envoy-1.11.2/terminating-gateway-no-services === RUN Test_endpointsFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets === RUN Test_endpointsFromSnapshot/envoy-1.11.2/terminating-gateway-default-service-subset === RUN Test_endpointsFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service -2021-10-31T23:11:58.379Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:11:58.379Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul ---- PASS: Test_endpointsFromSnapshot (0.77s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5 (0.22s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/defaults (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway (0.02s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-using-federation-states (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-no-services (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain (0.01s) +2022-12-04T05:48:03.726Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:03.726Z [DEBUG] Test_endpointsFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service: generating endpoints for: cluster=bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +--- PASS: Test_endpointsFromSnapshot (0.31s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5 (0.09s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/defaults (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-using-federation-states (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-no-services (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-failover (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.02s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-default-chain-and-custom-cluster (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-service-subsets (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-default-service-subset (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/connect-proxy-with-default-chain-and-custom-cluster (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-service-subsets (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/mesh-gateway-default-service-subset (0.01s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-gateway-no-services (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-chain (0.00s) @@ -71704,7 +71821,7 @@ --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) @@ -71714,7 +71831,7 @@ --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/terminating-gateway-default-service-subset (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.14.5/ingress-multiple-listeners-duplicate-service (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6 (0.12s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6 (0.08s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/defaults (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states (0.00s) @@ -71726,7 +71843,7 @@ --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.00s) @@ -71740,7 +71857,7 @@ --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-chain-and-failover (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-double-failover-through-remote-gateway (0.00s) @@ -71755,87 +71872,87 @@ --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/terminating-gateway-default-service-subset (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.13.6/ingress-multiple-listeners-duplicate-service (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7 (0.19s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7 (0.08s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/defaults (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway (0.02s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-no-services (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-failover (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-default-chain-and-custom-cluster (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-service-subsets (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-default-service-subset (0.02s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/connect-proxy-with-default-chain-and-custom-cluster (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-service-subsets (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/mesh-gateway-default-service-subset (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-gateway-no-services (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-chain-and-failover (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway-no-services (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets (0.01s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/terminating-gateway-default-service-subset (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.12.7/ingress-multiple-listeners-duplicate-service (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2 (0.24s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2 (0.07s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/defaults (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-using-federation-states (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-using-federation-states (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-no-services (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-failover (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/connect-proxy-with-default-chain-and-custom-cluster (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-service-subsets (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-default-service-subset (0.01s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-gateway (0.02s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-service-subsets (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/mesh-gateway-default-service-subset (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-gateway-no-services (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-chain-and-failover (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway-triggered (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway (0.02s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway (0.00s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway-triggered (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-double-failover-through-local-gateway-triggered (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/terminating-gateway (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/terminating-gateway (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/terminating-gateway-no-services (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets (0.00s) - --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/terminating-gateway-default-service-subset (0.01s) + --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/terminating-gateway-default-service-subset (0.00s) --- PASS: Test_endpointsFromSnapshot/envoy-1.11.2/ingress-multiple-listeners-duplicate-service (0.00s) === RUN TestDetermineEnvoyVersionFromNode === RUN TestDetermineEnvoyVersionFromNode/empty @@ -71885,7 +72002,7 @@ === RUN TestListenersFromSnapshot/envoy-1.14.5/custom-upstream === RUN TestListenersFromSnapshot/envoy-1.14.5/custom-upstream-typed === RUN TestListenersFromSnapshot/envoy-1.14.5/custom-upstream-typed-ignored-with-disco-chain -2021-10-31T23:11:58.543Z [WARN] TestListenersFromSnapshot/envoy-1.14.5/custom-upstream-typed-ignored-with-disco-chain: ignoring escape hatch setting because already configured for: discovery chain=db upstream=db config=envoy_listener_json +2022-12-04T05:48:03.768Z [WARN] TestListenersFromSnapshot/envoy-1.14.5/custom-upstream-typed-ignored-with-disco-chain: ignoring escape hatch setting because already configured for: discovery chain=db upstream=db config=envoy_listener_json === RUN TestListenersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect === RUN TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain === RUN TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-http-chain @@ -71916,7 +72033,7 @@ === RUN TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets === RUN TestListenersFromSnapshot/envoy-1.14.5/ingress-http-multiple-services === RUN TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-no-api-cert -2021-10-31T23:11:58.824Z [ERROR] TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-no-api-cert.terminating_gateway: no client certificate available for linked service, skipping filter chain creation: service=api error= +2022-12-04T05:48:03.855Z [ERROR] TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-no-api-cert.terminating_gateway: no client certificate available for linked service, skipping filter chain creation: service=api error= === RUN TestListenersFromSnapshot/envoy-1.14.5/ingress-with-tls-listener === RUN TestListenersFromSnapshot/envoy-1.13.6 === RUN TestListenersFromSnapshot/envoy-1.13.6/defaults @@ -71931,7 +72048,7 @@ === RUN TestListenersFromSnapshot/envoy-1.13.6/custom-upstream === RUN TestListenersFromSnapshot/envoy-1.13.6/custom-upstream-typed === RUN TestListenersFromSnapshot/envoy-1.13.6/custom-upstream-typed-ignored-with-disco-chain -2021-10-31T23:11:58.958Z [WARN] TestListenersFromSnapshot/envoy-1.13.6/custom-upstream-typed-ignored-with-disco-chain: ignoring escape hatch setting because already configured for: discovery chain=db upstream=db config=envoy_listener_json +2022-12-04T05:48:03.897Z [WARN] TestListenersFromSnapshot/envoy-1.13.6/custom-upstream-typed-ignored-with-disco-chain: ignoring escape hatch setting because already configured for: discovery chain=db upstream=db config=envoy_listener_json === RUN TestListenersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect === RUN TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain === RUN TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-http-chain @@ -71962,7 +72079,7 @@ === RUN TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets === RUN TestListenersFromSnapshot/envoy-1.13.6/ingress-http-multiple-services === RUN TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-no-api-cert -2021-10-31T23:11:59.199Z [ERROR] TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-no-api-cert.terminating_gateway: no client certificate available for linked service, skipping filter chain creation: service=api error= +2022-12-04T05:48:03.996Z [ERROR] TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-no-api-cert.terminating_gateway: no client certificate available for linked service, skipping filter chain creation: service=api error= === RUN TestListenersFromSnapshot/envoy-1.13.6/ingress-with-tls-listener === RUN TestListenersFromSnapshot/envoy-1.12.7 === RUN TestListenersFromSnapshot/envoy-1.12.7/defaults @@ -71977,7 +72094,7 @@ === RUN TestListenersFromSnapshot/envoy-1.12.7/custom-upstream === RUN TestListenersFromSnapshot/envoy-1.12.7/custom-upstream-typed === RUN TestListenersFromSnapshot/envoy-1.12.7/custom-upstream-typed-ignored-with-disco-chain -2021-10-31T23:11:59.264Z [WARN] TestListenersFromSnapshot/envoy-1.12.7/custom-upstream-typed-ignored-with-disco-chain: ignoring escape hatch setting because already configured for: discovery chain=db upstream=db config=envoy_listener_json +2022-12-04T05:48:04.037Z [WARN] TestListenersFromSnapshot/envoy-1.12.7/custom-upstream-typed-ignored-with-disco-chain: ignoring escape hatch setting because already configured for: discovery chain=db upstream=db config=envoy_listener_json === RUN TestListenersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect === RUN TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain === RUN TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-http-chain @@ -72008,7 +72125,7 @@ === RUN TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets === RUN TestListenersFromSnapshot/envoy-1.12.7/ingress-http-multiple-services === RUN TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-no-api-cert -2021-10-31T23:11:59.444Z [ERROR] TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-no-api-cert.terminating_gateway: no client certificate available for linked service, skipping filter chain creation: service=api error= +2022-12-04T05:48:04.122Z [ERROR] TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-no-api-cert.terminating_gateway: no client certificate available for linked service, skipping filter chain creation: service=api error= === RUN TestListenersFromSnapshot/envoy-1.12.7/ingress-with-tls-listener === RUN TestListenersFromSnapshot/envoy-1.11.2 === RUN TestListenersFromSnapshot/envoy-1.11.2/defaults @@ -72023,7 +72140,7 @@ === RUN TestListenersFromSnapshot/envoy-1.11.2/custom-upstream === RUN TestListenersFromSnapshot/envoy-1.11.2/custom-upstream-typed === RUN TestListenersFromSnapshot/envoy-1.11.2/custom-upstream-typed-ignored-with-disco-chain -2021-10-31T23:11:59.507Z [WARN] TestListenersFromSnapshot/envoy-1.11.2/custom-upstream-typed-ignored-with-disco-chain: ignoring escape hatch setting because already configured for: discovery chain=db upstream=db config=envoy_listener_json +2022-12-04T05:48:04.161Z [WARN] TestListenersFromSnapshot/envoy-1.11.2/custom-upstream-typed-ignored-with-disco-chain: ignoring escape hatch setting because already configured for: discovery chain=db upstream=db config=envoy_listener_json === RUN TestListenersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect === RUN TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain === RUN TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-http-chain @@ -72054,38 +72171,38 @@ === RUN TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets === RUN TestListenersFromSnapshot/envoy-1.11.2/ingress-http-multiple-services === RUN TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway-no-api-cert -2021-10-31T23:11:59.684Z [ERROR] TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway-no-api-cert.terminating_gateway: no client certificate available for linked service, skipping filter chain creation: service=api error= +2022-12-04T05:48:04.243Z [ERROR] TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway-no-api-cert.terminating_gateway: no client certificate available for linked service, skipping filter chain creation: service=api error= === RUN TestListenersFromSnapshot/envoy-1.11.2/ingress-with-tls-listener ---- PASS: TestListenersFromSnapshot (1.32s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5 (0.47s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/defaults (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/listener-bind-address (0.02s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/listener-bind-port (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/listener-bind-address-port (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/http-public-listener (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/http-upstream (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-public-listener (0.01s) +--- PASS: TestListenersFromSnapshot (0.52s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5 (0.13s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/defaults (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/listener-bind-address (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/listener-bind-port (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/listener-bind-address-port (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/http-public-listener (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/http-upstream (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-public-listener (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-public-listener-typed (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-public-listener-ignores-tls (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-upstream (0.03s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-upstream (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-upstream-typed (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-upstream-typed-ignored-with-disco-chain (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-http-chain (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-http2-chain (0.02s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-grpc-chain (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/custom-upstream-typed-ignored-with-disco-chain (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/splitter-with-resolver-redirect (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-http-chain (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-http2-chain (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-grpc-chain (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-and-overrides (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/expose-paths-local-app-paths (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/expose-paths-new-cluster-http2 (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/mesh-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/mesh-gateway-using-federation-states (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/mesh-gateway-no-services (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/mesh-gateway-tagged-addresses (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/mesh-gateway-custom-addresses (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/mesh-gateway-custom-addresses (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-gateway-bind-addrs (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-gateway-no-services (0.00s) @@ -72093,22 +72210,22 @@ --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-with-tcp-chain-failover-through-local-gateway (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect (0.02s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway (0.02s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-no-services (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-custom-and-tagged-addresses (0.03s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets (0.04s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-http-multiple-services (0.02s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-no-api-cert (0.02s) - --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-with-tls-listener (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6 (0.35s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/defaults (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/listener-bind-address (0.02s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/listener-bind-port (0.02s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/listener-bind-address-port (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/http-public-listener (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/http-upstream (0.02s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/custom-public-listener (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-no-services (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-custom-and-tagged-addresses (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-service-subsets (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-http-multiple-services (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/terminating-gateway-no-api-cert (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.14.5/ingress-with-tls-listener (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6 (0.14s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/defaults (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/listener-bind-address (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/listener-bind-port (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/listener-bind-address-port (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/http-public-listener (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/http-upstream (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/custom-public-listener (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/custom-public-listener-typed (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/custom-public-listener-ignores-tls (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/custom-upstream (0.00s) @@ -72117,50 +72234,50 @@ --- PASS: TestListenersFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-http-chain (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-http2-chain (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-grpc-chain (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-http2-chain (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-grpc-chain (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-overrides (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/expose-paths-local-app-paths (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/expose-paths-new-cluster-http2 (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/expose-paths-new-cluster-http2 (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/mesh-gateway (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/mesh-gateway-no-services (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/mesh-gateway-tagged-addresses (0.02s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/mesh-gateway-custom-addresses (0.02s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/mesh-gateway-using-federation-states (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/mesh-gateway-no-services (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/mesh-gateway-tagged-addresses (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/mesh-gateway-custom-addresses (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-gateway-bind-addrs (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-gateway-no-services (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-no-services (0.02s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-custom-and-tagged-addresses (0.02s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-gateway-no-services (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-with-chain-and-overrides (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-with-tcp-chain-failover-through-local-gateway (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-no-services (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-custom-and-tagged-addresses (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-service-subsets (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-http-multiple-services (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-no-api-cert (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.13.6/terminating-gateway-no-api-cert (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.13.6/ingress-with-tls-listener (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7 (0.24s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/defaults (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7 (0.13s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/defaults (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/listener-bind-address (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/listener-bind-port (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/listener-bind-address-port (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/http-public-listener (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/http-upstream (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/http-public-listener (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/http-upstream (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-public-listener (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-public-listener-typed (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-public-listener-ignores-tls (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-upstream (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-upstream-typed (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-upstream-typed-ignored-with-disco-chain (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/custom-upstream-typed-ignored-with-disco-chain (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-tcp-chain (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-http-chain (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-http-chain (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-http2-chain (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-grpc-chain (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni (0.00s) @@ -72170,10 +72287,10 @@ --- PASS: TestListenersFromSnapshot/envoy-1.12.7/expose-paths-local-app-paths (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/expose-paths-new-cluster-http2 (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway-no-services (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway-tagged-addresses (0.02s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway-custom-addresses (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway-using-federation-states (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway-no-services (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway-tagged-addresses (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/mesh-gateway-custom-addresses (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-gateway-bind-addrs (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-gateway-no-services (0.00s) @@ -72182,14 +72299,14 @@ --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-no-services (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-custom-and-tagged-addresses (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-service-subsets (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-http-multiple-services (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/terminating-gateway-no-api-cert (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.12.7/ingress-with-tls-listener (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2 (0.25s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2 (0.12s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/defaults (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/listener-bind-address (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/listener-bind-port (0.00s) @@ -72197,22 +72314,22 @@ --- PASS: TestListenersFromSnapshot/envoy-1.11.2/http-public-listener (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/http-upstream (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-public-listener (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-public-listener-typed (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-public-listener-ignores-tls (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-upstream (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-public-listener-typed (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-public-listener-ignores-tls (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-upstream (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-upstream-typed (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/custom-upstream-typed-ignored-with-disco-chain (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-http-chain (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-http2-chain (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-grpc-chain (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-grpc-chain (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-remote-gateway (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/expose-paths-local-app-paths (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/expose-paths-new-cluster-http2 (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/connect-proxy-with-tcp-chain-failover-through-local-gateway (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/expose-paths-local-app-paths (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/expose-paths-new-cluster-http2 (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/mesh-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/mesh-gateway-using-federation-states (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/mesh-gateway-no-services (0.00s) @@ -72222,16 +72339,16 @@ --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-gateway-bind-addrs (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-gateway-no-services (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides (0.01s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-remote-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-with-tcp-chain-failover-through-local-gateway (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect (0.00s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway (0.02s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway-no-services (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway-custom-and-tagged-addresses (0.03s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway-no-services (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway-custom-and-tagged-addresses (0.01s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway-service-subsets (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-http-multiple-services (0.01s) - --- PASS: TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway-no-api-cert (0.02s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-http-multiple-services (0.00s) + --- PASS: TestListenersFromSnapshot/envoy-1.11.2/terminating-gateway-no-api-cert (0.00s) --- PASS: TestListenersFromSnapshot/envoy-1.11.2/ingress-with-tls-listener (0.00s) === RUN TestRoutesFromSnapshot === RUN TestRoutesFromSnapshot/envoy-1.14.5 @@ -72306,8 +72423,8 @@ === RUN TestRoutesFromSnapshot/envoy-1.11.2/ingress-with-grpc-router === RUN TestRoutesFromSnapshot/envoy-1.11.2/ingress-with-chain-and-router === RUN TestRoutesFromSnapshot/envoy-1.11.2/ingress-http-multiple-services ---- PASS: TestRoutesFromSnapshot (0.29s) - --- PASS: TestRoutesFromSnapshot/envoy-1.14.5 (0.07s) +--- PASS: TestRoutesFromSnapshot (0.14s) + --- PASS: TestRoutesFromSnapshot/envoy-1.14.5 (0.04s) --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/defaults-no-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/connect-proxy-with-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/connect-proxy-with-chain-external-sni (0.00s) @@ -72321,11 +72438,11 @@ --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/ingress-with-chain-external-sni (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/ingress-with-chain-and-overrides (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/ingress-splitter-with-resolver-redirect (0.00s) - --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/ingress-with-chain-and-splitter (0.01s) - --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/ingress-with-grpc-router (0.01s) - --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/ingress-with-chain-and-router (0.02s) + --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/ingress-with-chain-and-splitter (0.00s) + --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/ingress-with-grpc-router (0.00s) + --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/ingress-with-chain-and-router (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.14.5/ingress-http-multiple-services (0.00s) - --- PASS: TestRoutesFromSnapshot/envoy-1.13.6 (0.05s) + --- PASS: TestRoutesFromSnapshot/envoy-1.13.6 (0.04s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/defaults-no-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/connect-proxy-with-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-external-sni (0.00s) @@ -72333,7 +72450,7 @@ --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/splitter-with-resolver-redirect (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-splitter (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/connect-proxy-with-grpc-router (0.00s) - --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-router (0.01s) + --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/connect-proxy-with-chain-and-router (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-defaults-no-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-with-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-with-chain-external-sni (0.00s) @@ -72341,78 +72458,78 @@ --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-splitter-with-resolver-redirect (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-with-chain-and-splitter (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-with-grpc-router (0.00s) - --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-with-chain-and-router (0.01s) + --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-with-chain-and-router (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.13.6/ingress-http-multiple-services (0.00s) - --- PASS: TestRoutesFromSnapshot/envoy-1.12.7 (0.09s) + --- PASS: TestRoutesFromSnapshot/envoy-1.12.7 (0.03s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/defaults-no-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/connect-proxy-with-chain (0.00s) - --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni (0.01s) + --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-external-sni (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-overrides (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/splitter-with-resolver-redirect (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-splitter (0.00s) - --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/connect-proxy-with-grpc-router (0.01s) - --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-router (0.01s) - --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-defaults-no-chain (0.01s) + --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/connect-proxy-with-grpc-router (0.00s) + --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/connect-proxy-with-chain-and-router (0.00s) + --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-defaults-no-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-with-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-with-chain-external-sni (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-with-chain-and-overrides (0.00s) - --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect (0.01s) + --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-splitter-with-resolver-redirect (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-with-chain-and-splitter (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-with-grpc-router (0.00s) - --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-with-chain-and-router (0.02s) - --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-http-multiple-services (0.01s) - --- PASS: TestRoutesFromSnapshot/envoy-1.11.2 (0.08s) + --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-with-chain-and-router (0.00s) + --- PASS: TestRoutesFromSnapshot/envoy-1.12.7/ingress-http-multiple-services (0.00s) + --- PASS: TestRoutesFromSnapshot/envoy-1.11.2 (0.03s) --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/defaults-no-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/connect-proxy-with-chain (0.00s) - --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni (0.01s) - --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides (0.01s) + --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-external-sni (0.00s) + --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-overrides (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/splitter-with-resolver-redirect (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-splitter (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/connect-proxy-with-grpc-router (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/connect-proxy-with-chain-and-router (0.01s) - --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/ingress-defaults-no-chain (0.01s) + --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/ingress-defaults-no-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/ingress-with-chain (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/ingress-with-chain-external-sni (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/ingress-with-chain-and-overrides (0.00s) - --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect (0.01s) + --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/ingress-splitter-with-resolver-redirect (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/ingress-with-chain-and-splitter (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/ingress-with-grpc-router (0.00s) --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/ingress-with-chain-and-router (0.00s) - --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/ingress-http-multiple-services (0.01s) + --- PASS: TestRoutesFromSnapshot/envoy-1.11.2/ingress-http-multiple-services (0.00s) === RUN TestServer_StreamAggregatedResources_BasicProtocol -2021-10-31T23:12:00.011Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:12:00.012Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:12:00.038Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:12:00.039Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:12:00.055Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:12:00.055Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:04.395Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:04.396Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:04.419Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:04.419Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:04.443Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:04.443Z [DEBUG] TestServer_StreamAggregatedResources_BasicProtocol.envoy: generating endpoints for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul --- PASS: TestServer_StreamAggregatedResources_BasicProtocol (0.06s) === RUN TestServer_StreamAggregatedResources_ACLEnforcement server_test.go:396: DM-skipped --- SKIP: TestServer_StreamAggregatedResources_ACLEnforcement (0.00s) === RUN TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedDuringDiscoveryRequest -2021-10-31T23:12:00.070Z [DEBUG] TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedDuringDiscoveryRequest.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:12:00.071Z [DEBUG] TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedDuringDiscoveryRequest.envoy: Error handling ADS stream: error="rpc error: code = Unauthenticated desc = unauthenticated: ACL not found" +2022-12-04T05:48:04.458Z [DEBUG] TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedDuringDiscoveryRequest.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:04.459Z [DEBUG] TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedDuringDiscoveryRequest.envoy: Error handling ADS stream: error="rpc error: code = Unauthenticated desc = unauthenticated: ACL not found" --- PASS: TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedDuringDiscoveryRequest (0.01s) === RUN TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedInBackground -2021-10-31T23:12:00.089Z [DEBUG] TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedInBackground.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul -2021-10-31T23:12:00.193Z [DEBUG] TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedInBackground.envoy: Error handling ADS stream: error="rpc error: code = Unauthenticated desc = unauthenticated: ACL not found" +2022-12-04T05:48:04.471Z [DEBUG] TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedInBackground.envoy: generating cluster for: cluster=db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul +2022-12-04T05:48:04.575Z [DEBUG] TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedInBackground.envoy: Error handling ADS stream: error="rpc error: code = Unauthenticated desc = unauthenticated: ACL not found" --- PASS: TestServer_StreamAggregatedResources_ACLTokenDeleted_StreamTerminatedInBackground (0.12s) === RUN TestServer_Check === RUN TestServer_Check/auth_allowed -2021-10-31T23:12:00.194Z [DEBUG] TestServer_Check/auth_allowed.envoy: Connect AuthZ ALLOWED: source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db reason="default allow" +2022-12-04T05:48:04.576Z [DEBUG] TestServer_Check/auth_allowed.envoy: Connect AuthZ ALLOWED: source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db reason="default allow" === RUN TestServer_Check/auth_denied -2021-10-31T23:12:00.194Z [DEBUG] TestServer_Check/auth_denied.envoy: Connect AuthZ DENIED: source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db reason="default deny" +2022-12-04T05:48:04.576Z [DEBUG] TestServer_Check/auth_denied.envoy: Connect AuthZ DENIED: source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db reason="default deny" === RUN TestServer_Check/no_source === RUN TestServer_Check/no_dest === RUN TestServer_Check/dest_invalid_format -2021-10-31T23:12:00.194Z [DEBUG] TestServer_Check/dest_invalid_format.envoy: Connect AuthZ DENIED: bad destination URI: source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=not-a-spiffe-id +2022-12-04T05:48:04.576Z [DEBUG] TestServer_Check/dest_invalid_format.envoy: Connect AuthZ DENIED: bad destination URI: source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=not-a-spiffe-id === RUN TestServer_Check/dest_not_a_service_URI -2021-10-31T23:12:00.194Z [DEBUG] TestServer_Check/dest_not_a_service_URI.envoy: Connect AuthZ DENIED: bad destination service ID: source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=spiffe://trust-domain.consul +2022-12-04T05:48:04.576Z [DEBUG] TestServer_Check/dest_not_a_service_URI.envoy: Connect AuthZ DENIED: bad destination service ID: source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=spiffe://trust-domain.consul === RUN TestServer_Check/ACL_not_got_permission_for_authz_call -2021-10-31T23:12:00.194Z [DEBUG] TestServer_Check/ACL_not_got_permission_for_authz_call.envoy: Connect AuthZ failed ACL check: error="Permission denied" source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web dest=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db +2022-12-04T05:48:04.576Z [DEBUG] TestServer_Check/ACL_not_got_permission_for_authz_call.envoy: Connect AuthZ failed ACL check: error="Permission denied" source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web dest=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db === RUN TestServer_Check/Random_error_running_authz -2021-10-31T23:12:00.194Z [DEBUG] TestServer_Check/Random_error_running_authz.envoy: Connect AuthZ failed: error="gremlin attack" source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db +2022-12-04T05:48:04.576Z [DEBUG] TestServer_Check/Random_error_running_authz.envoy: Connect AuthZ failed: error="gremlin attack" source=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/web destination=spiffe://11111111-2222-3333-4444-555555555555.consul/ns/default/dc/dc1/svc/db --- PASS: TestServer_Check (0.00s) --- PASS: TestServer_Check/auth_allowed (0.00s) --- PASS: TestServer_Check/auth_denied (0.00s) @@ -72425,7 +72542,7 @@ === RUN TestServer_StreamAggregatedResources_IngressEmptyResponse --- PASS: TestServer_StreamAggregatedResources_IngressEmptyResponse (0.01s) PASS -ok github.com/hashicorp/consul/agent/xds 4.361s +ok github.com/hashicorp/consul/agent/xds 2.048s ? github.com/hashicorp/consul/agent/xds/proxysupport [no test files] ? github.com/hashicorp/consul/command [no test files] ? github.com/hashicorp/consul/command/acl [no test files] @@ -72434,93 +72551,95 @@ === RUN TestAgentTokensCommand === PAUSE TestAgentTokensCommand === CONT TestAgentTokensCommand_noTabs +--- PASS: TestAgentTokensCommand_noTabs (0.00s) === CONT TestAgentTokensCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestAgentTokensCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:11:58.832Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:11:58.853Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -11:58.855 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:afe758b3-60cf-6f15-1eb3-049b9611aeda Address:127.0.0.1:31447}]" -11:58.855 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= -11:58.856 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-afe758b3-60cf-6f15-1eb3-049b9611aeda.dc1 127.0.0.1 -11:58.856 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-afe758b3-60cf-6f15-1eb3-049b9611aeda 127.0.0.1 -2021-10-31T23:11:58.856Z [INFO] agent.router: Initializing LAN area manager -11:58.857 [INFO] TestAgent.server: Adding LAN server: server="Node-afe758b3-60cf-6f15-1eb3-049b9611aeda (Addr: tcp/127.0.0.1:31447) (DC: dc1)" -11:58.857 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-afe758b3-60cf-6f15-1eb3-049b9611aeda.dc1 area=wan -11:58.857 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=udp -11:58.857 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=tcp -11:58.857 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31443 network=tcp -11:58.857 [INFO] TestAgent: started state syncer -11:58.857 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31448 network=tcp -11:58.898 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -11:58.898 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 -11:58.898 [DEBUG] TestAgent.server.raft: votes: needed=1 -11:58.898 [DEBUG] TestAgent.server.raft: vote granted: from=afe758b3-60cf-6f15-1eb3-049b9611aeda term=2 tally=1 -11:58.898 [INFO] TestAgent.server.raft: election won: tally=1 -11:58.898 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" -11:58.899 [INFO] TestAgent.server: cluster leadership acquired -11:58.899 [INFO] TestAgent.server: initializing acls -11:58.899 [INFO] TestAgent.server: Created ACL 'global-management' policy -11:58.899 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -11:58.900 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -11:58.900 [INFO] TestAgent.server: Created ACL anonymous token from configuration -11:58.900 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -11:58.900 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -11:58.900 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-afe758b3-60cf-6f15-1eb3-049b9611aeda -11:58.900 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-afe758b3-60cf-6f15-1eb3-049b9611aeda.dc1 -11:58.901 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -11:58.901 [INFO] TestAgent.server: New leader elected: payload=Node-afe758b3-60cf-6f15-1eb3-049b9611aeda -11:58.901 [INFO] TestAgent.server: Updating LAN server: server="Node-afe758b3-60cf-6f15-1eb3-049b9611aeda (Addr: tcp/127.0.0.1:31447) (DC: dc1)" -11:58.902 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-afe758b3-60cf-6f15-1eb3-049b9611aeda.dc1 area=wan -11:58.922 [INFO] TestAgent: Synced node info -11:58.930 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -11:58.930 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -11:58.930 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -11:58.930 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -11:58.930 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -11:58.930 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-afe758b3-60cf-6f15-1eb3-049b9611aeda -11:58.930 [INFO] TestAgent.server: member joined, marking health alive: member=Node-afe758b3-60cf-6f15-1eb3-049b9611aeda -11:58.931 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-afe758b3-60cf-6f15-1eb3-049b9611aeda -11:58.932 [INFO] TestAgent.server: federation state anti-entropy synced -11:59.232 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:52918 latency=4.081573ms -11:59.239 [INFO] TestAgent: Updated agent's ACL token: token=default -11:59.239 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/default from=127.0.0.1:52920 latency=75.558µs -11:59.239 [INFO] TestAgent: Updated agent's ACL token: token=agent -11:59.239 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/agent from=127.0.0.1:52922 latency=37.187µs -11:59.240 [INFO] TestAgent: Updated agent's ACL token: token=agent_master -11:59.240 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/agent_master from=127.0.0.1:52924 latency=41.076µs -11:59.240 [INFO] TestAgent: Updated agent's ACL token: token=replication -11:59.240 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/replication from=127.0.0.1:52926 latency=23.357µs -11:59.240 [INFO] TestAgent: Requesting shutdown -11:59.240 [INFO] TestAgent.server: shutting down server -11:59.240 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -11:59.240 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -11:59.240 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -11:59.240 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -11:59.240 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -11:59.240 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -11:59.240 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -11:59.240 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -11:59.240 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -11:59.241 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -11:59.241 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -11:59.241 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -11:59.241 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -11:59.241 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -11:59.241 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:11:59.241Z [INFO] agent.router.manager: shutting down -2021-10-31T23:11:59.241Z [INFO] agent.router.manager: shutting down -11:59.241 [INFO] TestAgent: consul server down -11:59.241 [INFO] TestAgent: shutdown complete -11:59.241 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp -11:59.243 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp -11:59.243 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp -11:59.745 [INFO] TestAgent: Waiting for endpoints to shut down -11:59.745 [INFO] TestAgent: Endpoints down ---- PASS: TestAgentTokensCommand (0.98s) +2022-12-04T05:48:07.006Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:07.027Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:07.029 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:58ea7a45-d4a5-97c8-5509-aca12dad5f2a Address:127.0.0.1:19195}]" +48:07.030 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-58ea7a45-d4a5-97c8-5509-aca12dad5f2a.dc1 127.0.0.1 +48:07.030 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-58ea7a45-d4a5-97c8-5509-aca12dad5f2a 127.0.0.1 +2022-12-04T05:48:07.030Z [INFO] agent.router: Initializing LAN area manager +48:07.031 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp +48:07.031 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= +48:07.031 [INFO] TestAgent.server: Adding LAN server: server="Node-58ea7a45-d4a5-97c8-5509-aca12dad5f2a (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +48:07.031 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-58ea7a45-d4a5-97c8-5509-aca12dad5f2a.dc1 area=wan +48:07.031 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp +48:07.032 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp +48:07.032 [INFO] TestAgent: started state syncer +48:07.032 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp +48:07.068 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:07.068 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 +48:07.068 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:07.068 [DEBUG] TestAgent.server.raft: vote granted: from=58ea7a45-d4a5-97c8-5509-aca12dad5f2a term=2 tally=1 +48:07.068 [INFO] TestAgent.server.raft: election won: tally=1 +48:07.068 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" +48:07.068 [INFO] TestAgent.server: cluster leadership acquired +48:07.068 [INFO] TestAgent.server: initializing acls +48:07.069 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:07.069 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:07.069 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:07.070 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:07.070 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:07.070 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:07.070 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-58ea7a45-d4a5-97c8-5509-aca12dad5f2a +48:07.070 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-58ea7a45-d4a5-97c8-5509-aca12dad5f2a.dc1 +48:07.070 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:07.069 [INFO] TestAgent.server: New leader elected: payload=Node-58ea7a45-d4a5-97c8-5509-aca12dad5f2a +48:07.084 [INFO] TestAgent.server: Updating LAN server: server="Node-58ea7a45-d4a5-97c8-5509-aca12dad5f2a (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +48:07.076 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-58ea7a45-d4a5-97c8-5509-aca12dad5f2a.dc1 area=wan +48:07.100 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:07.100 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:07.100 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:07.100 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:07.100 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:07.100 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-58ea7a45-d4a5-97c8-5509-aca12dad5f2a +48:07.100 [INFO] TestAgent.server: member joined, marking health alive: member=Node-58ea7a45-d4a5-97c8-5509-aca12dad5f2a +48:07.108 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-58ea7a45-d4a5-97c8-5509-aca12dad5f2a +48:07.108 [INFO] TestAgent.server: federation state anti-entropy synced +48:07.246 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:07.246 [INFO] TestAgent: Synced node info +48:07.246 [DEBUG] TestAgent: Node info in sync +48:07.261 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:40344 latency=2.355021ms +48:07.263 [INFO] TestAgent: Updated agent's ACL token: token=default +48:07.263 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/default from=127.0.0.1:40346 latency=102.818µs +48:07.265 [INFO] TestAgent: Updated agent's ACL token: token=agent +48:07.265 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/agent from=127.0.0.1:40348 latency=76.233µs +48:07.270 [INFO] TestAgent: Updated agent's ACL token: token=agent_master +48:07.270 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/agent_master from=127.0.0.1:40350 latency=108.121µs +48:07.271 [INFO] TestAgent: Updated agent's ACL token: token=replication +48:07.271 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/token/replication from=127.0.0.1:40352 latency=53.058µs +48:07.272 [INFO] TestAgent: Requesting shutdown +48:07.272 [INFO] TestAgent.server: shutting down server +48:07.272 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:07.272 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:07.272 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:07.272 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:07.272 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:07.272 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:07.272 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:07.272 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:07.272 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:07.272 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:07.272 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:07.272 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:07.272 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:07.272 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:07.272 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:07.272Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:07.272Z [INFO] agent.router.manager: shutting down +48:07.272 [INFO] TestAgent: consul server down +48:07.272 [INFO] TestAgent: shutdown complete +48:07.272 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp +48:07.273 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp +48:07.273 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp +48:07.773 [INFO] TestAgent: Waiting for endpoints to shut down +48:07.773 [INFO] TestAgent: Endpoints down +--- PASS: TestAgentTokensCommand (0.82s) PASS -ok github.com/hashicorp/consul/command/acl/agenttokens 1.102s +ok github.com/hashicorp/consul/command/acl/agenttokens 0.929s ? github.com/hashicorp/consul/command/acl/authmethod [no test files] === RUN TestAuthMethodCreateCommand_noTabs === PAUSE TestAuthMethodCreateCommand_noTabs @@ -72536,362 +72655,369 @@ --- PASS: TestAuthMethodCreateCommand_noTabs (0.00s) === CONT TestAuthMethodCreateCommand_config [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +=== CONT TestAuthMethodCreateCommand_JSON === CONT TestAuthMethodCreateCommand === CONT TestAuthMethodCreateCommand_k8s [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -=== CONT TestAuthMethodCreateCommand_JSON -=== RUN TestAuthMethodCreateCommand_JSON/type_required -=== RUN TestAuthMethodCreateCommand_JSON/create_testing -=== RUN TestAuthMethodCreateCommand_JSON/create_testing_with_max_token_ttl -=== RUN TestAuthMethodCreateCommand_JSON/create_testing_with_token_type_global === RUN TestAuthMethodCreateCommand/type_required === RUN TestAuthMethodCreateCommand/name_required === RUN TestAuthMethodCreateCommand/invalid_type -=== RUN TestAuthMethodCreateCommand_k8s/k8s_host_required === RUN TestAuthMethodCreateCommand/create_testing -=== RUN TestAuthMethodCreateCommand_k8s/k8s_ca_cert_required -=== RUN TestAuthMethodCreateCommand_k8s/k8s_jwt_required -=== RUN TestAuthMethodCreateCommand_k8s/create_k8s === RUN TestAuthMethodCreateCommand/create_testing_with_max_token_ttl -=== RUN TestAuthMethodCreateCommand_k8s/create_k8s_with_cert_file === RUN TestAuthMethodCreateCommand/create_testing_with_token_type_global +=== RUN TestAuthMethodCreateCommand_JSON/type_required +=== RUN TestAuthMethodCreateCommand_JSON/create_testing +=== RUN TestAuthMethodCreateCommand_JSON/create_testing_with_max_token_ttl +=== RUN TestAuthMethodCreateCommand_JSON/create_testing_with_token_type_global === RUN TestAuthMethodCreateCommand_config/config_file === RUN TestAuthMethodCreateCommand_config/config_std-in === RUN TestAuthMethodCreateCommand_config/config_string -2021-10-31T23:12:01.496Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:01.550Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:01.552 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:101e74f9-f8ea-ad87-8e2a-88a9c73c47d7 Address:127.0.0.1:27384}]" -12:01.553 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-101e74f9-f8ea-ad87-8e2a-88a9c73c47d7.dc1 127.0.0.1 -12:01.553 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-101e74f9-f8ea-ad87-8e2a-88a9c73c47d7 127.0.0.1 -2021-10-31T23:12:01.553Z [INFO] agent.router: Initializing LAN area manager -12:01.553 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27384 [Follower]" leader= -12:01.554 [INFO] TestAgent.server: Adding LAN server: server="Node-101e74f9-f8ea-ad87-8e2a-88a9c73c47d7 (Addr: tcp/127.0.0.1:27384) (DC: dc1)" -12:01.554 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-101e74f9-f8ea-ad87-8e2a-88a9c73c47d7.dc1 area=wan -12:01.554 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27379 network=tcp -12:01.558 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27379 network=udp -12:01.558 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27380 network=tcp -12:01.558 [INFO] TestAgent: started state syncer -12:01.558 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27385 network=tcp -12:01.604 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -12:01.624 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:01.624 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27384 [Candidate]" term=2 -12:01.624 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:01.624 [DEBUG] TestAgent.server.raft: vote granted: from=101e74f9-f8ea-ad87-8e2a-88a9c73c47d7 term=2 tally=1 -12:01.624 [INFO] TestAgent.server.raft: election won: tally=1 -12:01.624 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27384 [Leader]" -12:01.624 [INFO] TestAgent.server: cluster leadership acquired -12:01.624 [INFO] TestAgent.server: initializing acls -12:01.624 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:01.624 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:01.625 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:01.625 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:01.625 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:01.625 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:01.625 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-101e74f9-f8ea-ad87-8e2a-88a9c73c47d7 -12:01.625 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-101e74f9-f8ea-ad87-8e2a-88a9c73c47d7.dc1 -12:01.626 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:01.626 [INFO] TestAgent.server: New leader elected: payload=Node-101e74f9-f8ea-ad87-8e2a-88a9c73c47d7 -12:01.626 [INFO] TestAgent.server: Updating LAN server: server="Node-101e74f9-f8ea-ad87-8e2a-88a9c73c47d7 (Addr: tcp/127.0.0.1:27384) (DC: dc1)" -12:01.626 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-101e74f9-f8ea-ad87-8e2a-88a9c73c47d7.dc1 area=wan -12:01.627 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:01.627 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:01.627 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:01.627 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:01.627 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:01.627 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-101e74f9-f8ea-ad87-8e2a-88a9c73c47d7 -12:01.627 [INFO] TestAgent.server: member joined, marking health alive: member=Node-101e74f9-f8ea-ad87-8e2a-88a9c73c47d7 -12:01.628 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-101e74f9-f8ea-ad87-8e2a-88a9c73c47d7 -12:01.628 [INFO] TestAgent.server: federation state anti-entropy synced -12:01.716 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:45102 latency=1.582939ms -12:01.717 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-f5751104-3d9e-3105-26c4-9d44ab5fe430 from=127.0.0.1:45104 latency=368.557µs -12:01.719 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:45106 latency=492.105µs -12:01.719 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-f85c5c12-72d5-1bc9-7cb7-7043ca7ffac4 from=127.0.0.1:45104 latency=135.744µs -12:01.733 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:45108 latency=3.740923ms -12:01.738 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-cbaa4ff2-4d86-0542-6dba-341489ebfa16 from=127.0.0.1:45104 latency=718.466µs -12:01.739 [INFO] TestAgent: Requesting shutdown -12:01.739 [INFO] TestAgent.server: shutting down server -12:01.739 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:01.739 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:01.739 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:01.739 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:01.739 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:01.739 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:01.739 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:01.739 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:01.739 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:01.739 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:01.739 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:01.739 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:01.739 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:01.739 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:01.739 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:01.739 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:01.739Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:01.739Z [INFO] agent.router.manager: shutting down -12:01.739 [INFO] TestAgent: consul server down -12:01.739 [INFO] TestAgent: shutdown complete -12:01.739 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27379 network=tcp -12:01.739 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27379 network=udp -12:01.739 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27380 network=tcp -12:02.241 [INFO] TestAgent: Waiting for endpoints to shut down -12:02.241 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodCreateCommand_JSON (0.85s) - --- PASS: TestAuthMethodCreateCommand_JSON/type_required (0.00s) - --- PASS: TestAuthMethodCreateCommand_JSON/create_testing (0.01s) - --- PASS: TestAuthMethodCreateCommand_JSON/create_testing_with_max_token_ttl (0.00s) - --- PASS: TestAuthMethodCreateCommand_JSON/create_testing_with_token_type_global (0.02s) -2021-10-31T23:12:01.515Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:01.560Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:01.562 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fc071158-f45c-bcc2-0081-d0f9f9309d4a Address:127.0.0.1:27377}]" -12:01.564 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fc071158-f45c-bcc2-0081-d0f9f9309d4a.dc1 127.0.0.1 -12:01.565 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fc071158-f45c-bcc2-0081-d0f9f9309d4a 127.0.0.1 -2021-10-31T23:12:01.565Z [INFO] agent.router: Initializing LAN area manager -12:01.569 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27377 [Follower]" leader= -12:01.571 [INFO] TestAgent.server: Adding LAN server: server="Node-fc071158-f45c-bcc2-0081-d0f9f9309d4a (Addr: tcp/127.0.0.1:27377) (DC: dc1)" -12:01.575 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fc071158-f45c-bcc2-0081-d0f9f9309d4a.dc1 area=wan -12:01.578 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27372 network=udp -12:01.578 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27372 network=tcp -12:01.579 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27373 network=tcp -12:01.579 [INFO] TestAgent: started state syncer -12:01.579 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27378 network=tcp -12:01.611 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:01.611 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27377 [Candidate]" term=2 -12:01.611 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:01.611 [DEBUG] TestAgent.server.raft: vote granted: from=fc071158-f45c-bcc2-0081-d0f9f9309d4a term=2 tally=1 -12:01.611 [INFO] TestAgent.server.raft: election won: tally=1 -12:01.611 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27377 [Leader]" -12:01.611 [INFO] TestAgent.server: cluster leadership acquired -12:01.611 [INFO] TestAgent.server: initializing acls -12:01.612 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:01.612 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:01.612 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:01.612 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:01.612 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:01.612 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:01.612 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-fc071158-f45c-bcc2-0081-d0f9f9309d4a -12:01.612 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-fc071158-f45c-bcc2-0081-d0f9f9309d4a.dc1 -12:01.613 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:01.613 [INFO] TestAgent.server: New leader elected: payload=Node-fc071158-f45c-bcc2-0081-d0f9f9309d4a -12:01.614 [INFO] TestAgent.server: Updating LAN server: server="Node-fc071158-f45c-bcc2-0081-d0f9f9309d4a (Addr: tcp/127.0.0.1:27377) (DC: dc1)" -12:01.614 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-fc071158-f45c-bcc2-0081-d0f9f9309d4a.dc1 area=wan -12:01.614 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:01.614 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:01.614 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:01.614 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:01.614 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:01.614 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fc071158-f45c-bcc2-0081-d0f9f9309d4a -12:01.614 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fc071158-f45c-bcc2-0081-d0f9f9309d4a -12:01.615 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fc071158-f45c-bcc2-0081-d0f9f9309d4a -12:01.615 [INFO] TestAgent.server: federation state anti-entropy synced -12:01.877 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:32824 latency=2.612541ms -12:01.888 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-0863e761-3c55-cff6-eeb2-cedeb645569a from=127.0.0.1:32830 latency=1.222295ms -12:01.902 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:32832 latency=9.802236ms -12:01.914 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-17aec0e6-c217-ff35-f850-16fb5d620d43 from=127.0.0.1:32830 latency=504.925µs -12:01.915 [INFO] TestAgent: Requesting shutdown -12:01.915 [INFO] TestAgent.server: shutting down server -12:01.915 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:01.915 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:01.915 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:01.915 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:01.915 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:01.915 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:01.915 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:01.915 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:01.915 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:01.915 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:01.915 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:01.915 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:01.915 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:01.915 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:01.915 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:01.915 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:01.915Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:01.915Z [INFO] agent.router.manager: shutting down -12:01.915 [INFO] TestAgent: consul server down -12:01.915 [INFO] TestAgent: shutdown complete -12:01.915 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27372 network=tcp -12:01.915 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27372 network=udp -12:01.915 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27373 network=tcp -12:02.417 [INFO] TestAgent: Waiting for endpoints to shut down -12:02.417 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodCreateCommand_k8s (1.03s) - --- PASS: TestAuthMethodCreateCommand_k8s/k8s_host_required (0.00s) - --- PASS: TestAuthMethodCreateCommand_k8s/k8s_ca_cert_required (0.00s) - --- PASS: TestAuthMethodCreateCommand_k8s/k8s_jwt_required (0.00s) - --- PASS: TestAuthMethodCreateCommand_k8s/create_k8s (0.02s) - --- PASS: TestAuthMethodCreateCommand_k8s/create_k8s_with_cert_file (0.03s) -2021-10-31T23:12:01.473Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:01.520Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:01.527 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:91f375dd-4528-4123-1e67-ec285af86382 Address:127.0.0.1:27370}]" -12:01.528 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-91f375dd-4528-4123-1e67-ec285af86382.dc1 127.0.0.1 -12:01.529 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-91f375dd-4528-4123-1e67-ec285af86382 127.0.0.1 -2021-10-31T23:12:01.529Z [INFO] agent.router: Initializing LAN area manager -12:01.546 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-91f375dd-4528-4123-1e67-ec285af86382.dc1 area=wan -12:01.546 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27365 network=udp -12:01.546 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27370 [Follower]" leader= -12:01.546 [INFO] TestAgent.server: Adding LAN server: server="Node-91f375dd-4528-4123-1e67-ec285af86382 (Addr: tcp/127.0.0.1:27370) (DC: dc1)" -12:01.546 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27365 network=tcp -12:01.547 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27366 network=tcp -12:01.547 [INFO] TestAgent: started state syncer -12:01.547 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27371 network=tcp -12:01.593 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:01.593 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27370 [Candidate]" term=2 -12:01.594 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:01.594 [DEBUG] TestAgent.server.raft: vote granted: from=91f375dd-4528-4123-1e67-ec285af86382 term=2 tally=1 -12:01.594 [INFO] TestAgent.server.raft: election won: tally=1 -12:01.594 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27370 [Leader]" -12:01.594 [INFO] TestAgent.server: cluster leadership acquired -12:01.594 [INFO] TestAgent.server: initializing acls -12:01.594 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:01.594 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:01.594 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:01.595 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:01.595 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:01.595 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:01.595 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-91f375dd-4528-4123-1e67-ec285af86382 -12:01.595 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-91f375dd-4528-4123-1e67-ec285af86382.dc1 -12:01.595 [INFO] TestAgent.server: New leader elected: payload=Node-91f375dd-4528-4123-1e67-ec285af86382 -12:01.595 [INFO] TestAgent.server: Updating LAN server: server="Node-91f375dd-4528-4123-1e67-ec285af86382 (Addr: tcp/127.0.0.1:27370) (DC: dc1)" -12:01.595 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-91f375dd-4528-4123-1e67-ec285af86382.dc1 area=wan -12:01.596 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:01.605 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:01.605 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:01.605 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:01.605 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:01.605 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:01.605 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-91f375dd-4528-4123-1e67-ec285af86382 -12:01.605 [INFO] TestAgent.server: member joined, marking health alive: member=Node-91f375dd-4528-4123-1e67-ec285af86382 -12:01.606 [INFO] TestAgent.server: federation state anti-entropy synced -12:01.606 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-91f375dd-4528-4123-1e67-ec285af86382 -12:01.782 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:01.782 [INFO] TestAgent: Synced node info -12:01.782 [DEBUG] TestAgent: Node info in sync -12:01.864 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/auth-method from=127.0.0.1:40226 error="Invalid Auth Method: Type should be one of: [jwt kubernetes testing]" -12:01.864 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:40226 latency=531.095µs -12:01.871 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:40228 latency=1.44307ms -12:01.875 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-34f4231a-076c-d306-b06b-713998cf55f9 from=127.0.0.1:40232 latency=461.258µs -12:01.894 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:40234 latency=17.043374ms -12:01.906 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-e6e17675-fc57-d6e7-fe49-d8102635333b from=127.0.0.1:40232 latency=427.852µs -12:01.914 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:40240 latency=885.892µs -12:01.922 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-a9f92833-a244-3392-5efb-a82720a8f700 from=127.0.0.1:40232 latency=291.358µs -12:01.924 [INFO] TestAgent: Requesting shutdown -12:01.924 [INFO] TestAgent.server: shutting down server -12:01.924 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:01.924 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:01.924 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:01.924 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:01.924 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:01.924 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:01.924 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:01.924 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:01.924 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:01.924 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:01.924 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:01.924 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:01.924 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:01.924 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:01.924 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:01.925Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:01.925Z [INFO] agent.router.manager: shutting down -12:01.925 [INFO] TestAgent: consul server down -12:01.925 [INFO] TestAgent: shutdown complete -12:01.925 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27365 network=tcp -12:01.925 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27365 network=udp -12:01.925 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27366 network=tcp -12:02.425 [INFO] TestAgent: Waiting for endpoints to shut down -12:02.425 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodCreateCommand (1.04s) +=== RUN TestAuthMethodCreateCommand_k8s/k8s_host_required +=== RUN TestAuthMethodCreateCommand_k8s/k8s_ca_cert_required +=== RUN TestAuthMethodCreateCommand_k8s/k8s_jwt_required +=== RUN TestAuthMethodCreateCommand_k8s/create_k8s +=== RUN TestAuthMethodCreateCommand_k8s/create_k8s_with_cert_file +2022-12-04T05:48:07.193Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:07.217Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:07.218 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:740888b3-4158-4295-ed28-f8e92306c8ba Address:127.0.0.1:11041}]" +48:07.219 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11041 [Follower]" leader= +48:07.219 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-740888b3-4158-4295-ed28-f8e92306c8ba.dc1 127.0.0.1 +48:07.219 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-740888b3-4158-4295-ed28-f8e92306c8ba 127.0.0.1 +2022-12-04T05:48:07.219Z [INFO] agent.router: Initializing LAN area manager +48:07.220 [INFO] TestAgent.server: Adding LAN server: server="Node-740888b3-4158-4295-ed28-f8e92306c8ba (Addr: tcp/127.0.0.1:11041) (DC: dc1)" +48:07.220 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11036 network=udp +48:07.220 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-740888b3-4158-4295-ed28-f8e92306c8ba.dc1 area=wan +48:07.220 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11036 network=tcp +48:07.220 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11037 network=tcp +48:07.220 [INFO] TestAgent: started state syncer +48:07.220 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11042 network=tcp +48:07.256 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:07.256 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11041 [Candidate]" term=2 +48:07.256 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:07.256 [DEBUG] TestAgent.server.raft: vote granted: from=740888b3-4158-4295-ed28-f8e92306c8ba term=2 tally=1 +48:07.256 [INFO] TestAgent.server.raft: election won: tally=1 +48:07.256 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11041 [Leader]" +48:07.257 [INFO] TestAgent.server: cluster leadership acquired +48:07.257 [INFO] TestAgent.server: initializing acls +48:07.257 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:07.257 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:07.258 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:07.258 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:07.258 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:07.258 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:07.258 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-740888b3-4158-4295-ed28-f8e92306c8ba +48:07.258 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-740888b3-4158-4295-ed28-f8e92306c8ba.dc1 +48:07.258 [INFO] TestAgent.server: New leader elected: payload=Node-740888b3-4158-4295-ed28-f8e92306c8ba +48:07.258 [INFO] TestAgent.server: Updating LAN server: server="Node-740888b3-4158-4295-ed28-f8e92306c8ba (Addr: tcp/127.0.0.1:11041) (DC: dc1)" +48:07.259 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-740888b3-4158-4295-ed28-f8e92306c8ba.dc1 area=wan +48:07.259 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:07.286 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:07.286 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:07.286 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:07.286 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:07.286 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:07.286 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-740888b3-4158-4295-ed28-f8e92306c8ba +48:07.286 [INFO] TestAgent.server: member joined, marking health alive: member=Node-740888b3-4158-4295-ed28-f8e92306c8ba +48:07.287 [INFO] TestAgent.server: federation state anti-entropy synced +48:07.289 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-740888b3-4158-4295-ed28-f8e92306c8ba +48:07.348 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47240 error="Invalid Auth Method: Type should be one of: [jwt kubernetes testing]" +48:07.348 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47240 latency=573.792µs +48:07.354 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47242 latency=924.429µs +48:07.356 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-91c8656a-1bad-57a6-ee8d-9b47e4ed431c from=127.0.0.1:47244 latency=673.446µs +48:07.362 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47246 latency=4.473682ms +48:07.364 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-e0e0dc33-8db8-6b46-0947-536656023973 from=127.0.0.1:47244 latency=395.703µs +48:07.402 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47248 latency=1.486518ms +48:07.403 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-292c1781-8ab5-1812-1c62-0dc37a133317 from=127.0.0.1:47244 latency=224.398µs +48:07.406 [INFO] TestAgent: Requesting shutdown +48:07.406 [INFO] TestAgent.server: shutting down server +48:07.406 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:07.406 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:07.406 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:07.406 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:07.406 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:07.406 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:07.406 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:07.406 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:07.406 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:07.406 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:07.407 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:07.407 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:07.407 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:07.407 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:07.407 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:07.407 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:07.407Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:07.407Z [INFO] agent.router.manager: shutting down +48:07.407 [INFO] TestAgent: consul server down +48:07.407 [INFO] TestAgent: shutdown complete +48:07.407 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11036 network=tcp +48:07.407 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11036 network=udp +48:07.407 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11037 network=tcp +48:07.907 [INFO] TestAgent: Waiting for endpoints to shut down +48:07.907 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodCreateCommand (0.82s) --- PASS: TestAuthMethodCreateCommand/type_required (0.00s) --- PASS: TestAuthMethodCreateCommand/name_required (0.00s) --- PASS: TestAuthMethodCreateCommand/invalid_type (0.00s) --- PASS: TestAuthMethodCreateCommand/create_testing (0.01s) - --- PASS: TestAuthMethodCreateCommand/create_testing_with_max_token_ttl (0.03s) - --- PASS: TestAuthMethodCreateCommand/create_testing_with_token_type_global (0.02s) -2021-10-31T23:12:01.471Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:01.501Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:01.514 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4b80a6ab-8876-86ee-ac2d-b472c8238341 Address:127.0.0.1:27363}]" -12:01.515 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4b80a6ab-8876-86ee-ac2d-b472c8238341.dc1 127.0.0.1 -12:01.516 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4b80a6ab-8876-86ee-ac2d-b472c8238341 127.0.0.1 -2021-10-31T23:12:01.516Z [INFO] agent.router: Initializing LAN area manager -12:01.516 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp -12:01.516 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= -12:01.547 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp -12:01.548 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp -12:01.548 [INFO] TestAgent: started state syncer -12:01.548 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp -12:01.548 [INFO] TestAgent.server: Adding LAN server: server="Node-4b80a6ab-8876-86ee-ac2d-b472c8238341 (Addr: tcp/127.0.0.1:27363) (DC: dc1)" -12:01.548 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4b80a6ab-8876-86ee-ac2d-b472c8238341.dc1 area=wan -12:01.573 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:01.573 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 -12:01.573 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:01.573 [DEBUG] TestAgent.server.raft: vote granted: from=4b80a6ab-8876-86ee-ac2d-b472c8238341 term=2 tally=1 -12:01.573 [INFO] TestAgent.server.raft: election won: tally=1 -12:01.573 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" -12:01.573 [INFO] TestAgent.server: cluster leadership acquired -12:01.573 [INFO] TestAgent.server: initializing acls -12:01.574 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:01.574 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:01.575 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:01.575 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:01.575 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:01.575 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:01.576 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4b80a6ab-8876-86ee-ac2d-b472c8238341 -12:01.576 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4b80a6ab-8876-86ee-ac2d-b472c8238341.dc1 -12:01.577 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:01.578 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4b80a6ab-8876-86ee-ac2d-b472c8238341.dc1 area=wan -12:01.580 [INFO] TestAgent.server: New leader elected: payload=Node-4b80a6ab-8876-86ee-ac2d-b472c8238341 -12:01.580 [INFO] TestAgent.server: Updating LAN server: server="Node-4b80a6ab-8876-86ee-ac2d-b472c8238341 (Addr: tcp/127.0.0.1:27363) (DC: dc1)" -12:01.604 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:01.604 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:01.604 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:01.604 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:01.604 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:01.604 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4b80a6ab-8876-86ee-ac2d-b472c8238341 -12:01.604 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4b80a6ab-8876-86ee-ac2d-b472c8238341 -12:01.604 [INFO] TestAgent.server: federation state anti-entropy synced -12:01.605 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4b80a6ab-8876-86ee-ac2d-b472c8238341 -12:01.687 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:01.687 [INFO] TestAgent: Synced node info -12:01.687 [DEBUG] TestAgent: Node info in sync -12:01.732 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:01.732 [DEBUG] TestAgent: Node info in sync -12:01.976 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:53398 latency=786.96µs -12:01.978 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-ee5f9e68-286e-9e97-b2f8-d2d154765adc from=127.0.0.1:53400 latency=530.017µs -12:01.980 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:53402 latency=523.295µs -12:01.980 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-808f00c0-7a20-69fc-0d6d-30f449e69150 from=127.0.0.1:53400 latency=136.576µs -12:01.982 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:53404 latency=530.827µs -12:01.989 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-b1e2cbbb-dda2-886a-89cc-550c75ecbf13 from=127.0.0.1:53400 latency=335.882µs -12:01.989 [INFO] TestAgent: Requesting shutdown -12:01.989 [INFO] TestAgent.server: shutting down server -12:01.989 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:01.989 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:01.989 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:01.989 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:01.989 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:01.989 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:01.989 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:01.989 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:01.989 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:01.990 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:01.990 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:01.990 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:01.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:01.990 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:01.990 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:01.990Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:01.990Z [INFO] agent.router.manager: shutting down -12:01.990 [INFO] TestAgent: consul server down -12:01.990 [INFO] TestAgent: shutdown complete -12:01.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp -12:01.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp -12:01.990 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp -12:02.492 [INFO] TestAgent: Waiting for endpoints to shut down -12:02.492 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodCreateCommand_config (1.11s) - --- PASS: TestAuthMethodCreateCommand_config/config_file (0.00s) + --- PASS: TestAuthMethodCreateCommand/create_testing_with_max_token_ttl (0.01s) + --- PASS: TestAuthMethodCreateCommand/create_testing_with_token_type_global (0.04s) +2022-12-04T05:48:07.144Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:07.208Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:07.210 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0a429082-f3d7-f8df-caec-0e313a947816 Address:127.0.0.1:11034}]" +48:07.212 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11034 [Follower]" leader= +48:07.214 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0a429082-f3d7-f8df-caec-0e313a947816.dc1 127.0.0.1 +48:07.215 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0a429082-f3d7-f8df-caec-0e313a947816 127.0.0.1 +2022-12-04T05:48:07.215Z [INFO] agent.router: Initializing LAN area manager +48:07.215 [INFO] TestAgent.server: Adding LAN server: server="Node-0a429082-f3d7-f8df-caec-0e313a947816 (Addr: tcp/127.0.0.1:11034) (DC: dc1)" +48:07.215 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0a429082-f3d7-f8df-caec-0e313a947816.dc1 area=wan +48:07.215 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11029 network=udp +48:07.215 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11029 network=tcp +48:07.216 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11030 network=tcp +48:07.216 [INFO] TestAgent: started state syncer +48:07.216 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11035 network=tcp +48:07.258 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:07.259 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11034 [Candidate]" term=2 +48:07.259 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:07.259 [DEBUG] TestAgent.server.raft: vote granted: from=0a429082-f3d7-f8df-caec-0e313a947816 term=2 tally=1 +48:07.259 [INFO] TestAgent.server.raft: election won: tally=1 +48:07.259 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11034 [Leader]" +48:07.259 [INFO] TestAgent.server: cluster leadership acquired +48:07.259 [INFO] TestAgent.server: initializing acls +48:07.259 [INFO] TestAgent.server: New leader elected: payload=Node-0a429082-f3d7-f8df-caec-0e313a947816 +48:07.259 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:07.259 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:07.266 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:07.267 [INFO] TestAgent.server: initializing acls +48:07.267 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:07.267 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:07.267 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:07.267 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +48:07.267 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:07.267 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0a429082-f3d7-f8df-caec-0e313a947816 +48:07.268 [INFO] TestAgent.server: Updating LAN server: server="Node-0a429082-f3d7-f8df-caec-0e313a947816 (Addr: tcp/127.0.0.1:11034) (DC: dc1)" +48:07.268 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0a429082-f3d7-f8df-caec-0e313a947816.dc1 +48:07.268 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0a429082-f3d7-f8df-caec-0e313a947816.dc1 area=wan +48:07.269 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0a429082-f3d7-f8df-caec-0e313a947816 +48:07.269 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0a429082-f3d7-f8df-caec-0e313a947816.dc1 +48:07.270 [INFO] TestAgent.server: Updating LAN server: server="Node-0a429082-f3d7-f8df-caec-0e313a947816 (Addr: tcp/127.0.0.1:11034) (DC: dc1)" +48:07.270 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0a429082-f3d7-f8df-caec-0e313a947816.dc1 area=wan +48:07.270 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:07.288 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:07.288 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:07.288 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:07.288 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:07.288 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:07.288 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a429082-f3d7-f8df-caec-0e313a947816 +48:07.288 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0a429082-f3d7-f8df-caec-0e313a947816 +48:07.289 [INFO] TestAgent.server: federation state anti-entropy synced +48:07.289 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a429082-f3d7-f8df-caec-0e313a947816 +48:07.289 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a429082-f3d7-f8df-caec-0e313a947816 +48:07.403 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:53402 latency=756.184µs +48:07.407 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-2226accf-2b6c-8baa-ca86-4b7e871a1279 from=127.0.0.1:53404 latency=379.768µs +48:07.412 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:53406 latency=1.697753ms +48:07.414 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-414609c7-0af9-dc63-8241-e232b9ea6693 from=127.0.0.1:53404 latency=335.212µs +48:07.416 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:53408 latency=562.393µs +48:07.417 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-72590e08-17f6-10dc-786b-cdc5229e385a from=127.0.0.1:53404 latency=252.738µs +48:07.417 [INFO] TestAgent: Requesting shutdown +48:07.417 [INFO] TestAgent.server: shutting down server +48:07.417 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:07.417 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:07.417 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:07.417 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:07.417 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:07.417 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:07.417 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:07.417 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:07.417 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:07.417 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:07.417 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:07.417 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:07.417 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:07.417 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:07.417 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:07.417 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:07.418Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:07.418Z [INFO] agent.router.manager: shutting down +48:07.418 [INFO] TestAgent: consul server down +48:07.418 [INFO] TestAgent: shutdown complete +48:07.418 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11029 network=tcp +48:07.418 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11029 network=udp +48:07.418 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11030 network=tcp +48:07.918 [INFO] TestAgent: Waiting for endpoints to shut down +48:07.918 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodCreateCommand_JSON (0.83s) + --- PASS: TestAuthMethodCreateCommand_JSON/type_required (0.00s) + --- PASS: TestAuthMethodCreateCommand_JSON/create_testing (0.02s) + --- PASS: TestAuthMethodCreateCommand_JSON/create_testing_with_max_token_ttl (0.01s) + --- PASS: TestAuthMethodCreateCommand_JSON/create_testing_with_token_type_global (0.00s) +2022-12-04T05:48:07.156Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:07.209Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:07.213 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b7047ecc-efe8-a3df-27d4-a23300b96c32 Address:127.0.0.1:11027}]" +48:07.213 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11027 [Follower]" leader= +48:07.213 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b7047ecc-efe8-a3df-27d4-a23300b96c32.dc1 127.0.0.1 +48:07.214 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b7047ecc-efe8-a3df-27d4-a23300b96c32 127.0.0.1 +2022-12-04T05:48:07.214Z [INFO] agent.router: Initializing LAN area manager +48:07.214 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=udp +48:07.215 [INFO] TestAgent.server: Adding LAN server: server="Node-b7047ecc-efe8-a3df-27d4-a23300b96c32 (Addr: tcp/127.0.0.1:11027) (DC: dc1)" +48:07.215 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b7047ecc-efe8-a3df-27d4-a23300b96c32.dc1 area=wan +48:07.215 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=tcp +48:07.215 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11023 network=tcp +48:07.215 [INFO] TestAgent: started state syncer +48:07.215 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11028 network=tcp +48:07.267 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:07.283 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:07.284 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11027 [Candidate]" term=2 +48:07.284 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:07.284 [DEBUG] TestAgent.server.raft: vote granted: from=b7047ecc-efe8-a3df-27d4-a23300b96c32 term=2 tally=1 +48:07.284 [INFO] TestAgent.server.raft: election won: tally=1 +48:07.284 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11027 [Leader]" +48:07.284 [INFO] TestAgent.server: cluster leadership acquired +48:07.284 [INFO] TestAgent.server: New leader elected: payload=Node-b7047ecc-efe8-a3df-27d4-a23300b96c32 +48:07.284 [INFO] TestAgent.server: initializing acls +48:07.284 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:07.284 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:07.285 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:07.285 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:07.285 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:07.285 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:07.285 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b7047ecc-efe8-a3df-27d4-a23300b96c32 +48:07.285 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b7047ecc-efe8-a3df-27d4-a23300b96c32.dc1 +48:07.286 [INFO] TestAgent.server: Updating LAN server: server="Node-b7047ecc-efe8-a3df-27d4-a23300b96c32 (Addr: tcp/127.0.0.1:11027) (DC: dc1)" +48:07.286 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:07.286 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b7047ecc-efe8-a3df-27d4-a23300b96c32.dc1 area=wan +48:07.288 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:07.288 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:07.288 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:07.288 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:07.288 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:07.288 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b7047ecc-efe8-a3df-27d4-a23300b96c32 +48:07.288 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b7047ecc-efe8-a3df-27d4-a23300b96c32 +48:07.288 [INFO] TestAgent.server: federation state anti-entropy synced +48:07.289 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b7047ecc-efe8-a3df-27d4-a23300b96c32 +48:07.577 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:51716 latency=746.846µs +48:07.579 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-ae663993-aa1d-ac14-353b-85fab4ef9fc5 from=127.0.0.1:51720 latency=424.981µs +48:07.581 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:51722 latency=646.75µs +48:07.582 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-6fef9219-5d02-ea80-96ba-69f30ee78c88 from=127.0.0.1:51720 latency=311.243µs +48:07.597 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:51724 latency=1.670295ms +48:07.599 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-7ee8f5d9-8ee2-c970-6fb2-ec41cce151dd from=127.0.0.1:51720 latency=476.575µs +48:07.599 [INFO] TestAgent: Requesting shutdown +48:07.600 [INFO] TestAgent.server: shutting down server +48:07.600 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:07.600 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:07.600 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:07.600 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:07.600 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:07.600 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:07.600 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:07.600 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:07.600 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:07.600 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:07.600 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:07.600 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:07.600 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:07.600 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:07.600 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:07.600 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:07.600Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:07.600Z [INFO] agent.router.manager: shutting down +48:07.600 [INFO] TestAgent: consul server down +48:07.600 [INFO] TestAgent: shutdown complete +48:07.600 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=tcp +48:07.600 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=udp +48:07.600 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11023 network=tcp +48:08.100 [INFO] TestAgent: Waiting for endpoints to shut down +48:08.100 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodCreateCommand_config (1.01s) + --- PASS: TestAuthMethodCreateCommand_config/config_file (0.01s) --- PASS: TestAuthMethodCreateCommand_config/config_std-in (0.00s) - --- PASS: TestAuthMethodCreateCommand_config/config_string (0.01s) + --- PASS: TestAuthMethodCreateCommand_config/config_string (0.02s) +2022-12-04T05:48:07.145Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:07.176Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:07.187 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1f10d1e8-1320-e81c-712d-872fc19e86af Address:127.0.0.1:11048}]" +48:07.188 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11048 [Follower]" leader= +48:07.192 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1f10d1e8-1320-e81c-712d-872fc19e86af.dc1 127.0.0.1 +48:07.192 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1f10d1e8-1320-e81c-712d-872fc19e86af 127.0.0.1 +2022-12-04T05:48:07.192Z [INFO] agent.router: Initializing LAN area manager +48:07.193 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11043 network=udp +48:07.194 [INFO] TestAgent.server: Adding LAN server: server="Node-1f10d1e8-1320-e81c-712d-872fc19e86af (Addr: tcp/127.0.0.1:11048) (DC: dc1)" +48:07.194 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1f10d1e8-1320-e81c-712d-872fc19e86af.dc1 area=wan +48:07.194 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11043 network=tcp +48:07.195 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11044 network=tcp +48:07.195 [INFO] TestAgent: started state syncer +48:07.195 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11049 network=tcp +48:07.245 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:07.267 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:07.267 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11048 [Candidate]" term=2 +48:07.267 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:07.267 [DEBUG] TestAgent.server.raft: vote granted: from=1f10d1e8-1320-e81c-712d-872fc19e86af term=2 tally=1 +48:07.267 [INFO] TestAgent.server.raft: election won: tally=1 +48:07.267 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11048 [Leader]" +48:07.267 [INFO] TestAgent.server: cluster leadership acquired +48:07.267 [INFO] TestAgent.server: initializing acls +48:07.267 [INFO] TestAgent.server: New leader elected: payload=Node-1f10d1e8-1320-e81c-712d-872fc19e86af +48:07.270 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:07.270 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:07.271 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:07.271 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:07.271 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:07.271 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:07.271 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1f10d1e8-1320-e81c-712d-872fc19e86af +48:07.272 [INFO] TestAgent.server: Updating LAN server: server="Node-1f10d1e8-1320-e81c-712d-872fc19e86af (Addr: tcp/127.0.0.1:11048) (DC: dc1)" +48:07.274 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1f10d1e8-1320-e81c-712d-872fc19e86af.dc1 +48:07.275 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1f10d1e8-1320-e81c-712d-872fc19e86af.dc1 area=wan +48:07.275 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:07.288 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:07.288 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:07.288 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:07.288 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:07.288 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:07.288 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1f10d1e8-1320-e81c-712d-872fc19e86af +48:07.288 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1f10d1e8-1320-e81c-712d-872fc19e86af +48:07.289 [INFO] TestAgent.server: federation state anti-entropy synced +48:07.289 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1f10d1e8-1320-e81c-712d-872fc19e86af +48:07.376 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:07.376 [INFO] TestAgent: Synced node info +48:07.376 [DEBUG] TestAgent: Node info in sync +48:07.506 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:07.506 [DEBUG] TestAgent: Node info in sync +48:07.631 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:53978 latency=2.481142ms +48:07.633 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-8d43f1f5-17aa-82e0-1e31-a1955b079cad from=127.0.0.1:53980 latency=399.262µs +48:07.642 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:53982 latency=1.874731ms +48:07.643 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-880ec692-a6c9-e051-6f30-709dae6c01ef from=127.0.0.1:53980 latency=237.776µs +48:07.643 [INFO] TestAgent: Requesting shutdown +48:07.643 [INFO] TestAgent.server: shutting down server +48:07.643 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:07.643 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:07.643 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:07.643 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:07.643 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:07.643 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:07.643 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:07.644 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:07.644 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:07.644 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:07.644 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:07.644 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:07.644 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:07.644 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:07.644 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:07.644Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:07.644Z [INFO] agent.router.manager: shutting down +48:07.644 [INFO] TestAgent: consul server down +48:07.644 [INFO] TestAgent: shutdown complete +48:07.644 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11043 network=tcp +48:07.644 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11043 network=udp +48:07.644 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11044 network=tcp +48:08.144 [INFO] TestAgent: Waiting for endpoints to shut down +48:08.144 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodCreateCommand_k8s (1.05s) + --- PASS: TestAuthMethodCreateCommand_k8s/k8s_host_required (0.00s) + --- PASS: TestAuthMethodCreateCommand_k8s/k8s_ca_cert_required (0.00s) + --- PASS: TestAuthMethodCreateCommand_k8s/k8s_jwt_required (0.00s) + --- PASS: TestAuthMethodCreateCommand_k8s/create_k8s (0.01s) + --- PASS: TestAuthMethodCreateCommand_k8s/create_k8s_with_cert_file (0.01s) PASS ok github.com/hashicorp/consul/command/acl/authmethod/create 1.205s === RUN TestAuthMethodDeleteCommand_noTabs @@ -72900,95 +73026,96 @@ === PAUSE TestAuthMethodDeleteCommand === CONT TestAuthMethodDeleteCommand_noTabs === CONT TestAuthMethodDeleteCommand ---- PASS: TestAuthMethodDeleteCommand_noTabs (0.00s) [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestAuthMethodDeleteCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestAuthMethodDeleteCommand/name_required === RUN TestAuthMethodDeleteCommand/delete_notfound === RUN TestAuthMethodDeleteCommand/delete_works -2021-10-31T23:12:16.932Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:16.987Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:16.989 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8198843d-f1c1-ac9e-417c-27233069ebd2 Address:127.0.0.1:14090}]" -12:16.989 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= -12:16.994 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8198843d-f1c1-ac9e-417c-27233069ebd2.dc1 127.0.0.1 -12:16.994 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8198843d-f1c1-ac9e-417c-27233069ebd2 127.0.0.1 -2021-10-31T23:12:16.994Z [INFO] agent.router: Initializing LAN area manager -12:16.998 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8198843d-f1c1-ac9e-417c-27233069ebd2.dc1 area=wan -12:16.998 [INFO] TestAgent.server: Adding LAN server: server="Node-8198843d-f1c1-ac9e-417c-27233069ebd2 (Addr: tcp/127.0.0.1:14090) (DC: dc1)" -12:16.998 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp -12:16.998 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp -12:16.999 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp -12:16.999 [INFO] TestAgent: started state syncer -12:16.999 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp -12:17.049 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -12:17.053 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:17.053 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 -12:17.054 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:17.054 [DEBUG] TestAgent.server.raft: vote granted: from=8198843d-f1c1-ac9e-417c-27233069ebd2 term=2 tally=1 -12:17.054 [INFO] TestAgent.server.raft: election won: tally=1 -12:17.054 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" -12:17.054 [INFO] TestAgent.server: cluster leadership acquired -12:17.054 [INFO] TestAgent.server: initializing acls -12:17.054 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:17.054 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:17.055 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:17.055 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:17.055 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:17.055 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:17.055 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-8198843d-f1c1-ac9e-417c-27233069ebd2 -12:17.055 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-8198843d-f1c1-ac9e-417c-27233069ebd2.dc1 -12:17.056 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:17.057 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-8198843d-f1c1-ac9e-417c-27233069ebd2.dc1 area=wan -12:17.058 [INFO] TestAgent.server: New leader elected: payload=Node-8198843d-f1c1-ac9e-417c-27233069ebd2 -12:17.058 [INFO] TestAgent.server: Updating LAN server: server="Node-8198843d-f1c1-ac9e-417c-27233069ebd2 (Addr: tcp/127.0.0.1:14090) (DC: dc1)" -12:17.110 [INFO] TestAgent: Synced node info -12:17.110 [DEBUG] TestAgent: Node info in sync -12:17.135 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:17.135 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:17.135 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:17.135 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:17.135 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:17.135 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8198843d-f1c1-ac9e-417c-27233069ebd2 -12:17.135 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8198843d-f1c1-ac9e-417c-27233069ebd2 -12:17.136 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8198843d-f1c1-ac9e-417c-27233069ebd2 -12:17.136 [INFO] TestAgent.server: federation state anti-entropy synced -12:17.211 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-method/notfound from=127.0.0.1:47802 latency=4.961431ms -12:17.251 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47804 latency=1.89242ms -12:17.254 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-method/test-96e9dc93-c6dd-9f90-3ab6-2f4c768cea54 from=127.0.0.1:47806 latency=709.274µs -12:17.254 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-96e9dc93-c6dd-9f90-3ab6-2f4c768cea54 from=127.0.0.1:47804 latency=82.722µs -12:17.254 [INFO] TestAgent: Requesting shutdown -12:17.254 [INFO] TestAgent.server: shutting down server -12:17.255 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:17.255 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:17.255 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:17.255 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:17.255 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:17.255 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:17.255 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:17.255 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:17.255 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:17.255 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:17.255 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:17.255 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:17.255 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:17.255 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:17.255 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:17.255Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:17.255Z [INFO] agent.router.manager: shutting down -12:17.255 [INFO] TestAgent: consul server down -12:17.255 [INFO] TestAgent: shutdown complete -12:17.255 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp -12:17.255 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp -12:17.255 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp -12:17.757 [INFO] TestAgent: Waiting for endpoints to shut down -12:17.757 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodDeleteCommand (0.96s) +2022-12-04T05:48:10.623Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:10.646Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:10.648 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dc1f1d23-43cf-e455-24a5-520e6b2a52c0 Address:127.0.0.1:27363}]" +48:10.649 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= +48:10.649 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dc1f1d23-43cf-e455-24a5-520e6b2a52c0.dc1 127.0.0.1 +48:10.650 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dc1f1d23-43cf-e455-24a5-520e6b2a52c0 127.0.0.1 +2022-12-04T05:48:10.650Z [INFO] agent.router: Initializing LAN area manager +48:10.650 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp +48:10.650 [INFO] TestAgent.server: Adding LAN server: server="Node-dc1f1d23-43cf-e455-24a5-520e6b2a52c0 (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +48:10.650 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dc1f1d23-43cf-e455-24a5-520e6b2a52c0.dc1 area=wan +48:10.650 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp +48:10.651 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp +48:10.651 [INFO] TestAgent: started state syncer +48:10.651 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp +48:10.701 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:10.703 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:10.703 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 +48:10.703 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:10.703 [DEBUG] TestAgent.server.raft: vote granted: from=dc1f1d23-43cf-e455-24a5-520e6b2a52c0 term=2 tally=1 +48:10.703 [INFO] TestAgent.server.raft: election won: tally=1 +48:10.703 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" +48:10.703 [INFO] TestAgent.server: cluster leadership acquired +48:10.703 [INFO] TestAgent.server: initializing acls +48:10.703 [INFO] TestAgent.server: New leader elected: payload=Node-dc1f1d23-43cf-e455-24a5-520e6b2a52c0 +48:10.703 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:10.703 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:10.704 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:10.704 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:10.704 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:10.704 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:10.704 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-dc1f1d23-43cf-e455-24a5-520e6b2a52c0 +48:10.704 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-dc1f1d23-43cf-e455-24a5-520e6b2a52c0.dc1 +48:10.705 [INFO] TestAgent.server: Updating LAN server: server="Node-dc1f1d23-43cf-e455-24a5-520e6b2a52c0 (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +48:10.705 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-dc1f1d23-43cf-e455-24a5-520e6b2a52c0.dc1 area=wan +48:10.705 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:10.721 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:10.721 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:10.721 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:10.721 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:10.721 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:10.721 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dc1f1d23-43cf-e455-24a5-520e6b2a52c0 +48:10.721 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dc1f1d23-43cf-e455-24a5-520e6b2a52c0 +48:10.721 [INFO] TestAgent.server: federation state anti-entropy synced +48:10.722 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dc1f1d23-43cf-e455-24a5-520e6b2a52c0 +48:10.870 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:10.871 [INFO] TestAgent: Synced node info +48:10.871 [DEBUG] TestAgent: Node info in sync +48:10.897 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-method/notfound from=127.0.0.1:59662 latency=1.527567ms +48:10.900 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:59664 latency=924.921µs +48:10.902 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/auth-method/test-c54fd93d-be57-21a5-9b2c-15757e343dac from=127.0.0.1:59666 latency=1.032391ms +48:10.902 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-c54fd93d-be57-21a5-9b2c-15757e343dac from=127.0.0.1:59664 latency=86.641µs +48:10.902 [INFO] TestAgent: Requesting shutdown +48:10.903 [INFO] TestAgent.server: shutting down server +48:10.903 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:10.903 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:10.903 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:10.903 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:10.903 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:10.903 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:10.903 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:10.903 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:10.903 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:10.903 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:10.903 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:10.903 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:10.903 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:10.903 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:10.903 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +2022-12-04T05:48:10.903Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:10.903Z [INFO] agent.router.manager: shutting down +48:10.903 [INFO] TestAgent: consul server down +48:10.903 [INFO] TestAgent: shutdown complete +48:10.903 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp +48:10.903 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp +48:10.903 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp +48:11.404 [INFO] TestAgent: Waiting for endpoints to shut down +48:11.404 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodDeleteCommand (0.81s) --- PASS: TestAuthMethodDeleteCommand/name_required (0.00s) - --- PASS: TestAuthMethodDeleteCommand/delete_notfound (0.01s) - --- PASS: TestAuthMethodDeleteCommand/delete_works (0.04s) + --- PASS: TestAuthMethodDeleteCommand/delete_notfound (0.00s) + --- PASS: TestAuthMethodDeleteCommand/delete_works (0.01s) PASS -ok github.com/hashicorp/consul/command/acl/authmethod/delete 1.218s +ok github.com/hashicorp/consul/command/acl/authmethod/delete 0.865s === RUN TestAuthMethodListCommand_noTabs === PAUSE TestAuthMethodListCommand_noTabs === RUN TestAuthMethodListCommand @@ -73005,169 +73132,171 @@ === RUN TestAuthMethodListCommand_JSON/found_some === RUN TestAuthMethodListCommand/found_none === RUN TestAuthMethodListCommand/found_some -2021-10-31T23:12:21.516Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:21.609Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:21.612 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f9f78bf8-cc4f-cbda-3160-243890b6b471 Address:127.0.0.1:15111}]" -12:21.613 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15111 [Follower]" leader= -12:21.613 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f9f78bf8-cc4f-cbda-3160-243890b6b471.dc1 127.0.0.1 -12:21.614 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f9f78bf8-cc4f-cbda-3160-243890b6b471 127.0.0.1 -2021-10-31T23:12:21.614Z [INFO] agent.router: Initializing LAN area manager -12:21.614 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=udp -12:21.614 [INFO] TestAgent.server: Adding LAN server: server="Node-f9f78bf8-cc4f-cbda-3160-243890b6b471 (Addr: tcp/127.0.0.1:15111) (DC: dc1)" -12:21.614 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f9f78bf8-cc4f-cbda-3160-243890b6b471.dc1 area=wan -12:21.614 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=tcp -12:21.615 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15107 network=tcp -12:21.615 [INFO] TestAgent: started state syncer -12:21.615 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15112 network=tcp -12:21.665 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -12:21.677 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:21.677 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15111 [Candidate]" term=2 -12:21.677 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:21.677 [DEBUG] TestAgent.server.raft: vote granted: from=f9f78bf8-cc4f-cbda-3160-243890b6b471 term=2 tally=1 -12:21.677 [INFO] TestAgent.server.raft: election won: tally=1 -12:21.677 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15111 [Leader]" -12:21.677 [INFO] TestAgent.server: cluster leadership acquired -12:21.677 [INFO] TestAgent.server: initializing acls -12:21.677 [INFO] TestAgent.server: New leader elected: payload=Node-f9f78bf8-cc4f-cbda-3160-243890b6b471 -12:21.677 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:21.677 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:21.678 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:21.678 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:21.678 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:21.678 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:21.678 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f9f78bf8-cc4f-cbda-3160-243890b6b471 -12:21.679 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f9f78bf8-cc4f-cbda-3160-243890b6b471.dc1 -12:21.679 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:21.679 [INFO] TestAgent.server: Updating LAN server: server="Node-f9f78bf8-cc4f-cbda-3160-243890b6b471 (Addr: tcp/127.0.0.1:15111) (DC: dc1)" -12:21.679 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f9f78bf8-cc4f-cbda-3160-243890b6b471.dc1 area=wan -12:21.699 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:21.699 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:21.699 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:21.699 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:21.699 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:21.699 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f9f78bf8-cc4f-cbda-3160-243890b6b471 -12:21.699 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f9f78bf8-cc4f-cbda-3160-243890b6b471 -12:21.700 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f9f78bf8-cc4f-cbda-3160-243890b6b471 -12:21.700 [INFO] TestAgent.server: federation state anti-entropy synced -12:21.725 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:21.728 [INFO] TestAgent: Synced node info -12:21.952 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:59154 latency=2.555778ms -12:21.958 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:59154 latency=4.893387ms -12:21.966 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:59154 latency=787.411µs -12:21.979 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:59154 latency=662.689µs -12:21.979 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:59154 latency=377.165µs -12:21.981 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-methods from=127.0.0.1:59156 latency=527.172µs -12:21.986 [INFO] TestAgent: Requesting shutdown -12:21.986 [INFO] TestAgent.server: shutting down server -12:21.986 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:21.986 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:21.986 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:21.986 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:21.986 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:21.986 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:21.986 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:21.987 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:21.987 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:21.987 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:21.987 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:21.987 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:21.987 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:21.987 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:21.987 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:21.987Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:21.987Z [INFO] agent.router.manager: shutting down -12:21.987 [INFO] TestAgent: consul server down -12:21.987 [INFO] TestAgent: shutdown complete -12:21.987 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=tcp -12:21.987 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=udp -12:21.987 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15107 network=tcp -12:22.487 [INFO] TestAgent: Waiting for endpoints to shut down -12:22.487 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodListCommand_JSON (1.14s) +2022-12-04T05:48:14.151Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:14.171Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:14.173 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5f5da7d7-961e-6854-d0ee-ec95eb9af2cf Address:127.0.0.1:10006}]" +48:14.173 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5f5da7d7-961e-6854-d0ee-ec95eb9af2cf.dc1 127.0.0.1 +48:14.174 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5f5da7d7-961e-6854-d0ee-ec95eb9af2cf 127.0.0.1 +2022-12-04T05:48:14.174Z [INFO] agent.router: Initializing LAN area manager +48:14.174 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=udp +48:14.174 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10006 [Follower]" leader= +48:14.174 [INFO] TestAgent.server: Adding LAN server: server="Node-5f5da7d7-961e-6854-d0ee-ec95eb9af2cf (Addr: tcp/127.0.0.1:10006) (DC: dc1)" +48:14.174 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5f5da7d7-961e-6854-d0ee-ec95eb9af2cf.dc1 area=wan +48:14.174 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=tcp +48:14.175 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10002 network=tcp +48:14.175 [INFO] TestAgent: started state syncer +48:14.175 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10007 network=tcp +48:14.226 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:14.239 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:14.239 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10006 [Candidate]" term=2 +48:14.239 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:14.239 [DEBUG] TestAgent.server.raft: vote granted: from=5f5da7d7-961e-6854-d0ee-ec95eb9af2cf term=2 tally=1 +48:14.239 [INFO] TestAgent.server.raft: election won: tally=1 +48:14.239 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10006 [Leader]" +48:14.239 [INFO] TestAgent.server: cluster leadership acquired +48:14.240 [INFO] TestAgent.server: initializing acls +48:14.240 [INFO] TestAgent.server: New leader elected: payload=Node-5f5da7d7-961e-6854-d0ee-ec95eb9af2cf +48:14.240 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:14.240 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:14.240 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:14.241 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:14.241 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:14.241 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:14.241 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5f5da7d7-961e-6854-d0ee-ec95eb9af2cf +48:14.241 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5f5da7d7-961e-6854-d0ee-ec95eb9af2cf.dc1 +48:14.242 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:14.242 [INFO] TestAgent.server: Updating LAN server: server="Node-5f5da7d7-961e-6854-d0ee-ec95eb9af2cf (Addr: tcp/127.0.0.1:10006) (DC: dc1)" +48:14.242 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5f5da7d7-961e-6854-d0ee-ec95eb9af2cf.dc1 area=wan +48:14.249 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:14.249 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:14.249 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:14.249 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:14.249 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:14.249 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5f5da7d7-961e-6854-d0ee-ec95eb9af2cf +48:14.249 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5f5da7d7-961e-6854-d0ee-ec95eb9af2cf +48:14.250 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5f5da7d7-961e-6854-d0ee-ec95eb9af2cf +48:14.250 [INFO] TestAgent.server: federation state anti-entropy synced +48:14.284 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:14.284 [INFO] TestAgent: Synced node info +48:14.291 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:56248 latency=1.650931ms +48:14.292 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:56248 latency=638.911µs +48:14.293 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:56248 latency=374.862µs +48:14.293 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:56248 latency=322.693µs +48:14.294 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:56248 latency=324.007µs +48:14.305 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-methods from=127.0.0.1:56250 latency=1.287849ms +48:14.305 [INFO] TestAgent: Requesting shutdown +48:14.305 [INFO] TestAgent.server: shutting down server +48:14.305 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:14.305 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:14.305 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:14.305 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:14.305 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:14.305 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:14.305 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:14.305 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:14.305 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:14.305 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:14.305 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:14.306 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:14.306 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:14.306 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:14.306 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:14.306Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:14.306Z [INFO] agent.router.manager: shutting down +48:14.306 [INFO] TestAgent: consul server down +48:14.306 [INFO] TestAgent: shutdown complete +48:14.306 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=tcp +48:14.306 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=udp +48:14.306 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10002 network=tcp +48:14.806 [INFO] TestAgent: Waiting for endpoints to shut down +48:14.806 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodListCommand_JSON (0.71s) --- PASS: TestAuthMethodListCommand_JSON/found_some (0.01s) -2021-10-31T23:12:21.512Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:21.554Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:21.588 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d01d16e0-97d8-fbd1-daa3-122c7b936375 Address:127.0.0.1:15118}]" -12:21.589 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d01d16e0-97d8-fbd1-daa3-122c7b936375.dc1 127.0.0.1 -12:21.589 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15118 [Follower]" leader= -12:21.604 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d01d16e0-97d8-fbd1-daa3-122c7b936375 127.0.0.1 -2021-10-31T23:12:21.604Z [INFO] agent.router: Initializing LAN area manager -12:21.605 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=udp -12:21.606 [INFO] TestAgent.server: Adding LAN server: server="Node-d01d16e0-97d8-fbd1-daa3-122c7b936375 (Addr: tcp/127.0.0.1:15118) (DC: dc1)" -12:21.606 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d01d16e0-97d8-fbd1-daa3-122c7b936375.dc1 area=wan -12:21.606 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=tcp -12:21.607 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15114 network=tcp -12:21.607 [INFO] TestAgent: started state syncer -12:21.607 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15119 network=tcp -12:21.641 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:21.641 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15118 [Candidate]" term=2 -12:21.642 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:21.642 [DEBUG] TestAgent.server.raft: vote granted: from=d01d16e0-97d8-fbd1-daa3-122c7b936375 term=2 tally=1 -12:21.642 [INFO] TestAgent.server.raft: election won: tally=1 -12:21.642 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15118 [Leader]" -12:21.642 [INFO] TestAgent.server: cluster leadership acquired -12:21.642 [INFO] TestAgent.server: initializing acls -12:21.642 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:21.642 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:21.643 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:21.643 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:21.643 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:21.643 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:21.643 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d01d16e0-97d8-fbd1-daa3-122c7b936375 -12:21.643 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d01d16e0-97d8-fbd1-daa3-122c7b936375.dc1 -12:21.644 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:21.645 [INFO] TestAgent.server: New leader elected: payload=Node-d01d16e0-97d8-fbd1-daa3-122c7b936375 -12:21.646 [INFO] TestAgent.server: Updating LAN server: server="Node-d01d16e0-97d8-fbd1-daa3-122c7b936375 (Addr: tcp/127.0.0.1:15118) (DC: dc1)" -12:21.646 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d01d16e0-97d8-fbd1-daa3-122c7b936375.dc1 area=wan -12:21.710 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:21.710 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:21.710 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:21.710 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:21.710 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:21.711 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d01d16e0-97d8-fbd1-daa3-122c7b936375 -12:21.711 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d01d16e0-97d8-fbd1-daa3-122c7b936375 -12:21.722 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d01d16e0-97d8-fbd1-daa3-122c7b936375 -12:21.728 [INFO] TestAgent.server: federation state anti-entropy synced -12:22.035 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:22.035 [INFO] TestAgent: Synced node info -12:22.036 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-methods from=127.0.0.1:40326 latency=613.453µs -12:22.038 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:40328 latency=684.418µs -12:22.046 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:40328 latency=815.066µs -12:22.049 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:40328 latency=578.466µs -12:22.056 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:40328 latency=4.501598ms -12:22.071 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:40328 latency=775.871µs -12:22.086 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-methods from=127.0.0.1:40330 latency=13.754082ms -12:22.087 [INFO] TestAgent: Requesting shutdown -12:22.087 [INFO] TestAgent.server: shutting down server -12:22.087 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:22.087 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:22.087 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:22.087 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:22.087 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:22.087 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:22.087 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:22.087 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:22.087 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:22.087 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:22.087 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:22.087 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:22.087 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:22.087 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:22.087 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:22.087Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:22.087Z [INFO] agent.router.manager: shutting down -12:22.087 [INFO] TestAgent: consul server down -12:22.087 [INFO] TestAgent: shutdown complete -12:22.087 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=tcp -12:22.087 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=udp -12:22.088 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15114 network=tcp -12:22.589 [INFO] TestAgent: Waiting for endpoints to shut down -12:22.589 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodListCommand (1.24s) +2022-12-04T05:48:14.146Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:14.163Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:14.168 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c0bde1aa-b165-6fc1-b815-41771e5fb05f Address:127.0.0.1:10013}]" +48:14.168 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10013 [Follower]" leader= +48:14.169 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c0bde1aa-b165-6fc1-b815-41771e5fb05f.dc1 127.0.0.1 +48:14.169 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c0bde1aa-b165-6fc1-b815-41771e5fb05f 127.0.0.1 +2022-12-04T05:48:14.169Z [INFO] agent.router: Initializing LAN area manager +48:14.170 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10008 network=udp +48:14.170 [INFO] TestAgent.server: Adding LAN server: server="Node-c0bde1aa-b165-6fc1-b815-41771e5fb05f (Addr: tcp/127.0.0.1:10013) (DC: dc1)" +48:14.170 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c0bde1aa-b165-6fc1-b815-41771e5fb05f.dc1 area=wan +48:14.170 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10008 network=tcp +48:14.171 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10009 network=tcp +48:14.171 [INFO] TestAgent: started state syncer +48:14.171 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10014 network=tcp +48:14.220 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:14.220 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:14.220 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10013 [Candidate]" term=2 +48:14.221 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:14.221 [DEBUG] TestAgent.server.raft: vote granted: from=c0bde1aa-b165-6fc1-b815-41771e5fb05f term=2 tally=1 +48:14.221 [INFO] TestAgent.server.raft: election won: tally=1 +48:14.221 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10013 [Leader]" +48:14.221 [INFO] TestAgent.server: cluster leadership acquired +48:14.221 [INFO] TestAgent.server: initializing acls +48:14.221 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:14.221 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:14.222 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:14.222 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:14.222 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:14.222 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:14.222 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c0bde1aa-b165-6fc1-b815-41771e5fb05f +48:14.222 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c0bde1aa-b165-6fc1-b815-41771e5fb05f.dc1 +48:14.228 [INFO] TestAgent.server: New leader elected: payload=Node-c0bde1aa-b165-6fc1-b815-41771e5fb05f +48:14.228 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c0bde1aa-b165-6fc1-b815-41771e5fb05f.dc1 area=wan +48:14.228 [INFO] TestAgent.server: Updating LAN server: server="Node-c0bde1aa-b165-6fc1-b815-41771e5fb05f (Addr: tcp/127.0.0.1:10013) (DC: dc1)" +48:14.230 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:14.246 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:14.246 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:14.246 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:14.246 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:14.246 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:14.247 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c0bde1aa-b165-6fc1-b815-41771e5fb05f +48:14.247 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c0bde1aa-b165-6fc1-b815-41771e5fb05f +48:14.247 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c0bde1aa-b165-6fc1-b815-41771e5fb05f +48:14.248 [INFO] TestAgent.server: federation state anti-entropy synced +48:14.312 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:14.313 [INFO] TestAgent: Synced node info +48:14.313 [DEBUG] TestAgent: Node info in sync +48:14.611 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-methods from=127.0.0.1:36468 latency=571.917µs +48:14.612 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36470 latency=892.486µs +48:14.613 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36470 latency=451.894µs +48:14.614 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36470 latency=335.536µs +48:14.615 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36470 latency=350.001µs +48:14.615 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:36470 latency=283.43µs +48:14.617 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-methods from=127.0.0.1:36472 latency=211.074µs +48:14.617 [INFO] TestAgent: Requesting shutdown +48:14.617 [INFO] TestAgent.server: shutting down server +48:14.617 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:14.617 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:14.617 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:14.617 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:14.617 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:14.617 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:14.617 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:14.617 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:14.617 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:14.617 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:14.617 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:14.617 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:14.617 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:14.617 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:14.617 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +2022-12-04T05:48:14.617Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:14.617Z [INFO] agent.router.manager: shutting down +48:14.617 [INFO] TestAgent: consul server down +48:14.617 [INFO] TestAgent: shutdown complete +48:14.617 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10008 network=tcp +48:14.617 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10008 network=udp +48:14.617 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10009 network=tcp +48:15.118 [INFO] TestAgent: Waiting for endpoints to shut down +48:15.118 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodListCommand (1.03s) --- PASS: TestAuthMethodListCommand/found_none (0.00s) - --- PASS: TestAuthMethodListCommand/found_some (0.02s) + --- PASS: TestAuthMethodListCommand/found_some (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/authmethod/list 1.920s +ok github.com/hashicorp/consul/command/acl/authmethod/list 1.084s === RUN TestAuthMethodReadCommand_noTabs === PAUSE TestAuthMethodReadCommand_noTabs === RUN TestAuthMethodReadCommand @@ -73175,171 +73304,171 @@ === RUN TestAuthMethodReadCommand_JSON === PAUSE TestAuthMethodReadCommand_JSON === CONT TestAuthMethodReadCommand_noTabs ---- PASS: TestAuthMethodReadCommand_noTabs (0.00s) === CONT TestAuthMethodReadCommand_JSON -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestAuthMethodReadCommand +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestAuthMethodReadCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestAuthMethodReadCommand/name_required === RUN TestAuthMethodReadCommand/not_found -=== RUN TestAuthMethodReadCommand_JSON/read_by_name === RUN TestAuthMethodReadCommand/read_by_name -2021-10-31T23:12:22.002Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:22.043Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:22.050 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9fc1b57c-0b65-fc86-cc83-93c80b5b33d9 Address:127.0.0.1:28384}]" -12:22.051 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9fc1b57c-0b65-fc86-cc83-93c80b5b33d9.dc1 127.0.0.1 -12:22.051 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28384 [Follower]" leader= -12:22.051 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9fc1b57c-0b65-fc86-cc83-93c80b5b33d9 127.0.0.1 -2021-10-31T23:12:22.051Z [INFO] agent.router: Initializing LAN area manager -12:22.051 [INFO] TestAgent.server: Adding LAN server: server="Node-9fc1b57c-0b65-fc86-cc83-93c80b5b33d9 (Addr: tcp/127.0.0.1:28384) (DC: dc1)" -12:22.051 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9fc1b57c-0b65-fc86-cc83-93c80b5b33d9.dc1 area=wan -12:22.052 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=udp -12:22.052 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=tcp -12:22.052 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28380 network=tcp -12:22.052 [INFO] TestAgent: started state syncer -12:22.052 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28385 network=tcp -12:22.101 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:22.101 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28384 [Candidate]" term=2 -12:22.102 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -12:22.102 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:22.102 [DEBUG] TestAgent.server.raft: vote granted: from=9fc1b57c-0b65-fc86-cc83-93c80b5b33d9 term=2 tally=1 -12:22.102 [INFO] TestAgent.server.raft: election won: tally=1 -12:22.102 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28384 [Leader]" -12:22.102 [INFO] TestAgent.server: cluster leadership acquired -12:22.102 [INFO] TestAgent.server: initializing acls -12:22.103 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:22.103 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:22.103 [INFO] TestAgent.server: New leader elected: payload=Node-9fc1b57c-0b65-fc86-cc83-93c80b5b33d9 -12:22.103 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:22.103 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:22.103 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:22.103 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:22.104 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9fc1b57c-0b65-fc86-cc83-93c80b5b33d9 -12:22.104 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9fc1b57c-0b65-fc86-cc83-93c80b5b33d9.dc1 -12:22.105 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:22.117 [INFO] TestAgent.server: Updating LAN server: server="Node-9fc1b57c-0b65-fc86-cc83-93c80b5b33d9 (Addr: tcp/127.0.0.1:28384) (DC: dc1)" -12:22.118 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9fc1b57c-0b65-fc86-cc83-93c80b5b33d9.dc1 area=wan -12:22.146 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:22.146 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:22.146 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:22.146 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:22.146 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:22.147 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9fc1b57c-0b65-fc86-cc83-93c80b5b33d9 -12:22.147 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9fc1b57c-0b65-fc86-cc83-93c80b5b33d9 -12:22.148 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9fc1b57c-0b65-fc86-cc83-93c80b5b33d9 -12:22.148 [INFO] TestAgent.server: federation state anti-entropy synced -12:22.176 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:45776 latency=1.696168ms -12:22.178 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-c2295435-967b-fe1f-2964-9504db9b41b0 from=127.0.0.1:45780 latency=456.088µs -12:22.197 [INFO] TestAgent: Requesting shutdown -12:22.198 [INFO] TestAgent.server: shutting down server -12:22.198 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:22.198 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:22.198 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:22.198 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:22.198 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:22.198 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:22.198 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:22.198 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:22.198 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:22.198 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:22.198 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:22.198 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:22.198 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:22.198 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:22.198 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:22.198 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:22.198Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:22.198Z [INFO] agent.router.manager: shutting down -12:22.198 [INFO] TestAgent: consul server down -12:22.198 [INFO] TestAgent: shutdown complete -12:22.198 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=tcp -12:22.198 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=udp -12:22.198 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28380 network=tcp -12:22.700 [INFO] TestAgent: Waiting for endpoints to shut down -12:22.700 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodReadCommand_JSON (0.81s) - --- PASS: TestAuthMethodReadCommand_JSON/read_by_name (0.02s) -2021-10-31T23:12:21.992Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:22.023Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:22.031 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:773fb95e-8559-4109-d6d5-24b9441cf29a Address:127.0.0.1:28391}]" -12:22.031 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-773fb95e-8559-4109-d6d5-24b9441cf29a.dc1 127.0.0.1 -12:22.032 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-773fb95e-8559-4109-d6d5-24b9441cf29a 127.0.0.1 -2021-10-31T23:12:22.032Z [INFO] agent.router: Initializing LAN area manager -12:22.032 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28386 network=udp -12:22.033 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28391 [Follower]" leader= -12:22.034 [INFO] TestAgent.server: Adding LAN server: server="Node-773fb95e-8559-4109-d6d5-24b9441cf29a (Addr: tcp/127.0.0.1:28391) (DC: dc1)" -12:22.034 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-773fb95e-8559-4109-d6d5-24b9441cf29a.dc1 area=wan -12:22.034 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28386 network=tcp -12:22.035 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28387 network=tcp -12:22.035 [INFO] TestAgent: started state syncer -12:22.035 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28392 network=tcp -12:22.088 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -12:22.103 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:22.103 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28391 [Candidate]" term=2 -12:22.103 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:22.103 [DEBUG] TestAgent.server.raft: vote granted: from=773fb95e-8559-4109-d6d5-24b9441cf29a term=2 tally=1 -12:22.103 [INFO] TestAgent.server.raft: election won: tally=1 -12:22.103 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28391 [Leader]" -12:22.103 [INFO] TestAgent.server: cluster leadership acquired -12:22.103 [INFO] TestAgent.server: initializing acls -12:22.103 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:22.103 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:22.104 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:22.104 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:22.104 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:22.104 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:22.104 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-773fb95e-8559-4109-d6d5-24b9441cf29a -12:22.104 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-773fb95e-8559-4109-d6d5-24b9441cf29a.dc1 -12:22.117 [INFO] TestAgent.server: New leader elected: payload=Node-773fb95e-8559-4109-d6d5-24b9441cf29a -12:22.117 [INFO] TestAgent.server: Updating LAN server: server="Node-773fb95e-8559-4109-d6d5-24b9441cf29a (Addr: tcp/127.0.0.1:28391) (DC: dc1)" -12:22.117 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-773fb95e-8559-4109-d6d5-24b9441cf29a.dc1 area=wan -12:22.118 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:22.147 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:22.147 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:22.147 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:22.147 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:22.147 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:22.148 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-773fb95e-8559-4109-d6d5-24b9441cf29a -12:22.148 [INFO] TestAgent.server: member joined, marking health alive: member=Node-773fb95e-8559-4109-d6d5-24b9441cf29a -12:22.148 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-773fb95e-8559-4109-d6d5-24b9441cf29a -12:22.148 [INFO] TestAgent.server: federation state anti-entropy synced -12:22.175 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/notfound from=127.0.0.1:47348 latency=682.133µs -12:22.198 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:47352 latency=21.865226ms -12:22.212 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-9bd58d7a-9d97-71ca-dfed-ec3124d588b3 from=127.0.0.1:47356 latency=1.180347ms -12:22.234 [INFO] TestAgent: Requesting shutdown -12:22.234 [INFO] TestAgent.server: shutting down server -12:22.234 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:22.234 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:22.234 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:22.234 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:22.234 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:22.234 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:22.234 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:22.234 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:22.234 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:22.234 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:22.234 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:22.234 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:22.234 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:22.234 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:22.234 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:22.234 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:22.235Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:22.235Z [INFO] agent.router.manager: shutting down -12:22.235 [INFO] TestAgent: consul server down -12:22.235 [INFO] TestAgent: shutdown complete -12:22.235 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28386 network=tcp -12:22.235 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28386 network=udp -12:22.235 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28387 network=tcp -12:22.737 [INFO] TestAgent: Waiting for endpoints to shut down -12:22.738 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodReadCommand (0.85s) +=== RUN TestAuthMethodReadCommand_JSON/read_by_name +2022-12-04T05:48:14.241Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:14.262Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:14.265 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0f23e585-a222-625f-0c01-ca9826f13ed6 Address:127.0.0.1:11034}]" +48:14.265 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11034 [Follower]" leader= +48:14.265 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0f23e585-a222-625f-0c01-ca9826f13ed6.dc1 127.0.0.1 +48:14.267 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0f23e585-a222-625f-0c01-ca9826f13ed6 127.0.0.1 +2022-12-04T05:48:14.267Z [INFO] agent.router: Initializing LAN area manager +48:14.267 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11029 network=udp +48:14.267 [INFO] TestAgent.server: Adding LAN server: server="Node-0f23e585-a222-625f-0c01-ca9826f13ed6 (Addr: tcp/127.0.0.1:11034) (DC: dc1)" +48:14.267 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0f23e585-a222-625f-0c01-ca9826f13ed6.dc1 area=wan +48:14.267 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11029 network=tcp +48:14.268 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11030 network=tcp +48:14.268 [INFO] TestAgent: started state syncer +48:14.268 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11035 network=tcp +48:14.304 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:14.304 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11034 [Candidate]" term=2 +48:14.304 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:14.304 [DEBUG] TestAgent.server.raft: vote granted: from=0f23e585-a222-625f-0c01-ca9826f13ed6 term=2 tally=1 +48:14.304 [INFO] TestAgent.server.raft: election won: tally=1 +48:14.304 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11034 [Leader]" +48:14.304 [INFO] TestAgent.server: cluster leadership acquired +48:14.305 [INFO] TestAgent.server: New leader elected: payload=Node-0f23e585-a222-625f-0c01-ca9826f13ed6 +48:14.305 [INFO] TestAgent.server: initializing acls +48:14.305 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:14.305 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:14.305 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:14.306 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:14.306 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:14.306 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:14.306 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0f23e585-a222-625f-0c01-ca9826f13ed6 +48:14.306 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0f23e585-a222-625f-0c01-ca9826f13ed6.dc1 +48:14.306 [INFO] TestAgent.server: Updating LAN server: server="Node-0f23e585-a222-625f-0c01-ca9826f13ed6 (Addr: tcp/127.0.0.1:11034) (DC: dc1)" +48:14.306 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0f23e585-a222-625f-0c01-ca9826f13ed6.dc1 area=wan +48:14.306 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:14.322 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:14.322 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:14.322 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:14.322 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:14.322 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:14.322 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0f23e585-a222-625f-0c01-ca9826f13ed6 +48:14.322 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0f23e585-a222-625f-0c01-ca9826f13ed6 +48:14.323 [INFO] TestAgent.server: federation state anti-entropy synced +48:14.323 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0f23e585-a222-625f-0c01-ca9826f13ed6 +48:14.376 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:14.377 [INFO] TestAgent: Synced node info +48:14.385 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/notfound from=127.0.0.1:53410 latency=587.643µs +48:14.387 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:53412 latency=888.968µs +48:14.392 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-38abfce9-7139-0dd6-be5f-9c792a362b69 from=127.0.0.1:53414 latency=3.471583ms +48:14.395 [INFO] TestAgent: Requesting shutdown +48:14.395 [INFO] TestAgent.server: shutting down server +48:14.395 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:14.395 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:14.395 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:14.395 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:14.395 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:14.395 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:14.395 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:14.395 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:14.395 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:14.395 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:14.395 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:14.395 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:14.395 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:14.395 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:14.395 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:14.396Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:14.396Z [INFO] agent.router.manager: shutting down +48:14.396 [INFO] TestAgent: consul server down +48:14.396 [INFO] TestAgent: shutdown complete +48:14.396 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11029 network=tcp +48:14.396 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11029 network=udp +48:14.396 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11030 network=tcp +48:14.896 [INFO] TestAgent: Waiting for endpoints to shut down +48:14.896 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodReadCommand (0.70s) --- PASS: TestAuthMethodReadCommand/name_required (0.00s) --- PASS: TestAuthMethodReadCommand/not_found (0.00s) - --- PASS: TestAuthMethodReadCommand/read_by_name (0.06s) + --- PASS: TestAuthMethodReadCommand/read_by_name (0.01s) +2022-12-04T05:48:14.242Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:14.259Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:14.261 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:eae7779f-1412-b767-d839-6c66e7a895a8 Address:127.0.0.1:11027}]" +48:14.261 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11027 [Follower]" leader= +48:14.262 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-eae7779f-1412-b767-d839-6c66e7a895a8.dc1 127.0.0.1 +48:14.262 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-eae7779f-1412-b767-d839-6c66e7a895a8 127.0.0.1 +2022-12-04T05:48:14.262Z [INFO] agent.router: Initializing LAN area manager +48:14.263 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-eae7779f-1412-b767-d839-6c66e7a895a8.dc1 area=wan +48:14.263 [INFO] TestAgent.server: Adding LAN server: server="Node-eae7779f-1412-b767-d839-6c66e7a895a8 (Addr: tcp/127.0.0.1:11027) (DC: dc1)" +48:14.263 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=udp +48:14.263 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=tcp +48:14.263 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11023 network=tcp +48:14.263 [INFO] TestAgent: started state syncer +48:14.263 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11028 network=tcp +48:14.314 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:14.321 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:14.321 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11027 [Candidate]" term=2 +48:14.322 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:14.322 [DEBUG] TestAgent.server.raft: vote granted: from=eae7779f-1412-b767-d839-6c66e7a895a8 term=2 tally=1 +48:14.322 [INFO] TestAgent.server.raft: election won: tally=1 +48:14.322 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11027 [Leader]" +48:14.322 [INFO] TestAgent.server: cluster leadership acquired +48:14.322 [INFO] TestAgent.server: New leader elected: payload=Node-eae7779f-1412-b767-d839-6c66e7a895a8 +48:14.322 [INFO] TestAgent.server: initializing acls +48:14.322 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:14.322 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:14.323 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:14.323 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:14.323 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:14.323 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:14.323 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-eae7779f-1412-b767-d839-6c66e7a895a8 +48:14.323 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-eae7779f-1412-b767-d839-6c66e7a895a8.dc1 +48:14.323 [INFO] TestAgent.server: Updating LAN server: server="Node-eae7779f-1412-b767-d839-6c66e7a895a8 (Addr: tcp/127.0.0.1:11027) (DC: dc1)" +48:14.323 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-eae7779f-1412-b767-d839-6c66e7a895a8.dc1 area=wan +48:14.324 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:14.325 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:14.325 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:14.325 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:14.325 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:14.325 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:14.325 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eae7779f-1412-b767-d839-6c66e7a895a8 +48:14.325 [INFO] TestAgent.server: member joined, marking health alive: member=Node-eae7779f-1412-b767-d839-6c66e7a895a8 +48:14.325 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eae7779f-1412-b767-d839-6c66e7a895a8 +48:14.325 [INFO] TestAgent.server: federation state anti-entropy synced +48:14.415 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:51726 latency=950.306µs +48:14.418 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-a37d77c9-406e-bee0-faff-62b2c0ffad93 from=127.0.0.1:51728 latency=1.315111ms +48:14.419 [INFO] TestAgent: Requesting shutdown +48:14.419 [INFO] TestAgent.server: shutting down server +48:14.419 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:14.419 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:14.419 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:14.419 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:14.419 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:14.419 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:14.419 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:14.419 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:14.419 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:14.419 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:14.419 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:14.419 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:14.419 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:14.419 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:14.419 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:14.419 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:14.419Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:14.419Z [INFO] agent.router.manager: shutting down +48:14.419 [INFO] TestAgent: consul server down +48:14.419 [INFO] TestAgent: shutdown complete +48:14.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=tcp +48:14.419 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=udp +48:14.419 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11023 network=tcp +48:14.919 [INFO] TestAgent: Waiting for endpoints to shut down +48:14.919 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodReadCommand_JSON (0.72s) + --- PASS: TestAuthMethodReadCommand_JSON/read_by_name (0.01s) PASS -ok github.com/hashicorp/consul/command/acl/authmethod/read 1.194s +ok github.com/hashicorp/consul/command/acl/authmethod/read 0.769s === RUN TestAuthMethodUpdateCommand_noTabs === PAUSE TestAuthMethodUpdateCommand_noTabs === RUN TestAuthMethodUpdateCommand @@ -73355,568 +73484,578 @@ === RUN TestAuthMethodUpdateCommand_config === PAUSE TestAuthMethodUpdateCommand_config === CONT TestAuthMethodUpdateCommand_noTabs -=== CONT TestAuthMethodUpdateCommand_k8s_noMerge -=== CONT TestAuthMethodUpdateCommand_k8s -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestAuthMethodUpdateCommand_noTabs (0.00s) === CONT TestAuthMethodUpdateCommand_config +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +=== CONT TestAuthMethodUpdateCommand_noMerge === CONT TestAuthMethodUpdateCommand_JSON ---- PASS: TestAuthMethodUpdateCommand_noTabs (0.01s) +=== CONT TestAuthMethodUpdateCommand_k8s_noMerge +=== CONT TestAuthMethodUpdateCommand_k8s === CONT TestAuthMethodUpdateCommand -=== CONT TestAuthMethodUpdateCommand_noMerge [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -=== RUN TestAuthMethodUpdateCommand_JSON/update_without_name -=== RUN TestAuthMethodUpdateCommand_JSON/update_all_fields -=== RUN TestAuthMethodUpdateCommand_config/config_file +=== RUN TestAuthMethodUpdateCommand_k8s/update_all_fields === RUN TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_host -=== RUN TestAuthMethodUpdateCommand/update_without_name -=== RUN TestAuthMethodUpdateCommand/update_nonexistent_method -=== RUN TestAuthMethodUpdateCommand/update_all_fields === RUN TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_ca_cert -=== RUN TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_jwt -=== RUN TestAuthMethodUpdateCommand_config/config_stdin -=== RUN TestAuthMethodUpdateCommand/update_config_field_and_prove_no_merging_happens -=== RUN TestAuthMethodUpdateCommand_k8s_noMerge/update_all_fields -=== RUN TestAuthMethodUpdateCommand_config/config_string -=== RUN TestAuthMethodUpdateCommand_config/config_with_no_merge -=== RUN TestAuthMethodUpdateCommand_k8s_noMerge/update_all_fields_with_cert_file === RUN TestAuthMethodUpdateCommand_noMerge/update_without_name === RUN TestAuthMethodUpdateCommand_noMerge/update_nonexistent_method === RUN TestAuthMethodUpdateCommand_noMerge/update_all_fields -=== RUN TestAuthMethodUpdateCommand_k8s/update_all_fields +=== RUN TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_jwt +=== RUN TestAuthMethodUpdateCommand_config/config_file === RUN TestAuthMethodUpdateCommand_k8s/update_all_fields_with_cert_file +=== RUN TestAuthMethodUpdateCommand_k8s_noMerge/update_all_fields +=== RUN TestAuthMethodUpdateCommand_config/config_stdin +=== RUN TestAuthMethodUpdateCommand_config/config_string === RUN TestAuthMethodUpdateCommand_k8s/update_all_fields_but_k8s_host +=== RUN TestAuthMethodUpdateCommand_k8s_noMerge/update_all_fields_with_cert_file +=== RUN TestAuthMethodUpdateCommand_config/config_with_no_merge +=== RUN TestAuthMethodUpdateCommand_JSON/update_without_name +=== RUN TestAuthMethodUpdateCommand_JSON/update_all_fields === RUN TestAuthMethodUpdateCommand_k8s/update_all_fields_but_k8s_ca_cert === RUN TestAuthMethodUpdateCommand_k8s/update_all_fields_but_k8s_jwt -2021-10-31T23:12:35.314Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:35.432Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:35.433 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6261b289-18c8-9ac4-5c9d-29332ceeaaef Address:127.0.0.1:11048}]" -12:35.449 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11048 [Follower]" leader= -12:35.458 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6261b289-18c8-9ac4-5c9d-29332ceeaaef.dc1 127.0.0.1 -12:35.473 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6261b289-18c8-9ac4-5c9d-29332ceeaaef 127.0.0.1 -2021-10-31T23:12:35.473Z [INFO] agent.router: Initializing LAN area manager -12:35.473 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11043 network=udp -12:35.474 [INFO] TestAgent.server: Adding LAN server: server="Node-6261b289-18c8-9ac4-5c9d-29332ceeaaef (Addr: tcp/127.0.0.1:11048) (DC: dc1)" -12:35.474 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6261b289-18c8-9ac4-5c9d-29332ceeaaef.dc1 area=wan -12:35.475 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11043 network=tcp -12:35.481 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11044 network=tcp -12:35.486 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:35.486 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11048 [Candidate]" term=2 -12:35.486 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:35.486 [DEBUG] TestAgent.server.raft: vote granted: from=6261b289-18c8-9ac4-5c9d-29332ceeaaef term=2 tally=1 -12:35.486 [INFO] TestAgent.server.raft: election won: tally=1 -12:35.486 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11048 [Leader]" -12:35.486 [INFO] TestAgent.server: cluster leadership acquired -12:35.486 [INFO] TestAgent.server: initializing acls -12:35.487 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:35.487 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:35.487 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:35.487 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:35.487 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:35.487 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:35.487 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-6261b289-18c8-9ac4-5c9d-29332ceeaaef -12:35.487 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-6261b289-18c8-9ac4-5c9d-29332ceeaaef.dc1 -12:35.488 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:35.488 [INFO] TestAgent.server: New leader elected: payload=Node-6261b289-18c8-9ac4-5c9d-29332ceeaaef -12:35.488 [INFO] TestAgent.server: Updating LAN server: server="Node-6261b289-18c8-9ac4-5c9d-29332ceeaaef (Addr: tcp/127.0.0.1:11048) (DC: dc1)" -12:35.488 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-6261b289-18c8-9ac4-5c9d-29332ceeaaef.dc1 area=wan -12:35.533 [INFO] TestAgent: started state syncer -12:35.534 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:35.534 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:35.534 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:35.534 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:35.534 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:35.534 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6261b289-18c8-9ac4-5c9d-29332ceeaaef -12:35.534 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6261b289-18c8-9ac4-5c9d-29332ceeaaef -12:35.536 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11049 network=tcp -12:35.536 [INFO] TestAgent: Synced node info -12:35.536 [DEBUG] TestAgent: Node info in sync -12:35.550 [INFO] TestAgent.server: federation state anti-entropy synced -12:35.576 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6261b289-18c8-9ac4-5c9d-29332ceeaaef -12:35.610 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:50922 latency=32.875754ms -12:35.633 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-7ebfd982-7509-1f42-1135-65d91abbb342 from=127.0.0.1:50938 latency=19.355597ms -12:35.640 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test-7ebfd982-7509-1f42-1135-65d91abbb342 from=127.0.0.1:50938 latency=1.587937ms -12:35.642 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-7ebfd982-7509-1f42-1135-65d91abbb342 from=127.0.0.1:50922 latency=659.058µs -12:35.642 [INFO] TestAgent: Requesting shutdown -12:35.642 [INFO] TestAgent.server: shutting down server -12:35.642 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:35.642 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:35.642 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:35.642 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:35.642 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:35.642 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:35.642 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:35.642 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:35.642 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:35.642 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:35.642 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:35.642 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:35.642 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:35.642 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:35.643 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:35.643Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:35.643Z [INFO] agent.router.manager: shutting down -12:35.643 [INFO] TestAgent: consul server down -12:35.643 [INFO] TestAgent: shutdown complete -12:35.643 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11043 network=tcp -12:35.643 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11043 network=udp -12:35.643 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11044 network=tcp -12:36.145 [INFO] TestAgent: Waiting for endpoints to shut down -12:36.145 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodUpdateCommand_JSON (1.06s) - --- PASS: TestAuthMethodUpdateCommand_JSON/update_without_name (0.00s) - --- PASS: TestAuthMethodUpdateCommand_JSON/update_all_fields (0.07s) -2021-10-31T23:12:35.359Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:35.433Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:35.473 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:60918f4e-ad8f-2a72-be51-591e879d2211 Address:127.0.0.1:11055}]" -12:35.473 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-60918f4e-ad8f-2a72-be51-591e879d2211.dc1 127.0.0.1 -12:35.474 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11055 [Follower]" leader= -12:35.474 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-60918f4e-ad8f-2a72-be51-591e879d2211 127.0.0.1 -2021-10-31T23:12:35.474Z [INFO] agent.router: Initializing LAN area manager -12:35.474 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11050 network=udp -12:35.474 [INFO] TestAgent.server: Adding LAN server: server="Node-60918f4e-ad8f-2a72-be51-591e879d2211 (Addr: tcp/127.0.0.1:11055) (DC: dc1)" -12:35.474 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-60918f4e-ad8f-2a72-be51-591e879d2211.dc1 area=wan -12:35.474 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11050 network=tcp -12:35.476 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11051 network=tcp -12:35.533 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -12:35.533 [INFO] TestAgent: started state syncer -12:35.538 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11056 network=tcp -12:35.539 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:35.546 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11055 [Candidate]" term=2 -12:35.547 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:35.547 [DEBUG] TestAgent.server.raft: vote granted: from=60918f4e-ad8f-2a72-be51-591e879d2211 term=2 tally=1 -12:35.547 [INFO] TestAgent.server.raft: election won: tally=1 -12:35.547 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11055 [Leader]" -12:35.547 [INFO] TestAgent.server: cluster leadership acquired -12:35.547 [INFO] TestAgent.server: New leader elected: payload=Node-60918f4e-ad8f-2a72-be51-591e879d2211 -12:35.547 [INFO] TestAgent.server: initializing acls -12:35.555 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:35.555 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:35.570 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:35.571 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:35.571 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:35.571 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:35.571 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-60918f4e-ad8f-2a72-be51-591e879d2211 -12:35.571 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-60918f4e-ad8f-2a72-be51-591e879d2211.dc1 -12:35.572 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:35.573 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:35.573 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:35.573 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:35.573 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:35.573 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:35.573 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-60918f4e-ad8f-2a72-be51-591e879d2211 -12:35.573 [INFO] TestAgent.server: member joined, marking health alive: member=Node-60918f4e-ad8f-2a72-be51-591e879d2211 -12:35.577 [INFO] TestAgent.server: Updating LAN server: server="Node-60918f4e-ad8f-2a72-be51-591e879d2211 (Addr: tcp/127.0.0.1:11055) (DC: dc1)" -12:35.577 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-60918f4e-ad8f-2a72-be51-591e879d2211.dc1 area=wan -12:35.578 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-60918f4e-ad8f-2a72-be51-591e879d2211 -12:35.584 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-d8f71302-21f8-a8e0-aad7-1d9e723487be from=127.0.0.1:60676 latency=477.686µs -12:35.586 [INFO] TestAgent.server: federation state anti-entropy synced -12:35.587 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:60678 latency=602.306µs -12:35.613 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-8acca1a5-51f9-11a1-f5da-d930ae889032 from=127.0.0.1:60682 latency=477.47µs -12:35.631 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test-8acca1a5-51f9-11a1-f5da-d930ae889032 from=127.0.0.1:60682 latency=668.881µs -12:35.631 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-8acca1a5-51f9-11a1-f5da-d930ae889032 from=127.0.0.1:60678 latency=232.913µs -12:35.633 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-8acca1a5-51f9-11a1-f5da-d930ae889032 from=127.0.0.1:60692 latency=244.914µs -12:35.638 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test-8acca1a5-51f9-11a1-f5da-d930ae889032 from=127.0.0.1:60692 latency=626.743µs -12:35.640 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-8acca1a5-51f9-11a1-f5da-d930ae889032 from=127.0.0.1:60678 latency=1.481727ms -12:35.641 [INFO] TestAgent: Requesting shutdown -12:35.641 [INFO] TestAgent.server: shutting down server -12:35.641 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:35.641 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:35.641 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:35.641 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:35.641 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:35.641 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:35.641 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:35.641 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:35.641 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:35.641 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:35.641 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:35.641 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:35.641 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:35.641 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:35.641 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -12:35.641 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:12:35.642Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:35.642Z [INFO] agent.router.manager: shutting down -12:35.642 [INFO] TestAgent: consul server down -12:35.642 [INFO] TestAgent: shutdown complete -12:35.642 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11050 network=tcp -12:35.642 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11050 network=udp -12:35.642 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11051 network=tcp -12:36.149 [INFO] TestAgent: Waiting for endpoints to shut down -12:36.149 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodUpdateCommand (1.07s) - --- PASS: TestAuthMethodUpdateCommand/update_without_name (0.00s) - --- PASS: TestAuthMethodUpdateCommand/update_nonexistent_method (0.00s) - --- PASS: TestAuthMethodUpdateCommand/update_all_fields (0.04s) - --- PASS: TestAuthMethodUpdateCommand/update_config_field_and_prove_no_merging_happens (0.01s) -2021-10-31T23:12:35.267Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:35.343Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:35.364 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1de8c5ef-c611-1a1c-efba-8025ca415fc2 Address:127.0.0.1:11034}]" -12:35.365 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1de8c5ef-c611-1a1c-efba-8025ca415fc2.dc1 127.0.0.1 -12:35.366 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1de8c5ef-c611-1a1c-efba-8025ca415fc2 127.0.0.1 -2021-10-31T23:12:35.366Z [INFO] agent.router: Initializing LAN area manager -12:35.366 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11029 network=udp -12:35.366 [INFO] TestAgent.server: Adding LAN server: server="Node-1de8c5ef-c611-1a1c-efba-8025ca415fc2 (Addr: tcp/127.0.0.1:11034) (DC: dc1)" -12:35.366 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11034 [Follower]" leader= -12:35.366 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11029 network=tcp -12:35.366 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1de8c5ef-c611-1a1c-efba-8025ca415fc2.dc1 area=wan -12:35.367 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11030 network=tcp -12:35.367 [INFO] TestAgent: started state syncer -12:35.367 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11035 network=tcp -12:35.431 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -12:35.449 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:35.449 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11034 [Candidate]" term=2 -12:35.450 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:35.450 [DEBUG] TestAgent.server.raft: vote granted: from=1de8c5ef-c611-1a1c-efba-8025ca415fc2 term=2 tally=1 -12:35.450 [INFO] TestAgent.server.raft: election won: tally=1 -12:35.450 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11034 [Leader]" -12:35.451 [INFO] TestAgent.server: cluster leadership acquired -12:35.451 [INFO] TestAgent.server: New leader elected: payload=Node-1de8c5ef-c611-1a1c-efba-8025ca415fc2 -12:35.451 [INFO] TestAgent.server: initializing acls -12:35.452 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:35.452 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:35.453 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:35.458 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:35.458 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:35.458 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:35.458 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1de8c5ef-c611-1a1c-efba-8025ca415fc2 -12:35.459 [INFO] TestAgent.server: Updating LAN server: server="Node-1de8c5ef-c611-1a1c-efba-8025ca415fc2 (Addr: tcp/127.0.0.1:11034) (DC: dc1)" -12:35.475 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1de8c5ef-c611-1a1c-efba-8025ca415fc2.dc1 -12:35.476 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:35.477 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1de8c5ef-c611-1a1c-efba-8025ca415fc2.dc1 area=wan -12:35.558 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:35.558 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:35.558 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:35.558 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:35.558 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:35.558 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1de8c5ef-c611-1a1c-efba-8025ca415fc2 -12:35.558 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1de8c5ef-c611-1a1c-efba-8025ca415fc2 -12:35.574 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1de8c5ef-c611-1a1c-efba-8025ca415fc2 -12:35.577 [INFO] TestAgent.server: federation state anti-entropy synced -12:35.589 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:52176 latency=2.171379ms -12:35.627 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/testdae888e0-3f16-5ac6-dcaf-a7d15ca61100 from=127.0.0.1:52192 latency=471.564µs -12:35.630 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/testdae888e0-3f16-5ac6-dcaf-a7d15ca61100 from=127.0.0.1:52192 latency=672.533µs -12:35.631 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/testdae888e0-3f16-5ac6-dcaf-a7d15ca61100 from=127.0.0.1:52176 latency=279.092µs -12:35.632 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:52176 latency=499.221µs -12:35.642 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:35.642 [INFO] TestAgent: Synced node info -12:35.645 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test669b48f6-152a-6bb8-6da5-3eafc663a5da from=127.0.0.1:52198 latency=191.557µs -12:35.645 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test669b48f6-152a-6bb8-6da5-3eafc663a5da from=127.0.0.1:52198 latency=561.998µs -12:35.646 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test669b48f6-152a-6bb8-6da5-3eafc663a5da from=127.0.0.1:52176 latency=272.001µs -12:35.647 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:52176 latency=407.345µs -12:35.670 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/testc034cb27-68dd-b4de-df01-479016bbb36a from=127.0.0.1:52202 latency=15.558396ms -12:35.674 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/testc034cb27-68dd-b4de-df01-479016bbb36a from=127.0.0.1:52202 latency=1.907823ms -12:35.676 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/testc034cb27-68dd-b4de-df01-479016bbb36a from=127.0.0.1:52176 latency=1.187476ms -12:35.686 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:52176 latency=953.083µs -12:35.706 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/testeed6ea0a-939f-5f43-c850-b32a9620eca3 from=127.0.0.1:52206 latency=579.678µs -12:35.708 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/testeed6ea0a-939f-5f43-c850-b32a9620eca3 from=127.0.0.1:52206 latency=987.078µs -12:35.708 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/testeed6ea0a-939f-5f43-c850-b32a9620eca3 from=127.0.0.1:52176 latency=194.633µs -12:35.709 [INFO] TestAgent: Requesting shutdown -12:35.709 [INFO] TestAgent.server: shutting down server -12:35.709 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:35.709 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:35.709 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:35.709 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:35.709 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:35.709 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:35.709 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:35.709 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:35.709 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:35.709 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:35.709 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:35.709 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:35.709 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:35.709 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:35.709 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:35.709Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:35.709Z [INFO] agent.router.manager: shutting down -12:35.714 [INFO] TestAgent: consul server down -12:35.714 [INFO] TestAgent: shutdown complete -12:35.714 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11029 network=tcp -12:35.714 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11029 network=udp -12:35.714 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11030 network=tcp -12:36.214 [INFO] TestAgent: Waiting for endpoints to shut down -12:36.214 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodUpdateCommand_config (1.13s) - --- PASS: TestAuthMethodUpdateCommand_config/config_file (0.05s) - --- PASS: TestAuthMethodUpdateCommand_config/config_stdin (0.01s) - --- PASS: TestAuthMethodUpdateCommand_config/config_string (0.03s) - --- PASS: TestAuthMethodUpdateCommand_config/config_with_no_merge (0.03s) -2021-10-31T23:12:35.271Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:35.393Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:35.432 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:79130523-3927-b8a8-4419-0f60455f0cf1 Address:127.0.0.1:11041}]" -12:35.433 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-79130523-3927-b8a8-4419-0f60455f0cf1.dc1 127.0.0.1 -12:35.433 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-79130523-3927-b8a8-4419-0f60455f0cf1 127.0.0.1 -2021-10-31T23:12:35.433Z [INFO] agent.router: Initializing LAN area manager -12:35.450 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11036 network=tcp -12:35.451 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11041 [Follower]" leader= -12:35.451 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-79130523-3927-b8a8-4419-0f60455f0cf1.dc1 area=wan -12:35.451 [INFO] TestAgent.server: Adding LAN server: server="Node-79130523-3927-b8a8-4419-0f60455f0cf1 (Addr: tcp/127.0.0.1:11041) (DC: dc1)" -12:35.452 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11036 network=udp -12:35.479 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11037 network=tcp -12:35.501 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -12:35.509 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:35.509 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11041 [Candidate]" term=2 -12:35.510 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:35.510 [DEBUG] TestAgent.server.raft: vote granted: from=79130523-3927-b8a8-4419-0f60455f0cf1 term=2 tally=1 -12:35.510 [INFO] TestAgent.server.raft: election won: tally=1 -12:35.510 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11041 [Leader]" -12:35.510 [INFO] TestAgent.server: cluster leadership acquired -12:35.510 [INFO] TestAgent.server: initializing acls -12:35.510 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:35.510 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:35.511 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:35.511 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:35.511 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:35.511 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:35.511 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-79130523-3927-b8a8-4419-0f60455f0cf1 -12:35.511 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-79130523-3927-b8a8-4419-0f60455f0cf1.dc1 -12:35.512 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:35.512 [INFO] TestAgent.server: New leader elected: payload=Node-79130523-3927-b8a8-4419-0f60455f0cf1 -12:35.512 [INFO] TestAgent.server: Updating LAN server: server="Node-79130523-3927-b8a8-4419-0f60455f0cf1 (Addr: tcp/127.0.0.1:11041) (DC: dc1)" -12:35.512 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-79130523-3927-b8a8-4419-0f60455f0cf1.dc1 area=wan -12:35.534 [INFO] TestAgent: started state syncer -12:35.534 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11042 network=tcp -12:35.537 [INFO] TestAgent: Synced node info -12:35.537 [DEBUG] TestAgent: Node info in sync -12:35.574 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:35.574 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:35.574 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:35.574 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:35.574 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:35.574 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-79130523-3927-b8a8-4419-0f60455f0cf1 -12:35.574 [INFO] TestAgent.server: member joined, marking health alive: member=Node-79130523-3927-b8a8-4419-0f60455f0cf1 -12:35.575 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-79130523-3927-b8a8-4419-0f60455f0cf1 -12:35.575 [INFO] TestAgent.server: federation state anti-entropy synced -12:35.584 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:45856 latency=2.545758ms -12:35.605 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-0c0fad6a-8941-f056-5121-f38b5079b952 from=127.0.0.1:45862 latency=16.676248ms -12:35.628 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:45856 latency=17.190238ms -12:35.630 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-d2cdf9ea-d473-738b-2ebd-cfe4e915eaa9 from=127.0.0.1:45872 latency=451.253µs -12:35.633 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:45856 latency=1.569688ms -12:35.644 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-10bd21e6-663b-7a45-f895-61278f3f58bd from=127.0.0.1:45878 latency=396.114µs -12:35.647 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:45856 latency=1.825807ms -12:35.665 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-8d6dc4fd-af08-84ea-6e62-f6861e1fb8f3 from=127.0.0.1:45882 latency=11.097406ms -12:35.706 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-8d6dc4fd-af08-84ea-6e62-f6861e1fb8f3 from=127.0.0.1:45882 latency=20.267801ms -12:35.708 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-8d6dc4fd-af08-84ea-6e62-f6861e1fb8f3 from=127.0.0.1:45856 latency=304.866µs -12:35.714 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:45856 latency=5.878262ms -12:35.719 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-935e098e-4ee9-c7f1-d9eb-89dc36f7d5d9 from=127.0.0.1:45886 latency=519.815µs -12:35.728 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-935e098e-4ee9-c7f1-d9eb-89dc36f7d5d9 from=127.0.0.1:45886 latency=2.712495ms -12:35.738 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-935e098e-4ee9-c7f1-d9eb-89dc36f7d5d9 from=127.0.0.1:45856 latency=441.804µs -12:35.742 [INFO] TestAgent: Requesting shutdown -12:35.742 [INFO] TestAgent.server: shutting down server -12:35.742 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:35.742 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:35.742 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:35.742 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:35.742 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:35.742 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:35.742 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:35.742 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:35.742 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:35.742 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:35.742 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:35.742 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:35.742 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:35.742 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:35.742 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:35.742Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:35.742Z [INFO] agent.router.manager: shutting down -12:35.742 [INFO] TestAgent: consul server down -12:35.742 [INFO] TestAgent: shutdown complete -12:35.742 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11036 network=tcp -12:35.742 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11036 network=udp -12:35.743 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11037 network=tcp -12:36.243 [INFO] TestAgent: Waiting for endpoints to shut down -12:36.243 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodUpdateCommand_k8s_noMerge (1.17s) - --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_host (0.03s) - --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_ca_cert (0.02s) - --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_jwt (0.01s) - --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge/update_all_fields (0.06s) - --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge/update_all_fields_with_cert_file (0.03s) -2021-10-31T23:12:35.319Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:35.394Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:35.427 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f3ab709c-d90d-71b4-bdbd-c1130515ba2e Address:127.0.0.1:11062}]" -12:35.427 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f3ab709c-d90d-71b4-bdbd-c1130515ba2e.dc1 127.0.0.1 -12:35.428 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f3ab709c-d90d-71b4-bdbd-c1130515ba2e 127.0.0.1 -2021-10-31T23:12:35.428Z [INFO] agent.router: Initializing LAN area manager -12:35.428 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11057 network=udp -12:35.429 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11062 [Follower]" leader= -12:35.430 [INFO] TestAgent.server: Adding LAN server: server="Node-f3ab709c-d90d-71b4-bdbd-c1130515ba2e (Addr: tcp/127.0.0.1:11062) (DC: dc1)" -12:35.430 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f3ab709c-d90d-71b4-bdbd-c1130515ba2e.dc1 area=wan -12:35.430 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11057 network=tcp -12:35.430 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11058 network=tcp -12:35.430 [INFO] TestAgent: started state syncer -12:35.431 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11063 network=tcp -12:35.482 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:35.482 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11062 [Candidate]" term=2 -12:35.482 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:35.482 [DEBUG] TestAgent.server.raft: vote granted: from=f3ab709c-d90d-71b4-bdbd-c1130515ba2e term=2 tally=1 -12:35.482 [INFO] TestAgent.server.raft: election won: tally=1 -12:35.482 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11062 [Leader]" -12:35.482 [INFO] TestAgent.server: cluster leadership acquired -12:35.482 [INFO] TestAgent.server: initializing acls -12:35.482 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:35.482 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:35.483 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:35.483 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:35.483 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:35.483 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:35.483 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f3ab709c-d90d-71b4-bdbd-c1130515ba2e -12:35.483 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f3ab709c-d90d-71b4-bdbd-c1130515ba2e.dc1 -12:35.484 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:35.484 [INFO] TestAgent.server: New leader elected: payload=Node-f3ab709c-d90d-71b4-bdbd-c1130515ba2e -12:35.484 [INFO] TestAgent.server: Updating LAN server: server="Node-f3ab709c-d90d-71b4-bdbd-c1130515ba2e (Addr: tcp/127.0.0.1:11062) (DC: dc1)" -12:35.484 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f3ab709c-d90d-71b4-bdbd-c1130515ba2e.dc1 area=wan -12:35.547 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:35.547 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:35.547 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:35.547 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:35.547 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:35.547 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f3ab709c-d90d-71b4-bdbd-c1130515ba2e -12:35.547 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f3ab709c-d90d-71b4-bdbd-c1130515ba2e -12:35.554 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f3ab709c-d90d-71b4-bdbd-c1130515ba2e -12:35.555 [INFO] TestAgent.server: federation state anti-entropy synced -12:35.794 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:35.794 [INFO] TestAgent: Synced node info -12:35.840 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-6837d97f-f823-e427-3d60-339379ee22ea from=127.0.0.1:41400 latency=492.41µs -12:35.846 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:41402 latency=4.688841ms -12:35.852 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-e789a227-2e46-39e9-0dc2-31c8ce3a501c from=127.0.0.1:41404 latency=616.626µs -12:35.852 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test-e789a227-2e46-39e9-0dc2-31c8ce3a501c from=127.0.0.1:41404 latency=504.015µs -12:35.853 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-e789a227-2e46-39e9-0dc2-31c8ce3a501c from=127.0.0.1:41402 latency=167.744µs -12:35.853 [INFO] TestAgent: Requesting shutdown -12:35.853 [INFO] TestAgent.server: shutting down server -12:35.853 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:35.853 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:35.853 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:35.853 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:35.853 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:35.853 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:35.853 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:35.854 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:35.854 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:35.854 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:35.854 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:35.854 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:35.854 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:35.854 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:35.854 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:35.854Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:35.854Z [INFO] agent.router.manager: shutting down -12:35.854 [INFO] TestAgent: consul server down -12:35.854 [INFO] TestAgent: shutdown complete -12:35.854 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11057 network=tcp -12:35.854 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11057 network=udp -12:35.854 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11058 network=tcp -12:36.355 [INFO] TestAgent: Waiting for endpoints to shut down -12:36.355 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodUpdateCommand_noMerge (1.27s) +=== RUN TestAuthMethodUpdateCommand/update_without_name +=== RUN TestAuthMethodUpdateCommand/update_nonexistent_method +=== RUN TestAuthMethodUpdateCommand/update_all_fields +=== RUN TestAuthMethodUpdateCommand/update_config_field_and_prove_no_merging_happens +2022-12-04T05:48:19.661Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:19.773Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:19.775 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c6124577-04cd-d90b-5d40-3c2cec4111c0 Address:127.0.0.1:20223}]" +48:19.792 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20223 [Follower]" leader= +48:19.793 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c6124577-04cd-d90b-5d40-3c2cec4111c0.dc1 127.0.0.1 +48:19.794 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c6124577-04cd-d90b-5d40-3c2cec4111c0 127.0.0.1 +2022-12-04T05:48:19.794Z [INFO] agent.router: Initializing LAN area manager +48:19.794 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20218 network=udp +48:19.794 [INFO] TestAgent.server: Adding LAN server: server="Node-c6124577-04cd-d90b-5d40-3c2cec4111c0 (Addr: tcp/127.0.0.1:20223) (DC: dc1)" +48:19.794 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c6124577-04cd-d90b-5d40-3c2cec4111c0.dc1 area=wan +48:19.794 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20218 network=tcp +48:19.795 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20219 network=tcp +48:19.795 [INFO] TestAgent: started state syncer +48:19.801 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20224 network=tcp +48:19.844 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:19.844 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20223 [Candidate]" term=2 +48:19.844 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:19.844 [DEBUG] TestAgent.server.raft: vote granted: from=c6124577-04cd-d90b-5d40-3c2cec4111c0 term=2 tally=1 +48:19.844 [INFO] TestAgent.server.raft: election won: tally=1 +48:19.844 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20223 [Leader]" +48:19.844 [INFO] TestAgent.server: cluster leadership acquired +48:19.845 [INFO] TestAgent.server: initializing acls +48:19.848 [INFO] TestAgent.server: New leader elected: payload=Node-c6124577-04cd-d90b-5d40-3c2cec4111c0 +48:19.848 [INFO] TestAgent.server: initializing acls +48:19.849 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:19.849 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:19.849 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:19.849 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:19.849 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:19.849 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:19.849 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +48:19.850 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c6124577-04cd-d90b-5d40-3c2cec4111c0 +48:19.850 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c6124577-04cd-d90b-5d40-3c2cec4111c0.dc1 +48:19.850 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c6124577-04cd-d90b-5d40-3c2cec4111c0.dc1 area=wan +48:19.850 [INFO] TestAgent.server: Updating LAN server: server="Node-c6124577-04cd-d90b-5d40-3c2cec4111c0 (Addr: tcp/127.0.0.1:20223) (DC: dc1)" +48:19.856 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:19.856 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:19.857 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c6124577-04cd-d90b-5d40-3c2cec4111c0 +48:19.857 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c6124577-04cd-d90b-5d40-3c2cec4111c0.dc1 +48:19.857 [INFO] TestAgent.server: Updating LAN server: server="Node-c6124577-04cd-d90b-5d40-3c2cec4111c0 (Addr: tcp/127.0.0.1:20223) (DC: dc1)" +48:19.857 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c6124577-04cd-d90b-5d40-3c2cec4111c0.dc1 area=wan +48:19.858 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:19.859 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:19.859 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:19.859 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:19.859 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:19.859 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:19.859 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c6124577-04cd-d90b-5d40-3c2cec4111c0 +48:19.859 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c6124577-04cd-d90b-5d40-3c2cec4111c0 +48:19.860 [INFO] TestAgent.server: federation state anti-entropy synced +48:19.860 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c6124577-04cd-d90b-5d40-3c2cec4111c0 +48:19.861 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c6124577-04cd-d90b-5d40-3c2cec4111c0 +48:19.969 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-2cf0ce51-251f-e99a-5aad-009ca030ce2c from=127.0.0.1:33836 latency=596.007µs +48:19.971 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:33838 latency=621.891µs +48:19.998 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-0faa6d5a-fe97-bbca-442e-5256afd47f58 from=127.0.0.1:33840 latency=402.515µs +48:20.000 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test-0faa6d5a-fe97-bbca-442e-5256afd47f58 from=127.0.0.1:33840 latency=549.599µs +48:20.034 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-0faa6d5a-fe97-bbca-442e-5256afd47f58 from=127.0.0.1:33838 latency=281.035µs +48:20.034 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:20.034 [INFO] TestAgent: Requesting shutdown +48:20.034 [INFO] TestAgent.server: shutting down server +48:20.034 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:20.034 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:20.034 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:20.034 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:20.034 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:20.034 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:20.034 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:20.035 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:20.035 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:20.035 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:20.035 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:20.035 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:20.035 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:20.035 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:20.035 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:20.035 [INFO] TestAgent: Synced node info +2022-12-04T05:48:20.035Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:20.035Z [INFO] agent.router.manager: shutting down +48:20.035 [INFO] TestAgent: consul server down +48:20.035 [INFO] TestAgent: shutdown complete +48:20.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20218 network=tcp +48:20.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20218 network=udp +48:20.035 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20219 network=tcp +48:20.536 [INFO] TestAgent: Waiting for endpoints to shut down +48:20.536 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodUpdateCommand_noMerge (1.13s) --- PASS: TestAuthMethodUpdateCommand_noMerge/update_without_name (0.00s) - --- PASS: TestAuthMethodUpdateCommand_noMerge/update_nonexistent_method (0.00s) - --- PASS: TestAuthMethodUpdateCommand_noMerge/update_all_fields (0.01s) -2021-10-31T23:12:35.315Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:35.515Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:35.550 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:556a7890-7ee2-0d34-7118-956ff5e43ef8 Address:127.0.0.1:11027}]" -12:35.550 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11027 [Follower]" leader= -12:35.575 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-556a7890-7ee2-0d34-7118-956ff5e43ef8.dc1 127.0.0.1 -12:35.575 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-556a7890-7ee2-0d34-7118-956ff5e43ef8 127.0.0.1 -2021-10-31T23:12:35.575Z [INFO] agent.router: Initializing LAN area manager -12:35.576 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=udp -12:35.576 [INFO] TestAgent.server: Adding LAN server: server="Node-556a7890-7ee2-0d34-7118-956ff5e43ef8 (Addr: tcp/127.0.0.1:11027) (DC: dc1)" -12:35.576 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-556a7890-7ee2-0d34-7118-956ff5e43ef8.dc1 area=wan -12:35.576 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=tcp -12:35.576 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11023 network=tcp -12:35.577 [INFO] TestAgent: started state syncer -12:35.577 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11028 network=tcp -12:35.611 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:35.611 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11027 [Candidate]" term=2 -12:35.611 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:35.611 [DEBUG] TestAgent.server.raft: vote granted: from=556a7890-7ee2-0d34-7118-956ff5e43ef8 term=2 tally=1 -12:35.611 [INFO] TestAgent.server.raft: election won: tally=1 -12:35.611 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11027 [Leader]" -12:35.611 [INFO] TestAgent.server: cluster leadership acquired -12:35.611 [INFO] TestAgent.server: initializing acls -12:35.611 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:35.611 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:35.612 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:35.612 [INFO] TestAgent.server: New leader elected: payload=Node-556a7890-7ee2-0d34-7118-956ff5e43ef8 -12:35.613 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:35.613 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:35.613 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:35.613 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-556a7890-7ee2-0d34-7118-956ff5e43ef8 -12:35.613 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-556a7890-7ee2-0d34-7118-956ff5e43ef8.dc1 -12:35.614 [INFO] TestAgent.server: Updating LAN server: server="Node-556a7890-7ee2-0d34-7118-956ff5e43ef8 (Addr: tcp/127.0.0.1:11027) (DC: dc1)" -12:35.614 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:35.626 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-556a7890-7ee2-0d34-7118-956ff5e43ef8.dc1 area=wan -12:35.635 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:35.635 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:35.635 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:35.635 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:35.635 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:35.635 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-556a7890-7ee2-0d34-7118-956ff5e43ef8 -12:35.635 [INFO] TestAgent.server: member joined, marking health alive: member=Node-556a7890-7ee2-0d34-7118-956ff5e43ef8 -12:35.636 [INFO] TestAgent.server: federation state anti-entropy synced -12:35.636 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-556a7890-7ee2-0d34-7118-956ff5e43ef8 -12:35.707 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:35.708 [INFO] TestAgent: Synced node info -12:35.872 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42136 latency=2.257425ms -12:35.882 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-fe723a08-985a-71aa-5dea-86df070f77e0 from=127.0.0.1:42138 latency=479.011µs -12:35.890 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-fe723a08-985a-71aa-5dea-86df070f77e0 from=127.0.0.1:42138 latency=6.350154ms -12:35.898 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-fe723a08-985a-71aa-5dea-86df070f77e0 from=127.0.0.1:42136 latency=392.597µs -12:35.901 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42136 latency=1.658576ms -12:35.912 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-72661f44-647c-309b-6647-21df8edd913d from=127.0.0.1:42140 latency=410.538µs -12:35.926 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:35.926 [DEBUG] TestAgent: Node info in sync -12:35.926 [DEBUG] TestAgent: Node info in sync -12:35.934 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-72661f44-647c-309b-6647-21df8edd913d from=127.0.0.1:42140 latency=21.241822ms -12:35.942 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-72661f44-647c-309b-6647-21df8edd913d from=127.0.0.1:42136 latency=463.14µs -12:35.945 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42136 latency=1.748431ms -12:35.955 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-4e7209ff-da98-b900-1f72-c15a212a16f4 from=127.0.0.1:42142 latency=1.755038ms -12:35.957 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-4e7209ff-da98-b900-1f72-c15a212a16f4 from=127.0.0.1:42142 latency=1.579189ms -12:35.966 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-4e7209ff-da98-b900-1f72-c15a212a16f4 from=127.0.0.1:42136 latency=479.484µs -12:35.975 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42136 latency=1.813955ms -12:35.985 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-2d48bf17-b0bb-7094-950c-41dd5872f601 from=127.0.0.1:42144 latency=484.783µs -12:35.991 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-2d48bf17-b0bb-7094-950c-41dd5872f601 from=127.0.0.1:42144 latency=1.867568ms -12:35.998 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-2d48bf17-b0bb-7094-950c-41dd5872f601 from=127.0.0.1:42136 latency=440.726µs -12:36.007 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42136 latency=1.952843ms -12:36.010 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-14902414-f72f-0bb7-9386-4f0cd2c1835c from=127.0.0.1:42146 latency=627.985µs -12:36.016 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-14902414-f72f-0bb7-9386-4f0cd2c1835c from=127.0.0.1:42146 latency=1.692018ms -12:36.017 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-14902414-f72f-0bb7-9386-4f0cd2c1835c from=127.0.0.1:42136 latency=270.596µs -12:36.026 [INFO] TestAgent: Requesting shutdown -12:36.026 [INFO] TestAgent.server: shutting down server -12:36.026 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:36.026 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:36.026 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:36.026 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:36.026 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:36.026 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:36.026 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:36.026 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:36.026 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:36.026 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:36.026 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:36.026 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:36.026 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:36.026 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:36.026 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:36.026Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:36.026Z [INFO] agent.router.manager: shutting down -12:36.026 [INFO] TestAgent: consul server down -12:36.026 [INFO] TestAgent: shutdown complete -12:36.026 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=tcp -12:36.026 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=udp -12:36.026 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11023 network=tcp -12:36.529 [INFO] TestAgent: Waiting for endpoints to shut down -12:36.529 [INFO] TestAgent: Endpoints down ---- PASS: TestAuthMethodUpdateCommand_k8s (1.45s) - --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields (0.03s) - --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields_with_cert_file (0.04s) - --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields_but_k8s_host (0.03s) + --- PASS: TestAuthMethodUpdateCommand_noMerge/update_nonexistent_method (0.01s) + --- PASS: TestAuthMethodUpdateCommand_noMerge/update_all_fields (0.06s) +2022-12-04T05:48:19.522Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:19.615Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:19.629 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:294ebe46-db73-2347-263b-2527308aa734 Address:127.0.0.1:20216}]" +48:19.630 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-294ebe46-db73-2347-263b-2527308aa734.dc1 127.0.0.1 +48:19.630 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-294ebe46-db73-2347-263b-2527308aa734 127.0.0.1 +2022-12-04T05:48:19.630Z [INFO] agent.router: Initializing LAN area manager +48:19.631 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=udp +48:19.631 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20216 [Follower]" leader= +48:19.631 [INFO] TestAgent.server: Adding LAN server: server="Node-294ebe46-db73-2347-263b-2527308aa734 (Addr: tcp/127.0.0.1:20216) (DC: dc1)" +48:19.631 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-294ebe46-db73-2347-263b-2527308aa734.dc1 area=wan +48:19.631 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=tcp +48:19.632 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20212 network=tcp +48:19.632 [INFO] TestAgent: started state syncer +48:19.632 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20217 network=tcp +48:19.734 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:19.758 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:19.760 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20216 [Candidate]" term=2 +48:19.760 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:19.760 [DEBUG] TestAgent.server.raft: vote granted: from=294ebe46-db73-2347-263b-2527308aa734 term=2 tally=1 +48:19.760 [INFO] TestAgent.server.raft: election won: tally=1 +48:19.760 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20216 [Leader]" +48:19.760 [INFO] TestAgent.server: cluster leadership acquired +48:19.760 [INFO] TestAgent.server: initializing acls +48:19.761 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:19.761 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:19.761 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:19.762 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:19.762 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:19.762 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:19.762 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-294ebe46-db73-2347-263b-2527308aa734 +48:19.762 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-294ebe46-db73-2347-263b-2527308aa734.dc1 +48:19.762 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:19.765 [INFO] TestAgent.server: New leader elected: payload=Node-294ebe46-db73-2347-263b-2527308aa734 +48:19.765 [INFO] TestAgent.server: Updating LAN server: server="Node-294ebe46-db73-2347-263b-2527308aa734 (Addr: tcp/127.0.0.1:20216) (DC: dc1)" +48:19.765 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-294ebe46-db73-2347-263b-2527308aa734.dc1 area=wan +48:19.828 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:19.828 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:19.828 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:19.828 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:19.828 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:19.828 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-294ebe46-db73-2347-263b-2527308aa734 +48:19.828 [INFO] TestAgent.server: member joined, marking health alive: member=Node-294ebe46-db73-2347-263b-2527308aa734 +48:19.829 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-294ebe46-db73-2347-263b-2527308aa734 +48:19.835 [INFO] TestAgent.server: federation state anti-entropy synced +48:19.979 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42732 latency=1.14108ms +48:19.999 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/testec3a3d6a-3158-8b88-aa50-b465b843713e from=127.0.0.1:42734 latency=294.415µs +48:20.036 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/testec3a3d6a-3158-8b88-aa50-b465b843713e from=127.0.0.1:42734 latency=871.968µs +48:20.046 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/testec3a3d6a-3158-8b88-aa50-b465b843713e from=127.0.0.1:42732 latency=1.316557ms +48:20.086 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42732 latency=21.706225ms +48:20.089 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/testf520b365-662d-5a53-102b-c144dea8d3fd from=127.0.0.1:42736 latency=398.323µs +48:20.090 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:20.090 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/testf520b365-662d-5a53-102b-c144dea8d3fd from=127.0.0.1:42736 latency=596.442µs +48:20.090 [INFO] TestAgent: Synced node info +48:20.090 [DEBUG] TestAgent: Node info in sync +48:20.098 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/testf520b365-662d-5a53-102b-c144dea8d3fd from=127.0.0.1:42732 latency=189.473µs +48:20.099 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42732 latency=576.572µs +48:20.118 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test4fefd1bc-773d-bafa-4b41-cbbfb447c6cd from=127.0.0.1:42738 latency=409.188µs +48:20.119 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test4fefd1bc-773d-bafa-4b41-cbbfb447c6cd from=127.0.0.1:42738 latency=627.801µs +48:20.119 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test4fefd1bc-773d-bafa-4b41-cbbfb447c6cd from=127.0.0.1:42732 latency=182.17µs +48:20.120 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42732 latency=501.974µs +48:20.126 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/testafb9c4d4-0ed1-c60d-1ffe-c64c5618c210 from=127.0.0.1:42740 latency=372.38µs +48:20.129 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/testafb9c4d4-0ed1-c60d-1ffe-c64c5618c210 from=127.0.0.1:42740 latency=971.544µs +48:20.130 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/testafb9c4d4-0ed1-c60d-1ffe-c64c5618c210 from=127.0.0.1:42732 latency=202.729µs +48:20.130 [INFO] TestAgent: Requesting shutdown +48:20.130 [INFO] TestAgent.server: shutting down server +48:20.130 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:20.130 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:20.130 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:20.130 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:20.130 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:20.130 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:20.130 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:20.130 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:20.130 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:20.130 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:20.130 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:20.130 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:20.130 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:20.130 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:20.131 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:20.131Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:20.131Z [INFO] agent.router.manager: shutting down +48:20.131 [INFO] TestAgent: consul server down +48:20.131 [INFO] TestAgent: shutdown complete +48:20.131 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=tcp +48:20.131 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=udp +48:20.131 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20212 network=tcp +48:20.632 [INFO] TestAgent: Waiting for endpoints to shut down +48:20.632 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodUpdateCommand_config (1.23s) + --- PASS: TestAuthMethodUpdateCommand_config/config_file (0.08s) + --- PASS: TestAuthMethodUpdateCommand_config/config_stdin (0.04s) + --- PASS: TestAuthMethodUpdateCommand_config/config_string (0.02s) + --- PASS: TestAuthMethodUpdateCommand_config/config_with_no_merge (0.01s) +2022-12-04T05:48:19.593Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:19.733Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:19.747 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:61c2bfc3-89cf-f80b-3f77-25b22aa765d5 Address:127.0.0.1:20237}]" +48:19.756 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20237 [Follower]" leader= +48:19.757 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-61c2bfc3-89cf-f80b-3f77-25b22aa765d5.dc1 127.0.0.1 +48:19.757 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-61c2bfc3-89cf-f80b-3f77-25b22aa765d5 127.0.0.1 +2022-12-04T05:48:19.757Z [INFO] agent.router: Initializing LAN area manager +48:19.758 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20232 network=udp +48:19.759 [INFO] TestAgent.server: Adding LAN server: server="Node-61c2bfc3-89cf-f80b-3f77-25b22aa765d5 (Addr: tcp/127.0.0.1:20237) (DC: dc1)" +48:19.759 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-61c2bfc3-89cf-f80b-3f77-25b22aa765d5.dc1 area=wan +48:19.759 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20232 network=tcp +48:19.759 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20233 network=tcp +48:19.759 [INFO] TestAgent: started state syncer +48:19.759 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20238 network=tcp +48:19.825 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:19.825 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20237 [Candidate]" term=2 +48:19.825 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:19.825 [DEBUG] TestAgent.server.raft: vote granted: from=61c2bfc3-89cf-f80b-3f77-25b22aa765d5 term=2 tally=1 +48:19.825 [INFO] TestAgent.server.raft: election won: tally=1 +48:19.825 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20237 [Leader]" +48:19.828 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=2 mode=2 found=true leader=127.0.0.1:20237 +48:19.828 [INFO] TestAgent.server: cluster leadership acquired +48:19.828 [INFO] TestAgent.server: initializing acls +48:19.829 [INFO] TestAgent.server: New leader elected: payload=Node-61c2bfc3-89cf-f80b-3f77-25b22aa765d5 +48:19.829 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:19.829 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:19.830 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:19.830 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:19.830 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:19.830 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:19.830 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-61c2bfc3-89cf-f80b-3f77-25b22aa765d5 +48:19.831 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-61c2bfc3-89cf-f80b-3f77-25b22aa765d5.dc1 +48:19.831 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:19.832 [INFO] TestAgent.server: Updating LAN server: server="Node-61c2bfc3-89cf-f80b-3f77-25b22aa765d5 (Addr: tcp/127.0.0.1:20237) (DC: dc1)" +48:19.832 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-61c2bfc3-89cf-f80b-3f77-25b22aa765d5.dc1 area=wan +48:19.833 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:19.833 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:19.833 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:19.833 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:19.833 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:19.833 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-61c2bfc3-89cf-f80b-3f77-25b22aa765d5 +48:19.833 [INFO] TestAgent.server: member joined, marking health alive: member=Node-61c2bfc3-89cf-f80b-3f77-25b22aa765d5 +48:19.834 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-61c2bfc3-89cf-f80b-3f77-25b22aa765d5 +48:19.834 [INFO] TestAgent.server: federation state anti-entropy synced +48:19.850 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:19.850 [INFO] TestAgent: Synced node info +48:19.850 [DEBUG] TestAgent: Node info in sync +48:19.949 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:44054 latency=2.320274ms +48:19.958 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-6b06945c-237b-55d9-c682-3051e9ccf8d7 from=127.0.0.1:44056 latency=482.172µs +48:19.963 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:44054 latency=3.001621ms +48:19.970 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-993a8ccb-d83e-4f14-f4c3-76741134b587 from=127.0.0.1:44058 latency=1.072635ms +48:20.032 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:44054 latency=61.605608ms +48:20.042 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-43e44295-46e2-0942-cfc4-74f259862fa6 from=127.0.0.1:44060 latency=817.728µs +48:20.085 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:44054 latency=40.678668ms +48:20.098 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-bdb6730c-dab8-ca4b-7ff0-7b4e83fcf4d1 from=127.0.0.1:44062 latency=486.663µs +48:20.113 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-bdb6730c-dab8-ca4b-7ff0-7b4e83fcf4d1 from=127.0.0.1:44062 latency=14.351016ms +48:20.117 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-bdb6730c-dab8-ca4b-7ff0-7b4e83fcf4d1 from=127.0.0.1:44054 latency=386.54µs +48:20.120 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:44054 latency=1.800409ms +48:20.127 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-bd1603c5-ffd1-acc0-ef8b-b9322281cd17 from=127.0.0.1:44064 latency=304.328µs +48:20.131 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-bd1603c5-ffd1-acc0-ef8b-b9322281cd17 from=127.0.0.1:44064 latency=1.899741ms +48:20.141 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-bd1603c5-ffd1-acc0-ef8b-b9322281cd17 from=127.0.0.1:44054 latency=528.825µs +48:20.168 [INFO] TestAgent: Requesting shutdown +48:20.168 [INFO] TestAgent.server: shutting down server +48:20.168 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:20.168 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:20.168 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:20.168 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:20.168 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:20.168 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:20.168 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:20.168 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:20.168 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:20.168 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:20.168 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:20.168 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:20.168 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:20.168 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:20.169 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:20.169Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:20.169Z [INFO] agent.router.manager: shutting down +48:20.169 [INFO] TestAgent: consul server down +48:20.169 [INFO] TestAgent: shutdown complete +48:20.169 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20232 network=tcp +48:20.169 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20232 network=udp +48:20.169 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20233 network=tcp +48:20.672 [INFO] TestAgent: Waiting for endpoints to shut down +48:20.672 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodUpdateCommand_k8s_noMerge (1.27s) + --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_host (0.01s) + --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_ca_cert (0.01s) + --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge/update_missing_k8s_jwt (0.07s) + --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge/update_all_fields (0.07s) + --- PASS: TestAuthMethodUpdateCommand_k8s_noMerge/update_all_fields_with_cert_file (0.04s) +2022-12-04T05:48:19.588Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:19.733Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:19.746 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:83f30104-8bad-50f4-aeea-6a5b93e99c7e Address:127.0.0.1:20230}]" +48:19.747 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-83f30104-8bad-50f4-aeea-6a5b93e99c7e.dc1 127.0.0.1 +48:19.758 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-83f30104-8bad-50f4-aeea-6a5b93e99c7e 127.0.0.1 +2022-12-04T05:48:19.758Z [INFO] agent.router: Initializing LAN area manager +48:19.758 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20225 network=udp +48:19.758 [INFO] TestAgent.server: Adding LAN server: server="Node-83f30104-8bad-50f4-aeea-6a5b93e99c7e (Addr: tcp/127.0.0.1:20230) (DC: dc1)" +48:19.758 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-83f30104-8bad-50f4-aeea-6a5b93e99c7e.dc1 area=wan +48:19.759 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20225 network=tcp +48:19.759 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20226 network=tcp +48:19.759 [INFO] TestAgent: started state syncer +48:19.759 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20231 network=tcp +48:19.765 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20230 [Follower]" leader= +48:19.824 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:19.824 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20230 [Candidate]" term=2 +48:19.825 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:19.825 [DEBUG] TestAgent.server.raft: vote granted: from=83f30104-8bad-50f4-aeea-6a5b93e99c7e term=2 tally=1 +48:19.825 [INFO] TestAgent.server.raft: election won: tally=1 +48:19.825 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20230 [Leader]" +48:19.828 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=2 mode=2 found=true leader=127.0.0.1:20230 +48:19.828 [INFO] TestAgent.server: cluster leadership acquired +48:19.828 [INFO] TestAgent.server: initializing acls +48:19.828 [INFO] TestAgent.server: New leader elected: payload=Node-83f30104-8bad-50f4-aeea-6a5b93e99c7e +48:19.830 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:19.830 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:19.830 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:19.831 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:19.831 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:19.831 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:19.831 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-83f30104-8bad-50f4-aeea-6a5b93e99c7e +48:19.831 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-83f30104-8bad-50f4-aeea-6a5b93e99c7e.dc1 +48:19.831 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:19.832 [INFO] TestAgent.server: Updating LAN server: server="Node-83f30104-8bad-50f4-aeea-6a5b93e99c7e (Addr: tcp/127.0.0.1:20230) (DC: dc1)" +48:19.833 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:19.833 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:19.833 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:19.833 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:19.833 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:19.833 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-83f30104-8bad-50f4-aeea-6a5b93e99c7e +48:19.833 [INFO] TestAgent.server: member joined, marking health alive: member=Node-83f30104-8bad-50f4-aeea-6a5b93e99c7e +48:19.834 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-83f30104-8bad-50f4-aeea-6a5b93e99c7e +48:19.834 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-83f30104-8bad-50f4-aeea-6a5b93e99c7e.dc1 area=wan +48:19.834 [INFO] TestAgent.server: federation state anti-entropy synced +48:20.044 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:20.045 [INFO] TestAgent: Synced node info +48:20.184 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:50438 latency=17.730046ms +48:20.187 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-a475d01a-d82b-37e5-233b-1f8c5bccb9d1 from=127.0.0.1:50442 latency=366.493µs +48:20.188 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test-a475d01a-d82b-37e5-233b-1f8c5bccb9d1 from=127.0.0.1:50442 latency=846.334µs +48:20.189 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-a475d01a-d82b-37e5-233b-1f8c5bccb9d1 from=127.0.0.1:50438 latency=235.121µs +48:20.189 [INFO] TestAgent: Requesting shutdown +48:20.189 [INFO] TestAgent.server: shutting down server +48:20.189 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:20.189 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:20.189 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:20.189 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:20.189 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:20.189 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:20.189 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:20.189 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:20.189 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:20.189 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:20.189 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:20.190 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:20.190 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:20.190 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:20.190 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:20.190Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:20.190Z [INFO] agent.router.manager: shutting down +48:20.190 [INFO] TestAgent: consul server down +48:20.190 [INFO] TestAgent: shutdown complete +48:20.190 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20225 network=tcp +48:20.190 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20225 network=udp +48:20.190 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20226 network=tcp +48:20.691 [INFO] TestAgent: Waiting for endpoints to shut down +48:20.691 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodUpdateCommand_JSON (1.29s) + --- PASS: TestAuthMethodUpdateCommand_JSON/update_without_name (0.00s) + --- PASS: TestAuthMethodUpdateCommand_JSON/update_all_fields (0.02s) +2022-12-04T05:48:19.639Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:19.825Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:19.827 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cceea783-9fd7-bc86-6557-9819e00060d5 Address:127.0.0.1:20251}]" +48:19.828 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cceea783-9fd7-bc86-6557-9819e00060d5.dc1 127.0.0.1 +48:19.828 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20251 [Follower]" leader= +48:19.840 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cceea783-9fd7-bc86-6557-9819e00060d5 127.0.0.1 +2022-12-04T05:48:19.840Z [INFO] agent.router: Initializing LAN area manager +48:19.841 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20246 network=udp +48:19.841 [INFO] TestAgent.server: Adding LAN server: server="Node-cceea783-9fd7-bc86-6557-9819e00060d5 (Addr: tcp/127.0.0.1:20251) (DC: dc1)" +48:19.841 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cceea783-9fd7-bc86-6557-9819e00060d5.dc1 area=wan +48:19.891 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:19.891 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20251 [Candidate]" term=2 +48:19.891 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:19.891 [DEBUG] TestAgent.server.raft: vote granted: from=cceea783-9fd7-bc86-6557-9819e00060d5 term=2 tally=1 +48:19.891 [INFO] TestAgent.server.raft: election won: tally=1 +48:19.891 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20251 [Leader]" +48:19.891 [INFO] TestAgent.server: cluster leadership acquired +48:19.891 [INFO] TestAgent.server: initializing acls +48:19.892 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:19.892 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:19.892 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:19.892 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:19.892 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:19.892 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:19.893 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-cceea783-9fd7-bc86-6557-9819e00060d5 +48:19.893 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-cceea783-9fd7-bc86-6557-9819e00060d5.dc1 +48:19.893 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:19.895 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:19.895 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:19.895 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:19.895 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:19.895 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:19.895 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cceea783-9fd7-bc86-6557-9819e00060d5 +48:19.895 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cceea783-9fd7-bc86-6557-9819e00060d5 +48:19.896 [INFO] TestAgent.server: New leader elected: payload=Node-cceea783-9fd7-bc86-6557-9819e00060d5 +48:19.896 [INFO] TestAgent.server: Updating LAN server: server="Node-cceea783-9fd7-bc86-6557-9819e00060d5 (Addr: tcp/127.0.0.1:20251) (DC: dc1)" +48:19.896 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-cceea783-9fd7-bc86-6557-9819e00060d5.dc1 area=wan +48:19.897 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cceea783-9fd7-bc86-6557-9819e00060d5 +48:19.897 [INFO] TestAgent.server: federation state anti-entropy synced +48:19.901 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20246 network=tcp +48:19.902 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20247 network=tcp +48:19.902 [INFO] TestAgent: started state syncer +48:19.919 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20252 network=tcp +48:19.919 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:19.924 [INFO] TestAgent: Synced node info +48:19.929 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:37452 latency=7.371147ms +48:19.969 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-038d1b64-819b-5e4d-1598-f0703224bbef from=127.0.0.1:37456 latency=491.087µs +48:19.973 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-038d1b64-819b-5e4d-1598-f0703224bbef from=127.0.0.1:37456 latency=3.297145ms +48:20.035 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-038d1b64-819b-5e4d-1598-f0703224bbef from=127.0.0.1:37452 latency=420.411µs +48:20.088 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:37452 latency=47.494531ms +48:20.097 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-bfab64dd-f4e6-1d6d-a8ae-4908818dc0bc from=127.0.0.1:37458 latency=588.273µs +48:20.106 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-bfab64dd-f4e6-1d6d-a8ae-4908818dc0bc from=127.0.0.1:37458 latency=7.39145ms +48:20.117 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-bfab64dd-f4e6-1d6d-a8ae-4908818dc0bc from=127.0.0.1:37452 latency=393.973µs +48:20.119 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:37452 latency=1.641486ms +48:20.127 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-7e1d589a-b4e1-75ec-19ab-42a36a82437f from=127.0.0.1:37460 latency=1.14449ms +48:20.143 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-7e1d589a-b4e1-75ec-19ab-42a36a82437f from=127.0.0.1:37460 latency=2.945294ms +48:20.171 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-7e1d589a-b4e1-75ec-19ab-42a36a82437f from=127.0.0.1:37452 latency=450.586µs +48:20.181 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:37452 latency=4.845963ms +48:20.186 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-e38d0e60-ffc8-24ec-6a5b-86bf94d7d506 from=127.0.0.1:37462 latency=473.604µs +48:20.205 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-e38d0e60-ffc8-24ec-6a5b-86bf94d7d506 from=127.0.0.1:37462 latency=17.784426ms +48:20.206 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-e38d0e60-ffc8-24ec-6a5b-86bf94d7d506 from=127.0.0.1:37452 latency=300.984µs +48:20.209 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:37452 latency=1.860781ms +48:20.220 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-eb11f6bb-d766-3a97-dbc7-8132795c0380 from=127.0.0.1:37464 latency=515.76µs +48:20.223 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/k8s-eb11f6bb-d766-3a97-dbc7-8132795c0380 from=127.0.0.1:37464 latency=1.774566ms +48:20.225 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/k8s-eb11f6bb-d766-3a97-dbc7-8132795c0380 from=127.0.0.1:37452 latency=915.134µs +48:20.225 [INFO] TestAgent: Requesting shutdown +48:20.225 [INFO] TestAgent.server: shutting down server +48:20.225 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:20.225 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:20.225 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:20.225 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:20.225 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:20.225 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:20.225 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:20.225 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:20.225 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:20.225 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:20.225 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:20.225 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:20.225 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:20.225 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:20.225 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:20.226Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:20.226Z [INFO] agent.router.manager: shutting down +48:20.226 [INFO] TestAgent: consul server down +48:20.226 [INFO] TestAgent: shutdown complete +48:20.226 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20246 network=tcp +48:20.226 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20246 network=udp +48:20.226 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20247 network=tcp +48:20.728 [INFO] TestAgent: Waiting for endpoints to shut down +48:20.728 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodUpdateCommand_k8s (1.32s) + --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields (0.12s) + --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields_with_cert_file (0.08s) + --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields_but_k8s_host (0.05s) --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields_but_k8s_ca_cert (0.03s) --- PASS: TestAuthMethodUpdateCommand_k8s/update_all_fields_but_k8s_jwt (0.02s) +2022-12-04T05:48:19.721Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:19.803Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:19.913 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:66bfb70c-6f9f-070d-87f3-5ef6afe7d011 Address:127.0.0.1:20244}]" +48:19.914 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-66bfb70c-6f9f-070d-87f3-5ef6afe7d011.dc1 127.0.0.1 +48:19.915 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-66bfb70c-6f9f-070d-87f3-5ef6afe7d011 127.0.0.1 +2022-12-04T05:48:19.915Z [INFO] agent.router: Initializing LAN area manager +48:19.920 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20244 [Follower]" leader= +48:19.921 [INFO] TestAgent.server: Adding LAN server: server="Node-66bfb70c-6f9f-070d-87f3-5ef6afe7d011 (Addr: tcp/127.0.0.1:20244) (DC: dc1)" +48:19.921 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-66bfb70c-6f9f-070d-87f3-5ef6afe7d011.dc1 area=wan +48:19.921 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20239 network=tcp +48:19.921 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20239 network=udp +48:19.923 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20240 network=tcp +48:19.923 [INFO] TestAgent: started state syncer +48:19.924 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20245 network=tcp +48:19.964 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:19.964 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20244 [Candidate]" term=2 +48:19.964 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:19.964 [DEBUG] TestAgent.server.raft: vote granted: from=66bfb70c-6f9f-070d-87f3-5ef6afe7d011 term=2 tally=1 +48:19.964 [INFO] TestAgent.server.raft: election won: tally=1 +48:19.964 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20244 [Leader]" +48:19.964 [INFO] TestAgent.server: cluster leadership acquired +48:19.965 [INFO] TestAgent.server: initializing acls +48:19.965 [INFO] TestAgent.server: New leader elected: payload=Node-66bfb70c-6f9f-070d-87f3-5ef6afe7d011 +48:19.965 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:19.965 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:19.965 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:19.965 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:19.965 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:19.965 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:19.966 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-66bfb70c-6f9f-070d-87f3-5ef6afe7d011 +48:19.966 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-66bfb70c-6f9f-070d-87f3-5ef6afe7d011.dc1 +48:19.966 [INFO] TestAgent.server: Updating LAN server: server="Node-66bfb70c-6f9f-070d-87f3-5ef6afe7d011 (Addr: tcp/127.0.0.1:20244) (DC: dc1)" +48:19.966 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-66bfb70c-6f9f-070d-87f3-5ef6afe7d011.dc1 area=wan +48:19.967 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:19.968 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:19.968 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:19.968 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:19.968 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:19.968 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:19.968 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-66bfb70c-6f9f-070d-87f3-5ef6afe7d011 +48:19.968 [INFO] TestAgent.server: member joined, marking health alive: member=Node-66bfb70c-6f9f-070d-87f3-5ef6afe7d011 +48:19.969 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-66bfb70c-6f9f-070d-87f3-5ef6afe7d011 +48:19.969 [INFO] TestAgent.server: federation state anti-entropy synced +48:20.284 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:20.285 [INFO] TestAgent: Synced node info +48:20.285 [DEBUG] TestAgent: Node info in sync +48:20.358 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-6cba9439-877e-2226-606d-89246433987e from=127.0.0.1:52770 latency=403.284µs +48:20.359 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:52772 latency=634.937µs +48:20.373 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-6dd253ca-7a6a-f9a1-cbe5-7cf6dfb1fec0 from=127.0.0.1:52774 latency=396.726µs +48:20.374 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test-6dd253ca-7a6a-f9a1-cbe5-7cf6dfb1fec0 from=127.0.0.1:52774 latency=559.548µs +48:20.374 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-6dd253ca-7a6a-f9a1-cbe5-7cf6dfb1fec0 from=127.0.0.1:52772 latency=185.81µs +48:20.381 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-6dd253ca-7a6a-f9a1-cbe5-7cf6dfb1fec0 from=127.0.0.1:52776 latency=730.769µs +48:20.389 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method/test-6dd253ca-7a6a-f9a1-cbe5-7cf6dfb1fec0 from=127.0.0.1:52776 latency=775.559µs +48:20.390 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/auth-method/test-6dd253ca-7a6a-f9a1-cbe5-7cf6dfb1fec0 from=127.0.0.1:52772 latency=180.942µs +48:20.390 [INFO] TestAgent: Requesting shutdown +48:20.390 [INFO] TestAgent.server: shutting down server +48:20.390 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:20.390 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:20.390 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:20.390 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:20.390 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:20.390 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:20.390 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:20.390 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:20.390 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:20.390 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:20.390 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:20.390 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:20.390 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:20.390 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:20.390 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:20.390Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:20.390Z [INFO] agent.router.manager: shutting down +48:20.390 [INFO] TestAgent: consul server down +48:20.390 [INFO] TestAgent: shutdown complete +48:20.390 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20239 network=tcp +48:20.390 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20239 network=udp +48:20.390 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20240 network=tcp +48:20.892 [INFO] TestAgent: Waiting for endpoints to shut down +48:20.892 [INFO] TestAgent: Endpoints down +--- PASS: TestAuthMethodUpdateCommand (1.49s) + --- PASS: TestAuthMethodUpdateCommand/update_without_name (0.00s) + --- PASS: TestAuthMethodUpdateCommand/update_nonexistent_method (0.00s) + --- PASS: TestAuthMethodUpdateCommand/update_all_fields (0.01s) + --- PASS: TestAuthMethodUpdateCommand/update_config_field_and_prove_no_merging_happens (0.02s) PASS -ok github.com/hashicorp/consul/command/acl/authmethod/update 1.633s +ok github.com/hashicorp/consul/command/acl/authmethod/update 2.470s ? github.com/hashicorp/consul/command/acl/bindingrule [no test files] === RUN TestBindingRuleCreateCommand_noTabs === PAUSE TestBindingRuleCreateCommand_noTabs @@ -73925,13 +74064,12 @@ === RUN TestBindingRuleCreateCommand_JSON === PAUSE TestBindingRuleCreateCommand_JSON === CONT TestBindingRuleCreateCommand_noTabs -=== CONT TestBindingRuleCreateCommand --- PASS: TestBindingRuleCreateCommand_noTabs (0.00s) === CONT TestBindingRuleCreateCommand_JSON [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +=== CONT TestBindingRuleCreateCommand [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -=== RUN TestBindingRuleCreateCommand_JSON/create_it_with_no_selector === RUN TestBindingRuleCreateCommand/method_is_required === RUN TestBindingRuleCreateCommand/bind_type_required === RUN TestBindingRuleCreateCommand/bind_name_required @@ -73939,400 +74077,395 @@ === RUN TestBindingRuleCreateCommand/create_it_with_no_selector === RUN TestBindingRuleCreateCommand/create_it_with_a_match_selector === RUN TestBindingRuleCreateCommand/create_it_with_type_role -2021-10-31T23:12:42.204Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:42.263Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:42.275 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1f911c0e-68d1-ea77-ce16-5d127842a1cb Address:127.0.0.1:28391}]" -12:42.277 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28391 [Follower]" leader= -12:42.277 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1f911c0e-68d1-ea77-ce16-5d127842a1cb.dc1 127.0.0.1 -12:42.286 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1f911c0e-68d1-ea77-ce16-5d127842a1cb 127.0.0.1 -2021-10-31T23:12:42.286Z [INFO] agent.router: Initializing LAN area manager -12:42.287 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28386 network=udp -12:42.287 [INFO] TestAgent.server: Adding LAN server: server="Node-1f911c0e-68d1-ea77-ce16-5d127842a1cb (Addr: tcp/127.0.0.1:28391) (DC: dc1)" -12:42.287 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1f911c0e-68d1-ea77-ce16-5d127842a1cb.dc1 area=wan -12:42.310 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28386 network=tcp -12:42.311 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28387 network=tcp -12:42.311 [INFO] TestAgent: started state syncer -12:42.311 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28392 network=tcp -12:42.333 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:42.333 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28391 [Candidate]" term=2 -12:42.334 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:42.334 [DEBUG] TestAgent.server.raft: vote granted: from=1f911c0e-68d1-ea77-ce16-5d127842a1cb term=2 tally=1 -12:42.334 [INFO] TestAgent.server.raft: election won: tally=1 -12:42.334 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28391 [Leader]" -12:42.334 [INFO] TestAgent.server: cluster leadership acquired -12:42.334 [INFO] TestAgent.server: initializing acls -12:42.334 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:42.334 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:42.335 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:42.335 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:42.335 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:42.335 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:42.335 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1f911c0e-68d1-ea77-ce16-5d127842a1cb -12:42.335 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1f911c0e-68d1-ea77-ce16-5d127842a1cb.dc1 -12:42.336 [INFO] TestAgent.server: New leader elected: payload=Node-1f911c0e-68d1-ea77-ce16-5d127842a1cb -12:42.336 [INFO] TestAgent.server: Updating LAN server: server="Node-1f911c0e-68d1-ea77-ce16-5d127842a1cb (Addr: tcp/127.0.0.1:28391) (DC: dc1)" -12:42.336 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1f911c0e-68d1-ea77-ce16-5d127842a1cb.dc1 area=wan -12:42.336 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:42.410 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:42.410 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:42.410 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:42.410 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:42.410 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:42.410 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1f911c0e-68d1-ea77-ce16-5d127842a1cb -12:42.410 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1f911c0e-68d1-ea77-ce16-5d127842a1cb -12:42.411 [INFO] TestAgent.server: federation state anti-entropy synced -12:42.411 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1f911c0e-68d1-ea77-ce16-5d127842a1cb -12:42.598 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:42.598 [INFO] TestAgent: Synced node info -12:42.644 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:48302 latency=1.567365ms -12:42.654 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:48312 latency=1.024781ms -12:42.655 [INFO] TestAgent: Requesting shutdown -12:42.655 [INFO] TestAgent.server: shutting down server -12:42.655 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:42.655 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:42.655 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:42.655 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:42.655 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:42.655 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:42.655 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:42.655 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:42.655 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:42.655 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:42.655 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:42.655 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:42.655 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:42.655 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:42.655 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:42.655Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:42.655Z [INFO] agent.router.manager: shutting down -12:42.656 [INFO] TestAgent: consul server down -12:42.656 [INFO] TestAgent: shutdown complete -12:42.656 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28386 network=tcp -12:42.656 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28386 network=udp -12:42.656 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28387 network=tcp -12:43.157 [INFO] TestAgent: Waiting for endpoints to shut down -12:43.157 [INFO] TestAgent: Endpoints down ---- PASS: TestBindingRuleCreateCommand_JSON (1.11s) - --- PASS: TestBindingRuleCreateCommand_JSON/create_it_with_no_selector (0.01s) -2021-10-31T23:12:42.243Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:42.310Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:42.312 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:618c0b49-5b00-fd5e-6d2a-130d8d7ad6e7 Address:127.0.0.1:28384}]" -12:42.312 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28384 [Follower]" leader= -12:42.328 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-618c0b49-5b00-fd5e-6d2a-130d8d7ad6e7.dc1 127.0.0.1 -12:42.329 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-618c0b49-5b00-fd5e-6d2a-130d8d7ad6e7 127.0.0.1 -2021-10-31T23:12:42.329Z [INFO] agent.router: Initializing LAN area manager -12:42.329 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=udp -12:42.330 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=tcp -12:42.330 [INFO] TestAgent.server: Adding LAN server: server="Node-618c0b49-5b00-fd5e-6d2a-130d8d7ad6e7 (Addr: tcp/127.0.0.1:28384) (DC: dc1)" -12:42.330 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-618c0b49-5b00-fd5e-6d2a-130d8d7ad6e7.dc1 area=wan -12:42.330 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28380 network=tcp -12:42.330 [INFO] TestAgent: started state syncer -12:42.330 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28385 network=tcp -12:42.373 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:42.374 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28384 [Candidate]" term=2 -12:42.374 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:42.374 [DEBUG] TestAgent.server.raft: vote granted: from=618c0b49-5b00-fd5e-6d2a-130d8d7ad6e7 term=2 tally=1 -12:42.374 [INFO] TestAgent.server.raft: election won: tally=1 -12:42.374 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28384 [Leader]" -12:42.374 [INFO] TestAgent.server: cluster leadership acquired -12:42.374 [INFO] TestAgent.server: initializing acls -12:42.375 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:42.375 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:42.375 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:42.376 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:42.376 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:42.376 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:42.376 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-618c0b49-5b00-fd5e-6d2a-130d8d7ad6e7 -12:42.376 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-618c0b49-5b00-fd5e-6d2a-130d8d7ad6e7.dc1 -12:42.376 [INFO] TestAgent.server: New leader elected: payload=Node-618c0b49-5b00-fd5e-6d2a-130d8d7ad6e7 -12:42.377 [INFO] TestAgent.server: Updating LAN server: server="Node-618c0b49-5b00-fd5e-6d2a-130d8d7ad6e7 (Addr: tcp/127.0.0.1:28384) (DC: dc1)" -12:42.377 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-618c0b49-5b00-fd5e-6d2a-130d8d7ad6e7.dc1 area=wan -12:42.377 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:42.394 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:42.394 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:42.394 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:42.394 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:42.394 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:42.394 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-618c0b49-5b00-fd5e-6d2a-130d8d7ad6e7 -12:42.394 [INFO] TestAgent.server: member joined, marking health alive: member=Node-618c0b49-5b00-fd5e-6d2a-130d8d7ad6e7 -12:42.410 [INFO] TestAgent.server: federation state anti-entropy synced -12:42.410 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-618c0b49-5b00-fd5e-6d2a-130d8d7ad6e7 -12:42.594 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:42.594 [INFO] TestAgent: Synced node info -12:42.671 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:46744 latency=1.242744ms -12:42.683 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:46752 error="invalid Binding Rule: Selector is invalid: 1:4 (3): no match found, expected: "!=", ".", "==", "[", [ \t\r\n] or [a-zA-Z0-9_]" -12:42.684 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:46752 latency=1.141405ms -12:42.699 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:46758 latency=10.418748ms -12:42.723 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:46778 latency=1.259991ms -12:42.728 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:46786 latency=1.331507ms -12:42.728 [INFO] TestAgent: Requesting shutdown -12:42.728 [INFO] TestAgent.server: shutting down server -12:42.728 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:42.728 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:42.728 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:42.728 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:42.728 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:42.728 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:42.728 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:42.728 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:42.728 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:42.728 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:42.728 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:42.728 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:42.728 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:42.728 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:42.728 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:42.729Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:42.729Z [INFO] agent.router.manager: shutting down -12:42.729 [INFO] TestAgent: consul server down -12:42.729 [INFO] TestAgent: shutdown complete -12:42.729 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=tcp -12:42.729 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=udp -12:42.729 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28380 network=tcp -12:43.229 [INFO] TestAgent: Waiting for endpoints to shut down -12:43.229 [INFO] TestAgent: Endpoints down ---- PASS: TestBindingRuleCreateCommand (1.19s) +=== RUN TestBindingRuleCreateCommand_JSON/create_it_with_no_selector +2022-12-04T05:48:22.833Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:22.872Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:22.875 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1dd83d04-e203-e7fc-80c6-4cdca55887b4 Address:127.0.0.1:14097}]" +48:22.876 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1dd83d04-e203-e7fc-80c6-4cdca55887b4.dc1 127.0.0.1 +48:22.876 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14097 [Follower]" leader= +48:22.878 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1dd83d04-e203-e7fc-80c6-4cdca55887b4 127.0.0.1 +2022-12-04T05:48:22.878Z [INFO] agent.router: Initializing LAN area manager +48:22.883 [INFO] TestAgent.server: Adding LAN server: server="Node-1dd83d04-e203-e7fc-80c6-4cdca55887b4 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +48:22.883 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1dd83d04-e203-e7fc-80c6-4cdca55887b4.dc1 area=wan +48:22.884 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=tcp +48:22.884 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=udp +48:22.885 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14093 network=tcp +48:22.885 [INFO] TestAgent: started state syncer +48:22.885 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14098 network=tcp +48:22.936 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:22.949 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:22.949 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14097 [Candidate]" term=2 +48:22.949 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:22.949 [DEBUG] TestAgent.server.raft: vote granted: from=1dd83d04-e203-e7fc-80c6-4cdca55887b4 term=2 tally=1 +48:22.949 [INFO] TestAgent.server.raft: election won: tally=1 +48:22.949 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14097 [Leader]" +48:22.949 [INFO] TestAgent.server: cluster leadership acquired +48:22.949 [INFO] TestAgent.server: initializing acls +48:22.950 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:22.950 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:22.952 [INFO] TestAgent.server: New leader elected: payload=Node-1dd83d04-e203-e7fc-80c6-4cdca55887b4 +48:22.956 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:22.960 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:22.960 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:22.960 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:22.961 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-1dd83d04-e203-e7fc-80c6-4cdca55887b4 +48:22.961 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-1dd83d04-e203-e7fc-80c6-4cdca55887b4.dc1 +48:22.961 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:22.962 [INFO] TestAgent.server: Updating LAN server: server="Node-1dd83d04-e203-e7fc-80c6-4cdca55887b4 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +48:22.962 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-1dd83d04-e203-e7fc-80c6-4cdca55887b4.dc1 area=wan +48:22.990 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:22.990 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:22.990 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:22.990 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:22.990 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:22.990 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1dd83d04-e203-e7fc-80c6-4cdca55887b4 +48:22.990 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1dd83d04-e203-e7fc-80c6-4cdca55887b4 +48:22.991 [INFO] TestAgent.server: federation state anti-entropy synced +48:22.991 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1dd83d04-e203-e7fc-80c6-4cdca55887b4 +48:23.020 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:23.021 [INFO] TestAgent: Synced node info +48:23.021 [DEBUG] TestAgent: Node info in sync +48:23.040 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:60978 latency=1.238799ms +48:23.057 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60980 error="invalid Binding Rule: Selector is invalid: 1:4 (3): no match found, expected: "!=", ".", "==", "[", [ \t\r\n] or [a-zA-Z0-9_]" +48:23.058 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60980 latency=1.734405ms +48:23.070 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60982 latency=5.589743ms +48:23.076 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60984 latency=2.003123ms +48:23.079 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60986 latency=1.10847ms +48:23.079 [INFO] TestAgent: Requesting shutdown +48:23.079 [INFO] TestAgent.server: shutting down server +48:23.079 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:23.079 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:23.079 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:23.079 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:23.079 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:23.079 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:23.079 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:23.079 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:23.079 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:23.079 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:23.079 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:23.079 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:23.079 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:23.079 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:23.079 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:23.079Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:23.079Z [INFO] agent.router.manager: shutting down +48:23.080 [INFO] TestAgent: consul server down +48:23.080 [INFO] TestAgent: shutdown complete +48:23.080 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=tcp +48:23.080 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=udp +48:23.080 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14093 network=tcp +48:23.582 [INFO] TestAgent: Waiting for endpoints to shut down +48:23.582 [INFO] TestAgent: Endpoints down +--- PASS: TestBindingRuleCreateCommand (0.82s) --- PASS: TestBindingRuleCreateCommand/method_is_required (0.00s) --- PASS: TestBindingRuleCreateCommand/bind_type_required (0.00s) --- PASS: TestBindingRuleCreateCommand/bind_name_required (0.00s) --- PASS: TestBindingRuleCreateCommand/must_use_roughly_valid_selector (0.01s) - --- PASS: TestBindingRuleCreateCommand/create_it_with_no_selector (0.03s) - --- PASS: TestBindingRuleCreateCommand/create_it_with_a_match_selector (0.01s) + --- PASS: TestBindingRuleCreateCommand/create_it_with_no_selector (0.01s) + --- PASS: TestBindingRuleCreateCommand/create_it_with_a_match_selector (0.00s) --- PASS: TestBindingRuleCreateCommand/create_it_with_type_role (0.00s) +2022-12-04T05:48:22.865Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:22.888Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:22.895 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7cf6db1c-d588-b3e5-bdbc-542d5cd83964 Address:127.0.0.1:14090}]" +48:22.895 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= +48:22.896 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7cf6db1c-d588-b3e5-bdbc-542d5cd83964.dc1 127.0.0.1 +48:22.896 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7cf6db1c-d588-b3e5-bdbc-542d5cd83964 127.0.0.1 +2022-12-04T05:48:22.896Z [INFO] agent.router: Initializing LAN area manager +48:22.896 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp +48:22.897 [INFO] TestAgent.server: Adding LAN server: server="Node-7cf6db1c-d588-b3e5-bdbc-542d5cd83964 (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +48:22.897 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7cf6db1c-d588-b3e5-bdbc-542d5cd83964.dc1 area=wan +48:22.897 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp +48:22.897 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp +48:22.897 [INFO] TestAgent: started state syncer +48:22.898 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp +48:22.948 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:22.948 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 +48:22.948 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:22.948 [DEBUG] TestAgent.server.raft: vote granted: from=7cf6db1c-d588-b3e5-bdbc-542d5cd83964 term=2 tally=1 +48:22.948 [INFO] TestAgent.server.raft: election won: tally=1 +48:22.948 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" +48:22.948 [INFO] TestAgent.server: cluster leadership acquired +48:22.948 [INFO] TestAgent.server: initializing acls +48:22.949 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:22.949 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:22.949 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:22.950 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:22.950 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:22.950 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:22.950 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7cf6db1c-d588-b3e5-bdbc-542d5cd83964 +48:22.950 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7cf6db1c-d588-b3e5-bdbc-542d5cd83964.dc1 +48:22.952 [INFO] TestAgent.server: New leader elected: payload=Node-7cf6db1c-d588-b3e5-bdbc-542d5cd83964 +48:22.952 [INFO] TestAgent.server: Updating LAN server: server="Node-7cf6db1c-d588-b3e5-bdbc-542d5cd83964 (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +48:22.952 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7cf6db1c-d588-b3e5-bdbc-542d5cd83964.dc1 area=wan +48:22.955 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:22.997 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:22.997 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:22.997 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:22.997 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:22.997 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:22.997 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7cf6db1c-d588-b3e5-bdbc-542d5cd83964 +48:22.997 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7cf6db1c-d588-b3e5-bdbc-542d5cd83964 +48:22.998 [INFO] TestAgent.server: federation state anti-entropy synced +48:22.998 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7cf6db1c-d588-b3e5-bdbc-542d5cd83964 +48:23.157 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:45790 latency=1.098766ms +48:23.169 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:45792 latency=737.519µs +48:23.176 [INFO] TestAgent: Requesting shutdown +48:23.176 [INFO] TestAgent.server: shutting down server +48:23.176 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:23.176 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:23.176 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:23.176 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:23.176 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:23.176 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:23.176 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:23.176 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:23.176 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:23.176 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:23.176 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:23.176 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:23.177 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:23.177 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:23.177 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:23.177 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:23.177Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:23.177Z [INFO] agent.router.manager: shutting down +48:23.177 [INFO] TestAgent: consul server down +48:23.177 [INFO] TestAgent: shutdown complete +48:23.177 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp +48:23.177 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp +48:23.177 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp +48:23.680 [INFO] TestAgent: Waiting for endpoints to shut down +48:23.680 [INFO] TestAgent: Endpoints down +--- PASS: TestBindingRuleCreateCommand_JSON (0.92s) + --- PASS: TestBindingRuleCreateCommand_JSON/create_it_with_no_selector (0.02s) PASS -ok github.com/hashicorp/consul/command/acl/bindingrule/create 1.501s +ok github.com/hashicorp/consul/command/acl/bindingrule/create 1.095s === RUN TestBindingRuleDeleteCommand_noTabs === PAUSE TestBindingRuleDeleteCommand_noTabs === RUN TestBindingRuleDeleteCommand === PAUSE TestBindingRuleDeleteCommand === CONT TestBindingRuleDeleteCommand_noTabs === CONT TestBindingRuleDeleteCommand ---- PASS: TestBindingRuleDeleteCommand_noTabs (0.00s) [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestBindingRuleDeleteCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestBindingRuleDeleteCommand/id_required === RUN TestBindingRuleDeleteCommand/delete_works === RUN TestBindingRuleDeleteCommand/delete_works_via_prefixes === RUN TestBindingRuleDeleteCommand/delete_fails_when_prefix_matches_more_than_one_rule -2021-10-31T23:12:42.879Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:42.926Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:42.928 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a9ac9937-137a-8146-6129-956d4b8055fc Address:127.0.0.1:26342}]" -12:42.928 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:26342 [Follower]" leader= -12:42.929 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a9ac9937-137a-8146-6129-956d4b8055fc.dc1 127.0.0.1 -12:42.929 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a9ac9937-137a-8146-6129-956d4b8055fc 127.0.0.1 -2021-10-31T23:12:42.929Z [INFO] agent.router: Initializing LAN area manager -12:42.938 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26337 network=udp -12:42.938 [INFO] TestAgent.server: Adding LAN server: server="Node-a9ac9937-137a-8146-6129-956d4b8055fc (Addr: tcp/127.0.0.1:26342) (DC: dc1)" -12:42.938 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26337 network=tcp -12:42.938 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a9ac9937-137a-8146-6129-956d4b8055fc.dc1 area=wan -12:42.939 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:26338 network=tcp -12:42.939 [INFO] TestAgent: started state syncer -12:42.939 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:26343 network=tcp -12:42.985 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:42.985 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:26342 [Candidate]" term=2 -12:42.986 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:42.986 [DEBUG] TestAgent.server.raft: vote granted: from=a9ac9937-137a-8146-6129-956d4b8055fc term=2 tally=1 -12:42.986 [INFO] TestAgent.server.raft: election won: tally=1 -12:42.986 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:26342 [Leader]" -12:42.986 [INFO] TestAgent.server: cluster leadership acquired -12:42.986 [INFO] TestAgent.server: initializing acls -12:42.986 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:42.986 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:42.987 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:42.987 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:42.987 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:42.987 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:42.987 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a9ac9937-137a-8146-6129-956d4b8055fc -12:42.987 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a9ac9937-137a-8146-6129-956d4b8055fc.dc1 -12:42.988 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:42.990 [INFO] TestAgent.server: New leader elected: payload=Node-a9ac9937-137a-8146-6129-956d4b8055fc -12:42.990 [INFO] TestAgent.server: Updating LAN server: server="Node-a9ac9937-137a-8146-6129-956d4b8055fc (Addr: tcp/127.0.0.1:26342) (DC: dc1)" -12:42.990 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a9ac9937-137a-8146-6129-956d4b8055fc.dc1 area=wan -12:43.034 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:43.034 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:43.034 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:43.034 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:43.034 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:43.034 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a9ac9937-137a-8146-6129-956d4b8055fc -12:43.034 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a9ac9937-137a-8146-6129-956d4b8055fc -12:43.035 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a9ac9937-137a-8146-6129-956d4b8055fc -12:43.035 [INFO] TestAgent.server: federation state anti-entropy synced -12:43.054 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:60880 latency=6.131444ms -12:43.063 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60880 latency=1.65403ms -12:43.079 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/cb35f4a5-29bb-f78c-16e8-20f342a5e2fb from=127.0.0.1:60882 latency=795.348µs -12:43.082 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/cb35f4a5-29bb-f78c-16e8-20f342a5e2fb from=127.0.0.1:60880 latency=150.323µs -12:43.083 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60880 latency=755.249µs -12:43.085 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:60884 latency=328.238µs -12:43.089 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/1b4ab2c2-44ae-595e-913e-34c62126a4ca from=127.0.0.1:60884 latency=1.147622ms -12:43.098 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/1b4ab2c2-44ae-595e-913e-34c62126a4ca from=127.0.0.1:60880 latency=127.245µs -12:43.098 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:60880 latency=249.012µs -12:43.099 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60880 latency=752.628µs -12:43.100 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:60880 latency=163.829µs -12:43.101 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60880 latency=688.977µs -12:43.102 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:60880 latency=626.796µs -12:43.108 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60880 latency=886.769µs -12:43.109 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:60880 latency=218.437µs -12:43.110 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60880 latency=764.644µs -12:43.114 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:60880 latency=369.01µs -12:43.116 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60880 latency=935.205µs -12:43.116 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:60880 latency=212.749µs -12:43.121 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60880 latency=3.866974ms -12:43.128 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:60880 latency=1.894639ms -12:43.155 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60880 latency=1.645669ms -12:43.159 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:60880 latency=627.711µs -12:43.161 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:60886 latency=287.806µs -12:43.161 [INFO] TestAgent: Requesting shutdown -12:43.161 [INFO] TestAgent.server: shutting down server -12:43.161 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:43.161 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:43.161 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:43.161 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:43.161 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:43.161 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:43.161 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:43.161 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:43.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:43.161 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -12:43.161 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:43.161 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:43.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:43.161 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:43.161 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:43.161 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:43.166Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:43.166Z [INFO] agent.router.manager: shutting down -12:43.166 [INFO] TestAgent: consul server down -12:43.166 [INFO] TestAgent: shutdown complete -12:43.166 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26337 network=tcp -12:43.166 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26337 network=udp -12:43.166 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:26338 network=tcp -12:43.668 [INFO] TestAgent: Waiting for endpoints to shut down -12:43.668 [INFO] TestAgent: Endpoints down ---- PASS: TestBindingRuleDeleteCommand (0.89s) +2022-12-04T05:48:23.958Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:23.974Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:23.977 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8c7da91d-436f-515a-105e-bf2e801dd7db Address:127.0.0.1:18174}]" +48:23.977 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18174 [Follower]" leader= +48:23.977 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8c7da91d-436f-515a-105e-bf2e801dd7db.dc1 127.0.0.1 +48:23.978 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8c7da91d-436f-515a-105e-bf2e801dd7db 127.0.0.1 +2022-12-04T05:48:23.978Z [INFO] agent.router: Initializing LAN area manager +48:23.978 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=udp +48:23.978 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8c7da91d-436f-515a-105e-bf2e801dd7db.dc1 area=wan +48:23.978 [INFO] TestAgent.server: Adding LAN server: server="Node-8c7da91d-436f-515a-105e-bf2e801dd7db (Addr: tcp/127.0.0.1:18174) (DC: dc1)" +48:23.978 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=tcp +48:23.979 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18170 network=tcp +48:23.979 [INFO] TestAgent: started state syncer +48:23.979 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18175 network=tcp +48:24.032 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:24.032 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:24.032 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18174 [Candidate]" term=2 +48:24.032 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:24.032 [DEBUG] TestAgent.server.raft: vote granted: from=8c7da91d-436f-515a-105e-bf2e801dd7db term=2 tally=1 +48:24.032 [INFO] TestAgent.server.raft: election won: tally=1 +48:24.032 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18174 [Leader]" +48:24.032 [INFO] TestAgent.server: cluster leadership acquired +48:24.033 [INFO] TestAgent.server: New leader elected: payload=Node-8c7da91d-436f-515a-105e-bf2e801dd7db +48:24.033 [INFO] TestAgent.server: initializing acls +48:24.033 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:24.033 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:24.033 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:24.034 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:24.034 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:24.034 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:24.034 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-8c7da91d-436f-515a-105e-bf2e801dd7db +48:24.034 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-8c7da91d-436f-515a-105e-bf2e801dd7db.dc1 +48:24.034 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:24.036 [INFO] TestAgent.server: Updating LAN server: server="Node-8c7da91d-436f-515a-105e-bf2e801dd7db (Addr: tcp/127.0.0.1:18174) (DC: dc1)" +48:24.036 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-8c7da91d-436f-515a-105e-bf2e801dd7db.dc1 area=wan +48:24.065 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:24.065 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:24.065 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:24.065 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:24.065 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:24.065 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8c7da91d-436f-515a-105e-bf2e801dd7db +48:24.065 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8c7da91d-436f-515a-105e-bf2e801dd7db +48:24.076 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8c7da91d-436f-515a-105e-bf2e801dd7db +48:24.085 [INFO] TestAgent.server: federation state anti-entropy synced +48:24.107 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:50284 latency=1.912347ms +48:24.113 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50284 latency=1.369257ms +48:24.118 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/944337b6-f86d-040f-8dbb-ad24c8b56f31 from=127.0.0.1:50286 latency=624.735µs +48:24.119 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/944337b6-f86d-040f-8dbb-ad24c8b56f31 from=127.0.0.1:50284 latency=84.624µs +48:24.120 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50284 latency=990.014µs +48:24.126 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:50288 latency=385.624µs +48:24.145 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/a7894b94-619b-a1d2-be22-36302636c4c9 from=127.0.0.1:50288 latency=5.264636ms +48:24.149 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/a7894b94-619b-a1d2-be22-36302636c4c9 from=127.0.0.1:50284 latency=130.593µs +48:24.150 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:50284 latency=280.804µs +48:24.151 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50284 latency=828.447µs +48:24.151 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:50284 latency=197.53µs +48:24.158 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50284 latency=1.674086ms +48:24.159 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:50284 latency=313.041µs +48:24.160 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50284 latency=717.713µs +48:24.161 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:50284 latency=185.111µs +48:24.162 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:50290 latency=366.845µs +48:24.162 [INFO] TestAgent: Requesting shutdown +48:24.162 [INFO] TestAgent.server: shutting down server +48:24.162 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:24.162 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:24.162 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:24.163 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:24.163 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:24.163 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:24.163 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:24.163 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:24.163 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:24.163 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:24.163 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:24.163 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:24.163 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:24.163 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:24.163 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:24.163 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:24.163Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:24.163Z [INFO] agent.router.manager: shutting down +48:24.163 [INFO] TestAgent: consul server down +48:24.163 [INFO] TestAgent: shutdown complete +48:24.163 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=tcp +48:24.163 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=udp +48:24.163 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18170 network=tcp +48:24.664 [INFO] TestAgent: Waiting for endpoints to shut down +48:24.664 [INFO] TestAgent: Endpoints down +--- PASS: TestBindingRuleDeleteCommand (0.77s) --- PASS: TestBindingRuleDeleteCommand/id_required (0.00s) - --- PASS: TestBindingRuleDeleteCommand/delete_works (0.03s) - --- PASS: TestBindingRuleDeleteCommand/delete_works_via_prefixes (0.02s) - --- PASS: TestBindingRuleDeleteCommand/delete_fails_when_prefix_matches_more_than_one_rule (0.06s) + --- PASS: TestBindingRuleDeleteCommand/delete_works (0.01s) + --- PASS: TestBindingRuleDeleteCommand/delete_works_via_prefixes (0.03s) + --- PASS: TestBindingRuleDeleteCommand/delete_fails_when_prefix_matches_more_than_one_rule (0.01s) PASS -ok github.com/hashicorp/consul/command/acl/bindingrule/delete 1.674s +ok github.com/hashicorp/consul/command/acl/bindingrule/delete 0.858s === RUN TestBindingRuleListCommand_noTabs === PAUSE TestBindingRuleListCommand_noTabs === RUN TestBindingRuleListCommand === PAUSE TestBindingRuleListCommand === CONT TestBindingRuleListCommand_noTabs ---- PASS: TestBindingRuleListCommand_noTabs (0.00s) === CONT TestBindingRuleListCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestBindingRuleListCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestBindingRuleListCommand/normal === RUN TestBindingRuleListCommand/filter_by_method_1 === RUN TestBindingRuleListCommand/filter_by_method_2 === RUN TestBindingRuleListCommand/normal_json_formatted -2021-10-31T23:12:52.202Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:12:52.254Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -12:52.257 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7621b20a-aa63-b7fb-c360-cfd43348be08 Address:127.0.0.1:26342}]" -12:52.257 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:26342 [Follower]" leader= -12:52.270 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7621b20a-aa63-b7fb-c360-cfd43348be08.dc1 127.0.0.1 -12:52.270 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7621b20a-aa63-b7fb-c360-cfd43348be08 127.0.0.1 -2021-10-31T23:12:52.270Z [INFO] agent.router: Initializing LAN area manager -12:52.271 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26337 network=udp -12:52.271 [INFO] TestAgent.server: Adding LAN server: server="Node-7621b20a-aa63-b7fb-c360-cfd43348be08 (Addr: tcp/127.0.0.1:26342) (DC: dc1)" -12:52.271 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7621b20a-aa63-b7fb-c360-cfd43348be08.dc1 area=wan -12:52.271 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26337 network=tcp -12:52.272 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:26338 network=tcp -12:52.272 [INFO] TestAgent: started state syncer -12:52.272 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:26343 network=tcp -12:52.301 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -12:52.301 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:26342 [Candidate]" term=2 -12:52.301 [DEBUG] TestAgent.server.raft: votes: needed=1 -12:52.301 [DEBUG] TestAgent.server.raft: vote granted: from=7621b20a-aa63-b7fb-c360-cfd43348be08 term=2 tally=1 -12:52.301 [INFO] TestAgent.server.raft: election won: tally=1 -12:52.301 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:26342 [Leader]" -12:52.301 [INFO] TestAgent.server: cluster leadership acquired -12:52.301 [INFO] TestAgent.server: initializing acls -12:52.301 [INFO] TestAgent.server: Created ACL 'global-management' policy -12:52.301 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -12:52.302 [INFO] TestAgent.server: New leader elected: payload=Node-7621b20a-aa63-b7fb-c360-cfd43348be08 -12:52.302 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -12:52.302 [INFO] TestAgent.server: Created ACL anonymous token from configuration -12:52.302 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -12:52.302 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -12:52.303 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7621b20a-aa63-b7fb-c360-cfd43348be08 -12:52.303 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7621b20a-aa63-b7fb-c360-cfd43348be08.dc1 -12:52.303 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -12:52.305 [INFO] TestAgent.server: Updating LAN server: server="Node-7621b20a-aa63-b7fb-c360-cfd43348be08 (Addr: tcp/127.0.0.1:26342) (DC: dc1)" -12:52.306 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7621b20a-aa63-b7fb-c360-cfd43348be08.dc1 area=wan -12:52.338 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -12:52.338 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -12:52.338 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -12:52.338 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -12:52.338 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -12:52.338 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7621b20a-aa63-b7fb-c360-cfd43348be08 -12:52.338 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7621b20a-aa63-b7fb-c360-cfd43348be08 -12:52.339 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7621b20a-aa63-b7fb-c360-cfd43348be08 -12:52.339 [INFO] TestAgent.server: federation state anti-entropy synced -12:52.522 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -12:52.523 [INFO] TestAgent: Synced node info -12:52.523 [DEBUG] TestAgent: Node info in sync -12:52.722 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:60912 latency=18.179008ms -12:52.742 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:60912 latency=18.226206ms -12:52.752 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60912 latency=1.092278ms -12:52.753 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60912 latency=600.884µs -12:52.755 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60912 latency=1.012944ms -12:52.771 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60912 latency=1.372723ms -12:52.779 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60912 latency=1.046038ms -12:52.780 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60912 latency=698.041µs -12:52.781 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60912 latency=685.839µs -12:52.790 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60912 latency=9.110389ms -12:52.795 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60912 latency=1.051201ms -12:52.796 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60912 latency=762.966µs -12:52.802 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:60914 latency=413.366µs -12:52.804 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test-1 from=127.0.0.1:60916 latency=225.946µs -12:52.806 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test-2 from=127.0.0.1:60918 latency=628.852µs -12:52.815 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:60920 latency=563.069µs -12:52.816 [INFO] TestAgent: Requesting shutdown -12:52.816 [INFO] TestAgent.server: shutting down server -12:52.816 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -12:52.816 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -12:52.816 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -12:52.816 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -12:52.816 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -12:52.816 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -12:52.816 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -12:52.816 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -12:52.816 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -12:52.816 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -12:52.816 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -12:52.816 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -12:52.816 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -12:52.816 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -12:52.816 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:12:52.816Z [INFO] agent.router.manager: shutting down -2021-10-31T23:12:52.816Z [INFO] agent.router.manager: shutting down -12:52.816 [INFO] TestAgent: consul server down -12:52.816 [INFO] TestAgent: shutdown complete -12:52.816 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26337 network=tcp -12:52.817 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26337 network=udp -12:52.817 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:26338 network=tcp -12:53.317 [INFO] TestAgent: Waiting for endpoints to shut down -12:53.317 [INFO] TestAgent: Endpoints down ---- PASS: TestBindingRuleListCommand (1.23s) - --- PASS: TestBindingRuleListCommand/normal (0.01s) +2022-12-04T05:48:30.899Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:30.914Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:30.925 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:78e62883-a10d-77d1-d1d2-3ed9e98babe8 Address:127.0.0.1:20216}]" +48:30.926 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20216 [Follower]" leader= +48:30.926 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-78e62883-a10d-77d1-d1d2-3ed9e98babe8.dc1 127.0.0.1 +48:30.926 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-78e62883-a10d-77d1-d1d2-3ed9e98babe8 127.0.0.1 +2022-12-04T05:48:30.926Z [INFO] agent.router: Initializing LAN area manager +48:30.927 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=udp +48:30.927 [INFO] TestAgent.server: Adding LAN server: server="Node-78e62883-a10d-77d1-d1d2-3ed9e98babe8 (Addr: tcp/127.0.0.1:20216) (DC: dc1)" +48:30.927 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-78e62883-a10d-77d1-d1d2-3ed9e98babe8.dc1 area=wan +48:30.927 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=tcp +48:30.927 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20212 network=tcp +48:30.927 [INFO] TestAgent: started state syncer +48:30.927 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20217 network=tcp +48:30.966 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:30.966 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20216 [Candidate]" term=2 +48:30.966 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:30.966 [DEBUG] TestAgent.server.raft: vote granted: from=78e62883-a10d-77d1-d1d2-3ed9e98babe8 term=2 tally=1 +48:30.966 [INFO] TestAgent.server.raft: election won: tally=1 +48:30.966 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20216 [Leader]" +48:30.966 [INFO] TestAgent.server: cluster leadership acquired +48:30.966 [INFO] TestAgent.server: initializing acls +48:30.967 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:30.967 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:30.967 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:30.967 [INFO] TestAgent.server: New leader elected: payload=Node-78e62883-a10d-77d1-d1d2-3ed9e98babe8 +48:30.967 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:30.967 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:30.967 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:30.968 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-78e62883-a10d-77d1-d1d2-3ed9e98babe8 +48:30.968 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-78e62883-a10d-77d1-d1d2-3ed9e98babe8.dc1 +48:30.968 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-78e62883-a10d-77d1-d1d2-3ed9e98babe8.dc1 area=wan +48:30.968 [INFO] TestAgent.server: Updating LAN server: server="Node-78e62883-a10d-77d1-d1d2-3ed9e98babe8 (Addr: tcp/127.0.0.1:20216) (DC: dc1)" +48:30.968 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:30.984 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:30.984 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:30.984 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:30.984 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:30.984 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:30.984 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-78e62883-a10d-77d1-d1d2-3ed9e98babe8 +48:30.984 [INFO] TestAgent.server: member joined, marking health alive: member=Node-78e62883-a10d-77d1-d1d2-3ed9e98babe8 +48:30.985 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-78e62883-a10d-77d1-d1d2-3ed9e98babe8 +48:30.985 [INFO] TestAgent.server: federation state anti-entropy synced +48:31.098 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:31.099 [INFO] TestAgent: Synced node info +48:31.099 [DEBUG] TestAgent: Node info in sync +48:31.130 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42742 latency=4.90278ms +48:31.132 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42742 latency=865.954µs +48:31.134 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:42742 latency=903.993µs +48:31.138 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:42742 latency=1.054767ms +48:31.140 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:42742 latency=575.85µs +48:31.141 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:42742 latency=812.571µs +48:31.142 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:42742 latency=621.79µs +48:31.143 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:42742 latency=902.952µs +48:31.145 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:42742 latency=614.333µs +48:31.146 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:42742 latency=628.785µs +48:31.147 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:42742 latency=719.355µs +48:31.148 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:42742 latency=453.541µs +48:31.149 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:42744 latency=302.013µs +48:31.151 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test-1 from=127.0.0.1:42746 latency=258.259µs +48:31.153 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test-2 from=127.0.0.1:42748 latency=200.023µs +48:31.155 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:42750 latency=254.365µs +48:31.156 [INFO] TestAgent: Requesting shutdown +48:31.156 [INFO] TestAgent.server: shutting down server +48:31.156 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:31.156 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:31.156 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:31.156 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:31.156 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:31.156 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:31.156 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:31.156 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:31.156 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:31.156 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:31.156 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:31.156 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:31.156 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:31.156 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:31.156 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:31.156Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:31.156Z [INFO] agent.router.manager: shutting down +48:31.156 [INFO] TestAgent: consul server down +48:31.156 [INFO] TestAgent: shutdown complete +48:31.156 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=tcp +48:31.156 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=udp +48:31.156 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20212 network=tcp +48:31.656 [INFO] TestAgent: Waiting for endpoints to shut down +48:31.656 [INFO] TestAgent: Endpoints down +--- PASS: TestBindingRuleListCommand (0.81s) + --- PASS: TestBindingRuleListCommand/normal (0.00s) --- PASS: TestBindingRuleListCommand/filter_by_method_1 (0.00s) - --- PASS: TestBindingRuleListCommand/filter_by_method_2 (0.01s) - --- PASS: TestBindingRuleListCommand/normal_json_formatted (0.01s) + --- PASS: TestBindingRuleListCommand/filter_by_method_2 (0.00s) + --- PASS: TestBindingRuleListCommand/normal_json_formatted (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/bindingrule/list 1.719s +ok github.com/hashicorp/consul/command/acl/bindingrule/list 0.856s === RUN TestBindingRuleReadCommand_noTabs === PAUSE TestBindingRuleReadCommand_noTabs === RUN TestBindingRuleReadCommand === PAUSE TestBindingRuleReadCommand === CONT TestBindingRuleReadCommand_noTabs ---- PASS: TestBindingRuleReadCommand_noTabs (0.00s) === CONT TestBindingRuleReadCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestBindingRuleReadCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestBindingRuleReadCommand/id_required @@ -74340,94 +74473,98 @@ === RUN TestBindingRuleReadCommand/read_by_id === RUN TestBindingRuleReadCommand/read_by_id_prefix === RUN TestBindingRuleReadCommand/read_by_id_json_formatted -2021-10-31T23:13:03.057Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:03.099Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:03.115 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fd3dff2e-d8dd-f3b5-d661-2b7e16d1d2f7 Address:127.0.0.1:22258}]" -13:03.115 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fd3dff2e-d8dd-f3b5-d661-2b7e16d1d2f7.dc1 127.0.0.1 -13:03.116 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fd3dff2e-d8dd-f3b5-d661-2b7e16d1d2f7 127.0.0.1 -2021-10-31T23:13:03.116Z [INFO] agent.router: Initializing LAN area manager -13:03.116 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=udp -13:03.116 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22258 [Follower]" leader= -13:03.117 [INFO] TestAgent.server: Adding LAN server: server="Node-fd3dff2e-d8dd-f3b5-d661-2b7e16d1d2f7 (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -13:03.117 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fd3dff2e-d8dd-f3b5-d661-2b7e16d1d2f7.dc1 area=wan -13:03.117 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=tcp -13:03.117 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22254 network=tcp -13:03.117 [INFO] TestAgent: started state syncer -13:03.117 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22259 network=tcp -13:03.161 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:03.162 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22258 [Candidate]" term=2 -13:03.162 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:03.162 [DEBUG] TestAgent.server.raft: vote granted: from=fd3dff2e-d8dd-f3b5-d661-2b7e16d1d2f7 term=2 tally=1 -13:03.162 [INFO] TestAgent.server.raft: election won: tally=1 -13:03.162 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22258 [Leader]" -13:03.162 [INFO] TestAgent.server: cluster leadership acquired -13:03.162 [INFO] TestAgent.server: initializing acls -13:03.162 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:03.162 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:03.163 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:03.163 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:03.163 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:03.163 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:03.163 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-fd3dff2e-d8dd-f3b5-d661-2b7e16d1d2f7 -13:03.163 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-fd3dff2e-d8dd-f3b5-d661-2b7e16d1d2f7.dc1 -13:03.164 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:03.173 [INFO] TestAgent.server: New leader elected: payload=Node-fd3dff2e-d8dd-f3b5-d661-2b7e16d1d2f7 -13:03.173 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-fd3dff2e-d8dd-f3b5-d661-2b7e16d1d2f7.dc1 area=wan -13:03.174 [INFO] TestAgent.server: Updating LAN server: server="Node-fd3dff2e-d8dd-f3b5-d661-2b7e16d1d2f7 (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -13:03.235 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:03.235 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:03.235 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:03.235 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:03.235 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:03.235 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fd3dff2e-d8dd-f3b5-d661-2b7e16d1d2f7 -13:03.235 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fd3dff2e-d8dd-f3b5-d661-2b7e16d1d2f7 -13:03.236 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fd3dff2e-d8dd-f3b5-d661-2b7e16d1d2f7 -13:03.237 [INFO] TestAgent.server: federation state anti-entropy synced -13:03.405 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:03.405 [INFO] TestAgent: Synced node info -13:03.408 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:49326 latency=1.08148ms -13:03.414 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/2928f11d-3e50-addd-1a5a-a1c3a0faa2af from=127.0.0.1:49328 latency=264.774µs -13:03.423 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:49326 latency=1.419632ms -13:03.438 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/91a8f4ea-c8d2-e2b5-7884-da0b31069a0f from=127.0.0.1:49330 latency=10.096141ms -13:03.448 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:49326 latency=2.017726ms -13:03.450 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:49332 latency=526.084µs -13:03.462 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/5f1e943b-45de-5158-cc51-721c6d9943cd from=127.0.0.1:49332 latency=315.604µs -13:03.468 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:49326 latency=824.423µs -13:03.469 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/3dca56b5-a297-444a-f6e3-9dd3a8bcf9c3 from=127.0.0.1:49334 latency=362.541µs -13:03.474 [INFO] TestAgent: Requesting shutdown -13:03.474 [INFO] TestAgent.server: shutting down server -13:03.474 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:03.474 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:03.474 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:03.474 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:03.474 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:03.474 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:03.474 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:03.474 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:03.474 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:03.474 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:03.474 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:03.474 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:03.474 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:03.474 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:03.474 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:03.474Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:03.474Z [INFO] agent.router.manager: shutting down -13:03.474 [INFO] TestAgent: consul server down -13:03.474 [INFO] TestAgent: shutdown complete -13:03.474 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=tcp -13:03.474 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=udp -13:03.474 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22254 network=tcp -13:03.977 [INFO] TestAgent: Waiting for endpoints to shut down -13:03.977 [INFO] TestAgent: Endpoints down ---- PASS: TestBindingRuleReadCommand (1.08s) +2022-12-04T05:48:32.918Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:32.950Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:32.952 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f2718d4e-6a4d-fe2c-bc2a-63e021dfdbfd Address:127.0.0.1:24300}]" +48:32.953 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f2718d4e-6a4d-fe2c-bc2a-63e021dfdbfd.dc1 127.0.0.1 +48:32.954 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f2718d4e-6a4d-fe2c-bc2a-63e021dfdbfd 127.0.0.1 +2022-12-04T05:48:32.954Z [INFO] agent.router: Initializing LAN area manager +48:32.954 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp +48:32.954 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= +48:32.954 [INFO] TestAgent.server: Adding LAN server: server="Node-f2718d4e-6a4d-fe2c-bc2a-63e021dfdbfd (Addr: tcp/127.0.0.1:24300) (DC: dc1)" +48:32.954 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f2718d4e-6a4d-fe2c-bc2a-63e021dfdbfd.dc1 area=wan +48:32.954 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp +48:32.955 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp +48:32.955 [INFO] TestAgent: started state syncer +48:32.955 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp +48:33.005 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:33.016 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:33.016 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 +48:33.016 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:33.016 [DEBUG] TestAgent.server.raft: vote granted: from=f2718d4e-6a4d-fe2c-bc2a-63e021dfdbfd term=2 tally=1 +48:33.016 [INFO] TestAgent.server.raft: election won: tally=1 +48:33.016 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" +48:33.016 [INFO] TestAgent.server: cluster leadership acquired +48:33.016 [INFO] TestAgent.server: initializing acls +48:33.017 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:33.017 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:33.017 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:33.018 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:33.018 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:33.018 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:33.018 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f2718d4e-6a4d-fe2c-bc2a-63e021dfdbfd +48:33.018 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f2718d4e-6a4d-fe2c-bc2a-63e021dfdbfd.dc1 +48:33.018 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:33.020 [INFO] TestAgent.server: New leader elected: payload=Node-f2718d4e-6a4d-fe2c-bc2a-63e021dfdbfd +48:33.020 [INFO] TestAgent.server: Updating LAN server: server="Node-f2718d4e-6a4d-fe2c-bc2a-63e021dfdbfd (Addr: tcp/127.0.0.1:24300) (DC: dc1)" +48:33.020 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f2718d4e-6a4d-fe2c-bc2a-63e021dfdbfd.dc1 area=wan +48:33.043 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:33.043 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:33.043 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:33.043 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:33.043 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:33.044 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f2718d4e-6a4d-fe2c-bc2a-63e021dfdbfd +48:33.044 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f2718d4e-6a4d-fe2c-bc2a-63e021dfdbfd +48:33.045 [INFO] TestAgent.server: federation state anti-entropy synced +48:33.045 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f2718d4e-6a4d-fe2c-bc2a-63e021dfdbfd +48:33.178 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:33.179 [INFO] TestAgent: Synced node info +48:33.204 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:32870 latency=1.385399ms +48:33.208 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:33.208 [DEBUG] TestAgent: Node info in sync +48:33.208 [DEBUG] TestAgent: Node info in sync +48:33.213 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/81f91d92-72b1-808e-4183-edf93d0957fa from=127.0.0.1:32872 latency=245.874µs +48:33.214 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:32870 latency=1.000757ms +48:33.216 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/377500e4-4d84-bf8f-037a-f7a1a1f15b34 from=127.0.0.1:32874 latency=272.087µs +48:33.221 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:32870 latency=1.082519ms +48:33.230 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:32878 latency=371.625µs +48:33.248 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/d4f535c2-c291-f020-6f62-fb22c998c77a from=127.0.0.1:32878 latency=8.216774ms +48:33.254 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:32870 latency=1.189008ms +48:33.256 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/cc18457a-84e5-a322-f170-275a794aa0ee from=127.0.0.1:32880 latency=297.231µs +48:33.257 [INFO] TestAgent: Requesting shutdown +48:33.257 [INFO] TestAgent.server: shutting down server +48:33.257 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:33.257 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:33.257 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:33.257 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:33.257 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:33.257 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:33.257 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:33.257 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:33.257 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:33.257 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:33.257 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:33.257 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:33.257 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:33.257 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:33.257 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:33.257Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:33.257Z [INFO] agent.router.manager: shutting down +48:33.257 [INFO] TestAgent: consul server down +48:33.257 [INFO] TestAgent: shutdown complete +48:33.257 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp +48:33.257 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp +48:33.257 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp +48:33.760 [INFO] TestAgent: Waiting for endpoints to shut down +48:33.760 [INFO] TestAgent: Endpoints down +--- PASS: TestBindingRuleReadCommand (0.89s) --- PASS: TestBindingRuleReadCommand/id_required (0.00s) - --- PASS: TestBindingRuleReadCommand/read_by_id_not_found (0.01s) - --- PASS: TestBindingRuleReadCommand/read_by_id (0.03s) - --- PASS: TestBindingRuleReadCommand/read_by_id_prefix (0.02s) - --- PASS: TestBindingRuleReadCommand/read_by_id_json_formatted (0.01s) + --- PASS: TestBindingRuleReadCommand/read_by_id_not_found (0.00s) + --- PASS: TestBindingRuleReadCommand/read_by_id (0.00s) + --- PASS: TestBindingRuleReadCommand/read_by_id_prefix (0.04s) + --- PASS: TestBindingRuleReadCommand/read_by_id_json_formatted (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/bindingrule/read 1.909s +ok github.com/hashicorp/consul/command/acl/bindingrule/read 1.175s === RUN TestBindingRuleUpdateCommand_noTabs === PAUSE TestBindingRuleUpdateCommand_noTabs === RUN TestBindingRuleUpdateCommand @@ -74435,296 +74572,332 @@ === RUN TestBindingRuleUpdateCommand_noMerge === PAUSE TestBindingRuleUpdateCommand_noMerge === CONT TestBindingRuleUpdateCommand_noTabs ---- PASS: TestBindingRuleUpdateCommand_noTabs (0.00s) === CONT TestBindingRuleUpdateCommand_noMerge -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestBindingRuleUpdateCommand_noTabs (0.00s) === CONT TestBindingRuleUpdateCommand +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -=== RUN TestBindingRuleUpdateCommand_noMerge/rule_id_required -=== RUN TestBindingRuleUpdateCommand_noMerge/rule_id_partial_matches_nothing -=== RUN TestBindingRuleUpdateCommand_noMerge/rule_id_exact_match_doesn't_exist -=== RUN TestBindingRuleUpdateCommand_noMerge/rule_id_partial_matches_multiple -=== RUN TestBindingRuleUpdateCommand_noMerge/must_use_roughly_valid_selector -=== RUN TestBindingRuleUpdateCommand_noMerge/update_all_fields -=== RUN TestBindingRuleUpdateCommand_noMerge/update_all_fields_-_partial -=== RUN TestBindingRuleUpdateCommand_noMerge/update_all_fields_but_description -=== RUN TestBindingRuleUpdateCommand_noMerge/missing_bind_name -=== RUN TestBindingRuleUpdateCommand_noMerge/update_all_fields_but_selector === RUN TestBindingRuleUpdateCommand/rule_id_required === RUN TestBindingRuleUpdateCommand/rule_id_partial_matches_nothing === RUN TestBindingRuleUpdateCommand/rule_id_exact_match_doesn't_exist === RUN TestBindingRuleUpdateCommand/rule_id_partial_matches_multiple +=== RUN TestBindingRuleUpdateCommand_noMerge/rule_id_required +=== RUN TestBindingRuleUpdateCommand_noMerge/rule_id_partial_matches_nothing +=== RUN TestBindingRuleUpdateCommand_noMerge/rule_id_exact_match_doesn't_exist +=== RUN TestBindingRuleUpdateCommand_noMerge/rule_id_partial_matches_multiple === RUN TestBindingRuleUpdateCommand/must_use_roughly_valid_selector === RUN TestBindingRuleUpdateCommand/update_all_fields === RUN TestBindingRuleUpdateCommand/update_all_fields_-_partial +=== RUN TestBindingRuleUpdateCommand_noMerge/must_use_roughly_valid_selector +=== RUN TestBindingRuleUpdateCommand_noMerge/update_all_fields +=== RUN TestBindingRuleUpdateCommand_noMerge/update_all_fields_-_partial === RUN TestBindingRuleUpdateCommand/update_all_fields_but_description === RUN TestBindingRuleUpdateCommand/update_all_fields_but_bind_name === RUN TestBindingRuleUpdateCommand/update_all_fields_but_must_exist === RUN TestBindingRuleUpdateCommand/update_all_fields_but_selector +=== RUN TestBindingRuleUpdateCommand_noMerge/update_all_fields_but_description === RUN TestBindingRuleUpdateCommand/update_all_fields_clear_selector +=== RUN TestBindingRuleUpdateCommand_noMerge/missing_bind_name === RUN TestBindingRuleUpdateCommand/update_all_fields_json_formatted -2021-10-31T23:13:02.366Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:02.409Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:02.420 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a1e32fae-e31f-b39d-dda2-9c26b0253546 Address:127.0.0.1:13069}]" -13:02.420 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a1e32fae-e31f-b39d-dda2-9c26b0253546.dc1 127.0.0.1 -13:02.421 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a1e32fae-e31f-b39d-dda2-9c26b0253546 127.0.0.1 -2021-10-31T23:13:02.421Z [INFO] agent.router: Initializing LAN area manager -13:02.421 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp -13:02.421 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= -13:02.454 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a1e32fae-e31f-b39d-dda2-9c26b0253546.dc1 area=wan -13:02.454 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp -13:02.455 [INFO] TestAgent.server: Adding LAN server: server="Node-a1e32fae-e31f-b39d-dda2-9c26b0253546 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" -13:02.456 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp -13:02.456 [INFO] TestAgent: started state syncer -13:02.456 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp -13:02.496 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:02.496 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 -13:02.496 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:02.496 [DEBUG] TestAgent.server.raft: vote granted: from=a1e32fae-e31f-b39d-dda2-9c26b0253546 term=2 tally=1 -13:02.496 [INFO] TestAgent.server.raft: election won: tally=1 -13:02.496 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" -13:02.496 [INFO] TestAgent.server: cluster leadership acquired -13:02.497 [INFO] TestAgent.server: initializing acls -13:02.497 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:02.497 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:02.497 [INFO] TestAgent.server: New leader elected: payload=Node-a1e32fae-e31f-b39d-dda2-9c26b0253546 -13:02.502 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:02.502 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:02.502 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:02.502 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:02.502 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a1e32fae-e31f-b39d-dda2-9c26b0253546 -13:02.502 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a1e32fae-e31f-b39d-dda2-9c26b0253546.dc1 -13:02.503 [INFO] TestAgent.server: Updating LAN server: server="Node-a1e32fae-e31f-b39d-dda2-9c26b0253546 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" -13:02.503 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a1e32fae-e31f-b39d-dda2-9c26b0253546.dc1 area=wan -13:02.503 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:02.557 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:02.557 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:02.557 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:02.557 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:02.557 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:02.557 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1e32fae-e31f-b39d-dda2-9c26b0253546 -13:02.557 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a1e32fae-e31f-b39d-dda2-9c26b0253546 -13:02.564 [INFO] TestAgent.server: federation state anti-entropy synced -13:02.564 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1e32fae-e31f-b39d-dda2-9c26b0253546 -13:02.590 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:51368 latency=4.003088ms -13:02.603 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:51370 latency=1.417239ms -13:02.605 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/cc71218c-e000-8c43-0151-e6faa857e979 from=127.0.0.1:51372 latency=204.263µs -13:02.615 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:51368 latency=927.831µs -13:02.616 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:51368 latency=1.039316ms -13:02.617 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:51368 latency=204.9µs -13:02.622 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:51368 latency=989.38µs -13:02.623 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:51368 latency=194.84µs -13:02.624 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:51368 latency=691.231µs -13:02.625 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:51368 latency=203.217µs -13:02.643 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:51374 latency=392.156µs -13:02.644 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:51368 latency=880.72µs -13:02.650 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/48ee1ff8-4099-2abd-53e8-58cbf0b0b2ab from=127.0.0.1:51376 latency=391.015µs -13:02.651 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule/48ee1ff8-4099-2abd-53e8-58cbf0b0b2ab from=127.0.0.1:51376 error="invalid Binding Rule: Selector is invalid: 1:4 (3): no match found, expected: "!=", ".", "==", "[", [ \t\r\n] or [a-zA-Z0-9_]" -13:02.651 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/48ee1ff8-4099-2abd-53e8-58cbf0b0b2ab from=127.0.0.1:51376 latency=690.635µs -13:02.653 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:51368 latency=857.669µs -13:02.662 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/dbe37c60-2b34-b99a-b417-01f1e2745363 from=127.0.0.1:51378 latency=318.156µs -13:02.671 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/dbe37c60-2b34-b99a-b417-01f1e2745363 from=127.0.0.1:51378 latency=1.177221ms -13:02.671 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/dbe37c60-2b34-b99a-b417-01f1e2745363 from=127.0.0.1:51368 latency=174.807µs -13:02.691 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:51368 latency=611.324µs -13:02.693 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/48ee1ff8-4099-2abd-53e8-58cbf0b0b2ab from=127.0.0.1:51368 latency=580.91µs -13:02.694 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/8904ef76-f1ef-ffa0-505d-4e660b3be161 from=127.0.0.1:51380 latency=584.203µs -13:02.707 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/8ab3174a-adbd-a749-3642-a1a181ecff56 from=127.0.0.1:51382 latency=553.507µs -13:02.708 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/beefb289-99e9-1bb6-ee13-d2bfd5894faf from=127.0.0.1:51384 latency=372.328µs -13:02.709 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/dbe37c60-2b34-b99a-b417-01f1e2745363 from=127.0.0.1:51386 latency=548.492µs -13:02.724 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:51388 latency=1.981486ms -13:02.726 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:51390 latency=434.533µs -13:02.731 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/5b491a09-d547-2221-3d38-4a3ba276905e from=127.0.0.1:51390 latency=313.757µs -13:02.732 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/5b491a09-d547-2221-3d38-4a3ba276905e from=127.0.0.1:51390 latency=881.054µs -13:02.732 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/5b491a09-d547-2221-3d38-4a3ba276905e from=127.0.0.1:51388 latency=136.774µs -13:02.734 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:51388 latency=688.404µs -13:02.739 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/5113cffe-a56e-b583-4b20-ecc324e8d5ce from=127.0.0.1:51392 latency=300.666µs -13:02.740 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/5113cffe-a56e-b583-4b20-ecc324e8d5ce from=127.0.0.1:51392 latency=762.82µs -13:02.741 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/5113cffe-a56e-b583-4b20-ecc324e8d5ce from=127.0.0.1:51388 latency=166.029µs -13:02.742 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:51388 latency=765.541µs -13:02.744 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/4257e2a6-b16c-332f-1c1d-a2f16255476a from=127.0.0.1:51394 latency=354.564µs -13:02.750 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:51388 latency=5.089491ms -13:02.752 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/d923c638-084b-8fe9-49dc-8675ceb61244 from=127.0.0.1:51396 latency=328.26µs -13:02.753 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/d923c638-084b-8fe9-49dc-8675ceb61244 from=127.0.0.1:51396 latency=501.103µs -13:02.754 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/d923c638-084b-8fe9-49dc-8675ceb61244 from=127.0.0.1:51388 latency=381.497µs -13:02.754 [INFO] TestAgent: Requesting shutdown -13:02.754 [INFO] TestAgent.server: shutting down server -13:02.754 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:02.754 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:02.754 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:02.754 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:02.754 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:02.754 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:02.754 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:02.754 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:02.754 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:02.754 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:02.754 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:02.754 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:02.754 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:02.754 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:02.754 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:02.754 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:02.754Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:02.754Z [INFO] agent.router.manager: shutting down -13:02.754 [INFO] TestAgent: consul server down -13:02.755 [INFO] TestAgent: shutdown complete -13:02.755 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp -13:02.755 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp -13:02.755 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp -13:03.257 [INFO] TestAgent: Waiting for endpoints to shut down -13:03.257 [INFO] TestAgent: Endpoints down ---- PASS: TestBindingRuleUpdateCommand_noMerge (1.05s) - --- PASS: TestBindingRuleUpdateCommand_noMerge/rule_id_required (0.00s) - --- PASS: TestBindingRuleUpdateCommand_noMerge/rule_id_partial_matches_nothing (0.01s) - --- PASS: TestBindingRuleUpdateCommand_noMerge/rule_id_exact_match_doesn't_exist (0.01s) - --- PASS: TestBindingRuleUpdateCommand_noMerge/rule_id_partial_matches_multiple (0.03s) - --- PASS: TestBindingRuleUpdateCommand_noMerge/must_use_roughly_valid_selector (0.01s) - --- PASS: TestBindingRuleUpdateCommand_noMerge/update_all_fields (0.03s) - --- PASS: TestBindingRuleUpdateCommand_noMerge/update_all_fields_-_partial (0.04s) - --- PASS: TestBindingRuleUpdateCommand_noMerge/update_all_fields_but_description (0.01s) - --- PASS: TestBindingRuleUpdateCommand_noMerge/missing_bind_name (0.00s) - --- PASS: TestBindingRuleUpdateCommand_noMerge/update_all_fields_but_selector (0.01s) -2021-10-31T23:13:02.394Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:02.474Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:02.475 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3e6e751e-3751-1f5b-79a8-9f9ef5e0e01e Address:127.0.0.1:13076}]" -13:02.476 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3e6e751e-3751-1f5b-79a8-9f9ef5e0e01e.dc1 127.0.0.1 -13:02.476 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3e6e751e-3751-1f5b-79a8-9f9ef5e0e01e 127.0.0.1 -2021-10-31T23:13:02.476Z [INFO] agent.router: Initializing LAN area manager -13:02.477 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=udp -13:02.477 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13076 [Follower]" leader= -13:02.477 [INFO] TestAgent.server: Adding LAN server: server="Node-3e6e751e-3751-1f5b-79a8-9f9ef5e0e01e (Addr: tcp/127.0.0.1:13076) (DC: dc1)" -13:02.477 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3e6e751e-3751-1f5b-79a8-9f9ef5e0e01e.dc1 area=wan -13:02.477 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=tcp -13:02.478 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13072 network=tcp -13:02.478 [INFO] TestAgent: started state syncer -13:02.478 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13077 network=tcp -13:02.530 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:02.540 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:02.540 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13076 [Candidate]" term=2 -13:02.540 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:02.540 [DEBUG] TestAgent.server.raft: vote granted: from=3e6e751e-3751-1f5b-79a8-9f9ef5e0e01e term=2 tally=1 -13:02.540 [INFO] TestAgent.server.raft: election won: tally=1 -13:02.540 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13076 [Leader]" -13:02.540 [INFO] TestAgent.server: cluster leadership acquired -13:02.541 [INFO] TestAgent.server: initializing acls -13:02.541 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:02.541 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:02.541 [INFO] TestAgent.server: New leader elected: payload=Node-3e6e751e-3751-1f5b-79a8-9f9ef5e0e01e -13:02.541 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:02.542 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:02.542 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:02.542 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:02.542 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-3e6e751e-3751-1f5b-79a8-9f9ef5e0e01e -13:02.542 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-3e6e751e-3751-1f5b-79a8-9f9ef5e0e01e.dc1 -13:02.543 [INFO] TestAgent.server: Updating LAN server: server="Node-3e6e751e-3751-1f5b-79a8-9f9ef5e0e01e (Addr: tcp/127.0.0.1:13076) (DC: dc1)" -13:02.543 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-3e6e751e-3751-1f5b-79a8-9f9ef5e0e01e.dc1 area=wan -13:02.543 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:02.567 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:02.567 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:02.567 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:02.567 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:02.567 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:02.567 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3e6e751e-3751-1f5b-79a8-9f9ef5e0e01e -13:02.567 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3e6e751e-3751-1f5b-79a8-9f9ef5e0e01e -13:02.568 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3e6e751e-3751-1f5b-79a8-9f9ef5e0e01e -13:02.580 [INFO] TestAgent.server: federation state anti-entropy synced -13:02.702 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:02.702 [INFO] TestAgent: Synced node info -13:02.702 [DEBUG] TestAgent: Node info in sync -13:02.820 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:41898 latency=15.292456ms -13:02.830 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:41900 latency=399.975µs -13:02.837 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/b52213ed-f51d-bfa5-868a-bb3f162a823c from=127.0.0.1:41902 latency=181.582µs -13:02.837 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:41898 latency=244.347µs -13:02.843 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:41898 latency=1.060578ms -13:02.850 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:41898 latency=395.172µs -13:02.851 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:41898 latency=730.675µs -13:02.852 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:41898 latency=175.796µs -13:02.854 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:41904 latency=460.348µs -13:02.857 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:41898 latency=826.832µs -13:02.865 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/349fc044-beba-8395-c38a-85f641947f2f from=127.0.0.1:41906 latency=398.675µs -13:02.866 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule/349fc044-beba-8395-c38a-85f641947f2f from=127.0.0.1:41906 error="invalid Binding Rule: Selector is invalid: 1:4 (3): no match found, expected: "!=", ".", "==", "[", [ \t\r\n] or [a-zA-Z0-9_]" -13:02.866 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/349fc044-beba-8395-c38a-85f641947f2f from=127.0.0.1:41906 latency=770.893µs -13:02.867 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:41898 latency=782.734µs -13:02.874 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/99ff191c-b886-31d4-a09c-171afb1de4dc from=127.0.0.1:41908 latency=410.135µs -13:02.876 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/99ff191c-b886-31d4-a09c-171afb1de4dc from=127.0.0.1:41908 latency=954.463µs -13:02.879 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/99ff191c-b886-31d4-a09c-171afb1de4dc from=127.0.0.1:41898 latency=386.087µs -13:02.879 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:41898 latency=189.2µs -13:02.880 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/08e261ff-d0a9-185a-b56a-d6cb37895f2f from=127.0.0.1:41898 latency=395.906µs -13:02.882 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/0ada6a93-6240-ca36-37ae-e714c320fa50 from=127.0.0.1:41910 latency=867.577µs -13:02.887 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/349fc044-beba-8395-c38a-85f641947f2f from=127.0.0.1:41912 latency=598.285µs -13:02.888 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/99ff191c-b886-31d4-a09c-171afb1de4dc from=127.0.0.1:41914 latency=676.907µs -13:02.899 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:41916 latency=1.072667ms -13:02.901 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:41918 latency=360.107µs -13:02.910 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/22218614-dd2d-08fd-0bd7-7b1bfd6ab324 from=127.0.0.1:41918 latency=346.951µs -13:02.911 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/22218614-dd2d-08fd-0bd7-7b1bfd6ab324 from=127.0.0.1:41918 latency=921.029µs -13:02.912 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/22218614-dd2d-08fd-0bd7-7b1bfd6ab324 from=127.0.0.1:41916 latency=190.092µs -13:02.913 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:41916 latency=689.776µs -13:02.920 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/e7465977-5876-8c54-8444-20e4fdd7c84b from=127.0.0.1:41920 latency=343.771µs -13:02.921 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/e7465977-5876-8c54-8444-20e4fdd7c84b from=127.0.0.1:41920 latency=790.101µs -13:02.921 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/e7465977-5876-8c54-8444-20e4fdd7c84b from=127.0.0.1:41916 latency=150.556µs -13:02.923 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:41916 latency=1.5045ms -13:02.927 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/627eb2c8-aeaa-f185-24bd-0b0060e6f631 from=127.0.0.1:41922 latency=313.276µs -13:02.928 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/627eb2c8-aeaa-f185-24bd-0b0060e6f631 from=127.0.0.1:41922 latency=879.515µs -13:02.931 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/627eb2c8-aeaa-f185-24bd-0b0060e6f631 from=127.0.0.1:41916 latency=274.298µs -13:02.936 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:41916 latency=845.107µs -13:02.938 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/e76c8cb6-57b8-fd98-274b-8d6feee63f9c from=127.0.0.1:41924 latency=469.292µs -13:02.943 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/e76c8cb6-57b8-fd98-274b-8d6feee63f9c from=127.0.0.1:41924 latency=1.098136ms -13:02.943 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/e76c8cb6-57b8-fd98-274b-8d6feee63f9c from=127.0.0.1:41916 latency=164.915µs -13:02.984 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:41916 latency=1.089149ms -13:02.994 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/bb6efe96-54ac-3e7e-0a9c-6f931be19a16 from=127.0.0.1:41926 latency=410.36µs -13:03.003 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/bb6efe96-54ac-3e7e-0a9c-6f931be19a16 from=127.0.0.1:41926 latency=1.088014ms -13:03.006 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/bb6efe96-54ac-3e7e-0a9c-6f931be19a16 from=127.0.0.1:41916 latency=351.313µs -13:03.008 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:41916 latency=783.764µs -13:03.014 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/1e1a7ebb-832d-7ffa-37ce-d326216a4e3c from=127.0.0.1:41928 latency=367.454µs -13:03.019 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/1e1a7ebb-832d-7ffa-37ce-d326216a4e3c from=127.0.0.1:41928 latency=745.839µs -13:03.019 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/1e1a7ebb-832d-7ffa-37ce-d326216a4e3c from=127.0.0.1:41916 latency=177.41µs -13:03.020 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:41916 latency=686.096µs -13:03.039 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/d7beff5e-06a0-ae56-4da0-573a1216f52a from=127.0.0.1:41930 latency=379.745µs -13:03.043 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/d7beff5e-06a0-ae56-4da0-573a1216f52a from=127.0.0.1:41930 latency=1.08956ms -13:03.044 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/d7beff5e-06a0-ae56-4da0-573a1216f52a from=127.0.0.1:41916 latency=174.779µs -13:03.044 [INFO] TestAgent: Requesting shutdown -13:03.044 [INFO] TestAgent.server: shutting down server -13:03.044 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:03.044 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:03.044 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:03.044 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:03.044 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:03.044 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:03.045 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:03.045 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:03.045 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:03.045 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:03.045 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:03.045 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:03.045 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:03.045 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:03.045 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:03.045Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:03.045Z [INFO] agent.router.manager: shutting down -13:03.045 [INFO] TestAgent: consul server down -13:03.045 [INFO] TestAgent: shutdown complete -13:03.045 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=tcp -13:03.045 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=udp -13:03.045 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13072 network=tcp -13:03.549 [INFO] TestAgent: Waiting for endpoints to shut down -13:03.549 [INFO] TestAgent: Endpoints down ---- PASS: TestBindingRuleUpdateCommand (1.34s) +=== RUN TestBindingRuleUpdateCommand_noMerge/update_all_fields_but_selector +2022-12-04T05:48:33.186Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:33.219Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:33.221 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:475ef0fb-46fd-70b9-3ddd-6bfbff0840bb Address:127.0.0.1:15111}]" +48:33.221 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-475ef0fb-46fd-70b9-3ddd-6bfbff0840bb.dc1 127.0.0.1 +48:33.222 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-475ef0fb-46fd-70b9-3ddd-6bfbff0840bb 127.0.0.1 +2022-12-04T05:48:33.222Z [INFO] agent.router: Initializing LAN area manager +48:33.222 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=udp +48:33.222 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15111 [Follower]" leader= +48:33.223 [INFO] TestAgent.server: Adding LAN server: server="Node-475ef0fb-46fd-70b9-3ddd-6bfbff0840bb (Addr: tcp/127.0.0.1:15111) (DC: dc1)" +48:33.223 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-475ef0fb-46fd-70b9-3ddd-6bfbff0840bb.dc1 area=wan +48:33.223 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=tcp +48:33.224 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15107 network=tcp +48:33.224 [INFO] TestAgent: started state syncer +48:33.224 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15112 network=tcp +48:33.276 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:33.276 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:33.276 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15111 [Candidate]" term=2 +48:33.276 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:33.276 [DEBUG] TestAgent.server.raft: vote granted: from=475ef0fb-46fd-70b9-3ddd-6bfbff0840bb term=2 tally=1 +48:33.276 [INFO] TestAgent.server.raft: election won: tally=1 +48:33.276 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15111 [Leader]" +48:33.276 [INFO] TestAgent.server: cluster leadership acquired +48:33.276 [INFO] TestAgent.server: initializing acls +48:33.277 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:33.277 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:33.277 [INFO] TestAgent.server: New leader elected: payload=Node-475ef0fb-46fd-70b9-3ddd-6bfbff0840bb +48:33.277 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:33.278 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:33.278 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:33.278 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:33.278 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-475ef0fb-46fd-70b9-3ddd-6bfbff0840bb +48:33.278 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-475ef0fb-46fd-70b9-3ddd-6bfbff0840bb.dc1 +48:33.279 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:33.280 [INFO] TestAgent.server: Updating LAN server: server="Node-475ef0fb-46fd-70b9-3ddd-6bfbff0840bb (Addr: tcp/127.0.0.1:15111) (DC: dc1)" +48:33.280 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-475ef0fb-46fd-70b9-3ddd-6bfbff0840bb.dc1 area=wan +48:33.337 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:33.337 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:33.337 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:33.337 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:33.337 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:33.337 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-475ef0fb-46fd-70b9-3ddd-6bfbff0840bb +48:33.337 [INFO] TestAgent.server: member joined, marking health alive: member=Node-475ef0fb-46fd-70b9-3ddd-6bfbff0840bb +48:33.338 [INFO] TestAgent.server: federation state anti-entropy synced +48:33.338 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-475ef0fb-46fd-70b9-3ddd-6bfbff0840bb +48:33.356 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:54224 latency=1.179343ms +48:33.362 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:54226 latency=1.244478ms +48:33.366 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/9e38e2a1-f3eb-8194-8f19-a2bbdd485576 from=127.0.0.1:54228 latency=199.498µs +48:33.368 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:54224 latency=343.66µs +48:33.370 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54224 latency=992.952µs +48:33.371 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:54224 latency=240.906µs +48:33.372 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54224 latency=805.084µs +48:33.381 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:54224 latency=5.238206ms +48:33.395 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54224 latency=974.316µs +48:33.398 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:54224 latency=1.275499ms +48:33.399 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54224 latency=747.454µs +48:33.399 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:54224 latency=170.003µs +48:33.405 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54224 latency=5.06814ms +48:33.406 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:54224 latency=232µs +48:33.408 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54224 latency=959.638µs +48:33.409 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:54224 latency=217.285µs +48:33.410 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54224 latency=697.438µs +48:33.412 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:54224 latency=372.336µs +48:33.413 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54224 latency=887.325µs +48:33.414 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:54224 latency=208.278µs +48:33.415 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:54230 latency=316.636µs +48:33.420 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54224 latency=925.09µs +48:33.422 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/a466095b-2401-3a3b-49de-4738e64c9820 from=127.0.0.1:54232 latency=342.472µs +48:33.423 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule/a466095b-2401-3a3b-49de-4738e64c9820 from=127.0.0.1:54232 error="invalid Binding Rule: Selector is invalid: 1:4 (3): no match found, expected: "!=", ".", "==", "[", [ \t\r\n] or [a-zA-Z0-9_]" +48:33.423 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/a466095b-2401-3a3b-49de-4738e64c9820 from=127.0.0.1:54232 latency=613.266µs +48:33.425 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54224 latency=777.563µs +48:33.430 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/cde835d3-c37d-6b55-4128-7e5b06124567 from=127.0.0.1:54234 latency=375.659µs +48:33.432 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/cde835d3-c37d-6b55-4128-7e5b06124567 from=127.0.0.1:54234 latency=886.254µs +48:33.432 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/cde835d3-c37d-6b55-4128-7e5b06124567 from=127.0.0.1:54224 latency=237.273µs +48:33.440 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:54224 latency=403.881µs +48:33.454 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/0598a097-aade-3a24-c80c-ed5d0f9a9766 from=127.0.0.1:54224 latency=744.122µs +48:33.455 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/1fe74e60-8fbe-5300-1fd1-6dd8bbef3ca2 from=127.0.0.1:54236 latency=596.281µs +48:33.456 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/49daab06-280b-29e7-085c-1578d3753a75 from=127.0.0.1:54238 latency=456.231µs +48:33.458 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/63338bce-28ab-0f4f-3601-a4187b9a73e1 from=127.0.0.1:54240 latency=1.346857ms +48:33.459 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/6a186a32-db4b-adc4-4c09-4aead2409331 from=127.0.0.1:54242 latency=489.764µs +48:33.461 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/7d4d0663-f388-a50a-0287-697c490c8ddf from=127.0.0.1:54244 latency=656.782µs +48:33.465 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/a0decaad-a632-c13e-ac39-0fa347b1efae from=127.0.0.1:54246 latency=2.703293ms +48:33.469 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/a466095b-2401-3a3b-49de-4738e64c9820 from=127.0.0.1:54248 latency=633.183µs +48:33.479 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/cde835d3-c37d-6b55-4128-7e5b06124567 from=127.0.0.1:54250 latency=630.692µs +48:33.485 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/dff9dfae-97b0-f62e-e24b-6b498903393d from=127.0.0.1:54252 latency=705.242µs +48:33.486 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54254 latency=808.911µs +48:33.488 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:54256 latency=189.29µs +48:33.488 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/3b9885e6-5799-123d-96aa-9d3190a81e9c from=127.0.0.1:54256 latency=326.258µs +48:33.490 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/3b9885e6-5799-123d-96aa-9d3190a81e9c from=127.0.0.1:54256 latency=1.288719ms +48:33.491 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/3b9885e6-5799-123d-96aa-9d3190a81e9c from=127.0.0.1:54254 latency=218.351µs +48:33.495 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54254 latency=3.30415ms +48:33.506 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/469922c5-69dc-51d3-c97f-02d6829678e0 from=127.0.0.1:54258 latency=8.864014ms +48:33.509 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/469922c5-69dc-51d3-c97f-02d6829678e0 from=127.0.0.1:54258 latency=1.49265ms +48:33.509 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/469922c5-69dc-51d3-c97f-02d6829678e0 from=127.0.0.1:54254 latency=284.25µs +48:33.513 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54254 latency=1.299756ms +48:33.516 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/1cd1c9e1-60b4-fd91-022f-0e3e41e671a8 from=127.0.0.1:54260 latency=263.638µs +48:33.518 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/1cd1c9e1-60b4-fd91-022f-0e3e41e671a8 from=127.0.0.1:54260 latency=1.216829ms +48:33.519 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/1cd1c9e1-60b4-fd91-022f-0e3e41e671a8 from=127.0.0.1:54254 latency=212.919µs +48:33.521 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54254 latency=1.789915ms +48:33.525 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/15c6e4c9-9d4e-a7fd-04ed-d4ef1d244818 from=127.0.0.1:54262 latency=512.693µs +48:33.531 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/15c6e4c9-9d4e-a7fd-04ed-d4ef1d244818 from=127.0.0.1:54262 latency=1.169527ms +48:33.532 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/15c6e4c9-9d4e-a7fd-04ed-d4ef1d244818 from=127.0.0.1:54254 latency=209.443µs +48:33.537 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54254 latency=1.026615ms +48:33.585 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/095cca6c-2dda-7a3d-49a1-ded3bdc92f67 from=127.0.0.1:54264 latency=360.215µs +48:33.586 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/095cca6c-2dda-7a3d-49a1-ded3bdc92f67 from=127.0.0.1:54264 latency=881.982µs +48:33.587 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/095cca6c-2dda-7a3d-49a1-ded3bdc92f67 from=127.0.0.1:54254 latency=205.679µs +48:33.588 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54254 latency=700.27µs +48:33.590 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/890c4196-52c7-da3a-87d4-9a14e98e368f from=127.0.0.1:54266 latency=313.575µs +48:33.593 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/890c4196-52c7-da3a-87d4-9a14e98e368f from=127.0.0.1:54266 latency=613.432µs +48:33.596 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:33.596 [INFO] TestAgent: Synced node info +48:33.597 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/890c4196-52c7-da3a-87d4-9a14e98e368f from=127.0.0.1:54254 latency=302.386µs +48:33.598 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:54254 latency=741.872µs +48:33.600 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/cfcb1ec7-71d6-d65b-73d7-c250b7e656fd from=127.0.0.1:54268 latency=233.39µs +48:33.601 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/cfcb1ec7-71d6-d65b-73d7-c250b7e656fd from=127.0.0.1:54268 latency=892.666µs +48:33.602 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/cfcb1ec7-71d6-d65b-73d7-c250b7e656fd from=127.0.0.1:54254 latency=163.469µs +48:33.602 [INFO] TestAgent: Requesting shutdown +48:33.602 [INFO] TestAgent.server: shutting down server +48:33.602 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:33.602 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:33.602 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:33.602 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:33.602 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:33.602 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:33.602 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:33.602 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:33.602 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:33.602 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:33.602 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:33.602 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:33.602 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:33.602 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:33.602 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:48:33.603Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:33.603Z [INFO] agent.router.manager: shutting down +48:33.603 [INFO] TestAgent: consul server down +48:33.603 [INFO] TestAgent: shutdown complete +48:33.603 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=tcp +48:33.603 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=udp +48:33.603 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15107 network=tcp +48:34.104 [INFO] TestAgent: Waiting for endpoints to shut down +48:34.104 [INFO] TestAgent: Endpoints down +--- PASS: TestBindingRuleUpdateCommand (0.99s) --- PASS: TestBindingRuleUpdateCommand/rule_id_required (0.00s) --- PASS: TestBindingRuleUpdateCommand/rule_id_partial_matches_nothing (0.01s) --- PASS: TestBindingRuleUpdateCommand/rule_id_exact_match_doesn't_exist (0.00s) - --- PASS: TestBindingRuleUpdateCommand/rule_id_partial_matches_multiple (0.02s) - --- PASS: TestBindingRuleUpdateCommand/must_use_roughly_valid_selector (0.01s) + --- PASS: TestBindingRuleUpdateCommand/rule_id_partial_matches_multiple (0.05s) + --- PASS: TestBindingRuleUpdateCommand/must_use_roughly_valid_selector (0.00s) --- PASS: TestBindingRuleUpdateCommand/update_all_fields (0.01s) - --- PASS: TestBindingRuleUpdateCommand/update_all_fields_-_partial (0.03s) - --- PASS: TestBindingRuleUpdateCommand/update_all_fields_but_description (0.01s) + --- PASS: TestBindingRuleUpdateCommand/update_all_fields_-_partial (0.06s) + --- PASS: TestBindingRuleUpdateCommand/update_all_fields_but_description (0.02s) --- PASS: TestBindingRuleUpdateCommand/update_all_fields_but_bind_name (0.01s) - --- PASS: TestBindingRuleUpdateCommand/update_all_fields_but_must_exist (0.02s) + --- PASS: TestBindingRuleUpdateCommand/update_all_fields_but_must_exist (0.01s) --- PASS: TestBindingRuleUpdateCommand/update_all_fields_but_selector (0.05s) --- PASS: TestBindingRuleUpdateCommand/update_all_fields_clear_selector (0.01s) - --- PASS: TestBindingRuleUpdateCommand/update_all_fields_json_formatted (0.02s) + --- PASS: TestBindingRuleUpdateCommand/update_all_fields_json_formatted (0.00s) +2022-12-04T05:48:33.182Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:33.217Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:33.225 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3b66587c-00fe-4a6d-fb57-7121d6602ea6 Address:127.0.0.1:15118}]" +48:33.226 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3b66587c-00fe-4a6d-fb57-7121d6602ea6.dc1 127.0.0.1 +48:33.226 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3b66587c-00fe-4a6d-fb57-7121d6602ea6 127.0.0.1 +2022-12-04T05:48:33.226Z [INFO] agent.router: Initializing LAN area manager +48:33.226 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=udp +48:33.226 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15118 [Follower]" leader= +48:33.227 [INFO] TestAgent.server: Adding LAN server: server="Node-3b66587c-00fe-4a6d-fb57-7121d6602ea6 (Addr: tcp/127.0.0.1:15118) (DC: dc1)" +48:33.227 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3b66587c-00fe-4a6d-fb57-7121d6602ea6.dc1 area=wan +48:33.227 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=tcp +48:33.227 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15114 network=tcp +48:33.227 [INFO] TestAgent: started state syncer +48:33.227 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15119 network=tcp +48:33.277 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:33.292 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:33.292 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15118 [Candidate]" term=2 +48:33.292 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:33.292 [DEBUG] TestAgent.server.raft: vote granted: from=3b66587c-00fe-4a6d-fb57-7121d6602ea6 term=2 tally=1 +48:33.292 [INFO] TestAgent.server.raft: election won: tally=1 +48:33.292 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15118 [Leader]" +48:33.292 [INFO] TestAgent.server: cluster leadership acquired +48:33.292 [INFO] TestAgent.server: initializing acls +48:33.293 [INFO] TestAgent.server: New leader elected: payload=Node-3b66587c-00fe-4a6d-fb57-7121d6602ea6 +48:33.293 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:33.293 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:33.293 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:33.293 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:33.293 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:33.293 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:33.294 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-3b66587c-00fe-4a6d-fb57-7121d6602ea6 +48:33.294 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-3b66587c-00fe-4a6d-fb57-7121d6602ea6.dc1 +48:33.294 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:33.294 [INFO] TestAgent.server: Updating LAN server: server="Node-3b66587c-00fe-4a6d-fb57-7121d6602ea6 (Addr: tcp/127.0.0.1:15118) (DC: dc1)" +48:33.294 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-3b66587c-00fe-4a6d-fb57-7121d6602ea6.dc1 area=wan +48:33.353 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:33.353 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:33.353 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:33.353 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:33.353 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:33.353 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3b66587c-00fe-4a6d-fb57-7121d6602ea6 +48:33.353 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3b66587c-00fe-4a6d-fb57-7121d6602ea6 +48:33.354 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3b66587c-00fe-4a6d-fb57-7121d6602ea6 +48:33.354 [INFO] TestAgent.server: federation state anti-entropy synced +48:33.408 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:50404 latency=1.038358ms +48:33.410 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:50406 latency=237.88µs +48:33.413 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/f58fefa3-ec6b-a1a9-5245-cd966badad6f from=127.0.0.1:50408 latency=167.552µs +48:33.413 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:50404 latency=233.428µs +48:33.415 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50404 latency=836.824µs +48:33.415 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:50404 latency=140.317µs +48:33.417 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50404 latency=1.893102ms +48:33.419 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:50404 latency=259.942µs +48:33.420 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50404 latency=798.168µs +48:33.421 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:50404 latency=190.958µs +48:33.422 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50404 latency=642.647µs +48:33.423 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:50404 latency=378.182µs +48:33.424 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50404 latency=818.776µs +48:33.430 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:50404 latency=375.258µs +48:33.431 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50404 latency=825.621µs +48:33.432 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:50404 latency=227.702µs +48:33.437 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50404 latency=4.375766ms +48:33.451 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:50404 latency=401.252µs +48:33.453 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50404 latency=1.146725ms +48:33.453 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:50404 latency=209.807µs +48:33.455 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50404 latency=1.052019ms +48:33.455 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:50404 latency=243.623µs +48:33.458 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:50410 latency=453.315µs +48:33.462 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50404 latency=2.00701ms +48:33.470 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/ab5bb4c9-9de4-f8d6-b1bc-3fb804ad0ce0 from=127.0.0.1:50412 latency=506.79µs +48:33.478 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/acl/binding-rule/ab5bb4c9-9de4-f8d6-b1bc-3fb804ad0ce0 from=127.0.0.1:50412 error="invalid Binding Rule: Selector is invalid: 1:4 (3): no match found, expected: "!=", ".", "==", "[", [ \t\r\n] or [a-zA-Z0-9_]" +48:33.478 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/ab5bb4c9-9de4-f8d6-b1bc-3fb804ad0ce0 from=127.0.0.1:50412 latency=653.19µs +48:33.480 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50404 latency=786.241µs +48:33.485 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/43312846-cb94-2067-bdc8-6e316e4761de from=127.0.0.1:50414 latency=395.927µs +48:33.486 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/43312846-cb94-2067-bdc8-6e316e4761de from=127.0.0.1:50414 latency=943.14µs +48:33.487 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/43312846-cb94-2067-bdc8-6e316e4761de from=127.0.0.1:50404 latency=159.993µs +48:33.487 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules?authmethod=test from=127.0.0.1:50404 latency=199.37µs +48:33.493 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/10fdd94a-2746-902d-e626-cdd62c0a1e4c from=127.0.0.1:50404 latency=638.616µs +48:33.506 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/21d92557-c265-ebd5-8fd1-310e4412bf1e from=127.0.0.1:50416 latency=9.094559ms +48:33.509 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/43312846-cb94-2067-bdc8-6e316e4761de from=127.0.0.1:50418 latency=1.504389ms +48:33.511 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/4b42b1ae-cec9-9777-bd2d-d3efa020790c from=127.0.0.1:50420 latency=1.111191ms +48:33.513 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/595d0489-e61b-276b-f44b-7c975dfee38d from=127.0.0.1:50422 latency=641.649µs +48:33.514 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/5c25a778-ad6e-1e50-9cc9-363ab2fb1443 from=127.0.0.1:50424 latency=502.997µs +48:33.515 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/79531ed9-40c3-36ce-159d-1c1d30ce5f36 from=127.0.0.1:50428 latency=583.897µs +48:33.517 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/94b92da6-02db-8f22-307e-b646db430d12 from=127.0.0.1:50430 latency=789.927µs +48:33.518 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/ab5bb4c9-9de4-f8d6-b1bc-3fb804ad0ce0 from=127.0.0.1:50432 latency=563.779µs +48:33.521 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/aee7e03c-6601-651f-d1fe-e56f6241a058 from=127.0.0.1:50436 latency=679.301µs +48:33.525 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/binding-rule/c1100b5a-01ce-953f-cd64-d93030004abb from=127.0.0.1:50438 latency=834.975µs +48:33.531 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50440 latency=942.43µs +48:33.577 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rules from=127.0.0.1:50442 latency=351.162µs +48:33.577 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/d4addbb6-8d67-3521-4075-25871f2fa172 from=127.0.0.1:50442 latency=136.12µs +48:33.578 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/d4addbb6-8d67-3521-4075-25871f2fa172 from=127.0.0.1:50442 latency=785.83µs +48:33.579 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/d4addbb6-8d67-3521-4075-25871f2fa172 from=127.0.0.1:50440 latency=149.57µs +48:33.585 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50440 latency=897.317µs +48:33.586 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/f868c2dd-540d-7c93-8f50-f2990ae1c7c1 from=127.0.0.1:50446 latency=203.311µs +48:33.588 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/f868c2dd-540d-7c93-8f50-f2990ae1c7c1 from=127.0.0.1:50446 latency=753.579µs +48:33.588 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/f868c2dd-540d-7c93-8f50-f2990ae1c7c1 from=127.0.0.1:50440 latency=161.873µs +48:33.589 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50440 latency=695.069µs +48:33.597 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/e5c435c3-a624-4992-84b8-099d24988657 from=127.0.0.1:50448 latency=377.623µs +48:33.598 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50440 latency=937.343µs +48:33.608 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/9220e0aa-68fd-b63d-d973-f002f00e5c32 from=127.0.0.1:50450 latency=8.635674ms +48:33.615 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule/9220e0aa-68fd-b63d-d973-f002f00e5c32 from=127.0.0.1:50450 latency=1.428293ms +48:33.615 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/binding-rule/9220e0aa-68fd-b63d-d973-f002f00e5c32 from=127.0.0.1:50440 latency=224.571µs +48:33.616 [INFO] TestAgent: Requesting shutdown +48:33.616 [INFO] TestAgent.server: shutting down server +48:33.616 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:33.616 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:33.616 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:33.616 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:33.616 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:33.616 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:33.616 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:33.616 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:33.616 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:33.616 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:33.616 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:33.616 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +48:33.616 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:33.616 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:33.616 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:33.616 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +2022-12-04T05:48:33.616Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:33.616Z [INFO] agent.router.manager: shutting down +48:33.616 [INFO] TestAgent: consul server down +48:33.616 [INFO] TestAgent: shutdown complete +48:33.616 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=tcp +48:33.616 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=udp +48:33.616 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15114 network=tcp +48:34.120 [INFO] TestAgent: Waiting for endpoints to shut down +48:34.120 [INFO] TestAgent: Endpoints down +--- PASS: TestBindingRuleUpdateCommand_noMerge (1.01s) + --- PASS: TestBindingRuleUpdateCommand_noMerge/rule_id_required (0.00s) + --- PASS: TestBindingRuleUpdateCommand_noMerge/rule_id_partial_matches_nothing (0.00s) + --- PASS: TestBindingRuleUpdateCommand_noMerge/rule_id_exact_match_doesn't_exist (0.00s) + --- PASS: TestBindingRuleUpdateCommand_noMerge/rule_id_partial_matches_multiple (0.05s) + --- PASS: TestBindingRuleUpdateCommand_noMerge/must_use_roughly_valid_selector (0.02s) + --- PASS: TestBindingRuleUpdateCommand_noMerge/update_all_fields (0.01s) + --- PASS: TestBindingRuleUpdateCommand_noMerge/update_all_fields_-_partial (0.09s) + --- PASS: TestBindingRuleUpdateCommand_noMerge/update_all_fields_but_description (0.01s) + --- PASS: TestBindingRuleUpdateCommand_noMerge/missing_bind_name (0.01s) + --- PASS: TestBindingRuleUpdateCommand_noMerge/update_all_fields_but_selector (0.02s) PASS -ok github.com/hashicorp/consul/command/acl/bindingrule/update 1.965s +ok github.com/hashicorp/consul/command/acl/bindingrule/update 1.213s === RUN TestBootstrapCommand_noTabs === PAUSE TestBootstrapCommand_noTabs === RUN TestBootstrapCommand_Pretty @@ -74738,155 +74911,170 @@ === CONT TestBootstrapCommand_Pretty [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:13:14.122Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:14.207Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:14.209 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:377dfa0a-e9f8-2559-6a4d-6e770a60aa45 Address:127.0.0.1:13076}]" -13:14.209 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13076 [Follower]" leader= -13:14.210 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-377dfa0a-e9f8-2559-6a4d-6e770a60aa45.dc1 127.0.0.1 -13:14.210 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-377dfa0a-e9f8-2559-6a4d-6e770a60aa45 127.0.0.1 -2021-10-31T23:13:14.210Z [INFO] agent.router: Initializing LAN area manager -13:14.211 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=udp -13:14.211 [INFO] TestAgent.server: Adding LAN server: server="Node-377dfa0a-e9f8-2559-6a4d-6e770a60aa45 (Addr: tcp/127.0.0.1:13076) (DC: dc1)" -13:14.211 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-377dfa0a-e9f8-2559-6a4d-6e770a60aa45.dc1 area=wan -13:14.211 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=tcp -13:14.212 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13072 network=tcp -13:14.212 [INFO] TestAgent: started state syncer -13:14.212 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13077 network=tcp -13:14.262 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:14.273 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:14.273 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13076 [Candidate]" term=2 -13:14.274 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:14.274 [DEBUG] TestAgent.server.raft: vote granted: from=377dfa0a-e9f8-2559-6a4d-6e770a60aa45 term=2 tally=1 -13:14.274 [INFO] TestAgent.server.raft: election won: tally=1 -13:14.274 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13076 [Leader]" -13:14.274 [INFO] TestAgent.server: cluster leadership acquired -13:14.274 [INFO] TestAgent.server: initializing acls -13:14.274 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:14.275 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:14.275 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:14.275 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:14.275 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-377dfa0a-e9f8-2559-6a4d-6e770a60aa45 -13:14.275 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-377dfa0a-e9f8-2559-6a4d-6e770a60aa45.dc1 -13:14.275 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:14.277 [INFO] TestAgent.server: New leader elected: payload=Node-377dfa0a-e9f8-2559-6a4d-6e770a60aa45 -13:14.278 [INFO] TestAgent.server: Updating LAN server: server="Node-377dfa0a-e9f8-2559-6a4d-6e770a60aa45 (Addr: tcp/127.0.0.1:13076) (DC: dc1)" -13:14.278 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-377dfa0a-e9f8-2559-6a4d-6e770a60aa45.dc1 area=wan -13:14.378 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:14.378 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:14.378 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:14.378 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:14.378 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:14.394 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-377dfa0a-e9f8-2559-6a4d-6e770a60aa45 -13:14.394 [INFO] TestAgent.server: member joined, marking health alive: member=Node-377dfa0a-e9f8-2559-6a4d-6e770a60aa45 -13:14.394 [INFO] TestAgent.server: federation state anti-entropy synced -13:14.395 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-377dfa0a-e9f8-2559-6a4d-6e770a60aa45 -13:14.423 [WARN] TestAgent.server.acl: failed to remove bootstrap file: error="remove /tmp/consul-test/TestBootstrapCommand_Pretty-TestAgent-agent155348742/acl-bootstrap-reset: no such file or directory" -13:14.424 [INFO] TestAgent.server.acl: ACL bootstrap completed -13:14.424 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/bootstrap from=127.0.0.1:42028 latency=1.106004ms -13:14.426 [INFO] TestAgent: Requesting shutdown -13:14.426 [INFO] TestAgent.server: shutting down server -13:14.426 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:14.426 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:14.426 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.426 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.426 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.426 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.426 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.426 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.426 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.426 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:14.426 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:14.426 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:14.426 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.426 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.426 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:14.426 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:14.426Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.426Z [INFO] agent.router.manager: shutting down -13:14.426 [INFO] TestAgent: consul server down -13:14.426 [INFO] TestAgent: shutdown complete -13:14.426 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=tcp -13:14.426 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=udp -13:14.426 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13072 network=tcp -13:14.929 [INFO] TestAgent: Waiting for endpoints to shut down -13:14.929 [INFO] TestAgent: Endpoints down ---- PASS: TestBootstrapCommand_Pretty (0.99s) -2021-10-31T23:13:14.119Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:14.176Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:14.206 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5e4942b3-869f-717f-c75a-07ec4ae80c39 Address:127.0.0.1:13069}]" -13:14.207 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5e4942b3-869f-717f-c75a-07ec4ae80c39.dc1 127.0.0.1 -13:14.207 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5e4942b3-869f-717f-c75a-07ec4ae80c39 127.0.0.1 -2021-10-31T23:13:14.207Z [INFO] agent.router: Initializing LAN area manager -13:14.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp -13:14.208 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= -13:14.208 [INFO] TestAgent.server: Adding LAN server: server="Node-5e4942b3-869f-717f-c75a-07ec4ae80c39 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" -13:14.208 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5e4942b3-869f-717f-c75a-07ec4ae80c39.dc1 area=wan -13:14.208 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp -13:14.209 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp -13:14.209 [INFO] TestAgent: started state syncer -13:14.209 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp -13:14.261 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:14.279 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:14.279 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 -13:14.279 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:14.279 [DEBUG] TestAgent.server.raft: vote granted: from=5e4942b3-869f-717f-c75a-07ec4ae80c39 term=2 tally=1 -13:14.279 [INFO] TestAgent.server.raft: election won: tally=1 -13:14.279 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" -13:14.279 [INFO] TestAgent.server: cluster leadership acquired -13:14.279 [INFO] TestAgent.server: initializing acls -13:14.280 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:14.280 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:14.280 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:14.280 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:14.280 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5e4942b3-869f-717f-c75a-07ec4ae80c39 -13:14.280 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5e4942b3-869f-717f-c75a-07ec4ae80c39.dc1 -13:14.281 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:14.281 [INFO] TestAgent.server: New leader elected: payload=Node-5e4942b3-869f-717f-c75a-07ec4ae80c39 -13:14.281 [INFO] TestAgent.server: Updating LAN server: server="Node-5e4942b3-869f-717f-c75a-07ec4ae80c39 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" -13:14.281 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5e4942b3-869f-717f-c75a-07ec4ae80c39.dc1 area=wan -13:14.330 [INFO] TestAgent: Synced node info -13:14.330 [DEBUG] TestAgent: Node info in sync -13:14.366 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:14.366 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:14.366 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:14.366 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:14.366 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:14.366 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5e4942b3-869f-717f-c75a-07ec4ae80c39 -13:14.367 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5e4942b3-869f-717f-c75a-07ec4ae80c39 -13:14.368 [INFO] TestAgent.server: federation state anti-entropy synced -13:14.368 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5e4942b3-869f-717f-c75a-07ec4ae80c39 -13:14.490 [WARN] TestAgent.server.acl: failed to remove bootstrap file: error="remove /tmp/consul-test/TestBootstrapCommand_JSON-TestAgent-agent249341364/acl-bootstrap-reset: no such file or directory" -13:14.490 [INFO] TestAgent.server.acl: ACL bootstrap completed -13:14.491 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/bootstrap from=127.0.0.1:51530 latency=903.56µs -13:14.494 [INFO] TestAgent: Requesting shutdown -13:14.494 [INFO] TestAgent.server: shutting down server -13:14.494 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:14.494 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:14.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:14.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:14.494 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:14.494 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:14.494 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:14.494 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:14.494 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:14.494 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:14.494 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:14.494 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:14.494 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:14.494 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:14.494 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:14.494Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:14.494Z [INFO] agent.router.manager: shutting down -13:14.495 [INFO] TestAgent: consul server down -13:14.495 [INFO] TestAgent: shutdown complete -13:14.495 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp -13:14.495 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp -13:14.495 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp -13:14.997 [INFO] TestAgent: Waiting for endpoints to shut down -13:14.997 [INFO] TestAgent: Endpoints down ---- PASS: TestBootstrapCommand_JSON (1.05s) +2022-12-04T05:48:40.516Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:40.541Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:40.543 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986 Address:127.0.0.1:28391}]" +48:40.544 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986.dc1 127.0.0.1 +48:40.544 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28391 [Follower]" leader= +48:40.545 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986 127.0.0.1 +2022-12-04T05:48:40.545Z [INFO] agent.router: Initializing LAN area manager +48:40.545 [INFO] TestAgent.server: Adding LAN server: server="Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986 (Addr: tcp/127.0.0.1:28391) (DC: dc1)" +48:40.545 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986.dc1 area=wan +48:40.546 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28386 network=udp +48:40.546 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28386 network=tcp +48:40.547 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28387 network=tcp +48:40.547 [INFO] TestAgent: started state syncer +48:40.547 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28392 network=tcp +48:40.593 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:40.593 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28391 [Candidate]" term=2 +48:40.594 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:40.594 [DEBUG] TestAgent.server.raft: vote granted: from=8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986 term=2 tally=1 +48:40.594 [INFO] TestAgent.server.raft: election won: tally=1 +48:40.594 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28391 [Leader]" +48:40.594 [INFO] TestAgent.server: cluster leadership acquired +48:40.594 [INFO] TestAgent.server: initializing acls +48:40.595 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:40.612 [INFO] TestAgent.server: initializing acls +48:40.616 [INFO] TestAgent.server: New leader elected: payload=Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986 +48:40.617 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:40.617 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:40.617 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:40.617 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986 +48:40.617 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986.dc1 +48:40.617 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986.dc1 area=wan +48:40.618 [INFO] TestAgent.server: Updating LAN server: server="Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986 (Addr: tcp/127.0.0.1:28391) (DC: dc1)" +48:40.620 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:40.632 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +48:40.620 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:40.633 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986 +48:40.633 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986.dc1 +48:40.633 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986.dc1 area=wan +48:40.633 [INFO] TestAgent.server: Updating LAN server: server="Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986 (Addr: tcp/127.0.0.1:28391) (DC: dc1)" +48:40.660 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:40.661 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:40.661 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:40.661 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:40.661 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:40.661 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986 +48:40.661 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986 +48:40.662 [INFO] TestAgent.server: federation state anti-entropy synced +48:40.662 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986 +48:40.662 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8b2fefaa-0a6e-63ff-1a66-eb9edc5f5986 +48:40.679 [WARN] TestAgent.server.acl: failed to remove bootstrap file: error="remove /tmp/consul-test/TestBootstrapCommand_Pretty-TestAgent-agent137861666/acl-bootstrap-reset: no such file or directory" +48:40.679 [INFO] TestAgent.server.acl: ACL bootstrap completed +48:40.680 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/bootstrap from=127.0.0.1:57086 latency=1.122564ms +48:40.681 [INFO] TestAgent: Requesting shutdown +48:40.681 [INFO] TestAgent.server: shutting down server +48:40.681 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:40.681 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:40.681 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:40.681 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:40.681 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:40.681 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:40.681 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:40.681 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:40.681 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:40.681 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:40.681 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:40.681 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:40.681 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:40.681 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:40.681 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:40.681 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:40.681Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:40.681Z [INFO] agent.router.manager: shutting down +48:40.682 [INFO] TestAgent: consul server down +48:40.682 [INFO] TestAgent: shutdown complete +48:40.682 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28386 network=tcp +48:40.682 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28386 network=udp +48:40.682 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28387 network=tcp +48:41.187 [INFO] TestAgent: Waiting for endpoints to shut down +48:41.187 [INFO] TestAgent: Endpoints down +--- PASS: TestBootstrapCommand_Pretty (0.75s) +2022-12-04T05:48:40.510Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:40.537Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:40.540 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:547ab9fa-5d7d-205f-bf9d-25d4f51f89f6 Address:127.0.0.1:28384}]" +48:40.541 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6.dc1 127.0.0.1 +48:40.541 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28384 [Follower]" leader= +48:40.541 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6 127.0.0.1 +2022-12-04T05:48:40.541Z [INFO] agent.router: Initializing LAN area manager +48:40.542 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6.dc1 area=wan +48:40.542 [INFO] TestAgent.server: Adding LAN server: server="Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6 (Addr: tcp/127.0.0.1:28384) (DC: dc1)" +48:40.542 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=udp +48:40.542 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=tcp +48:40.542 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28380 network=tcp +48:40.542 [INFO] TestAgent: started state syncer +48:40.543 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28385 network=tcp +48:40.580 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:40.580 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28384 [Candidate]" term=2 +48:40.580 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:40.580 [DEBUG] TestAgent.server.raft: vote granted: from=547ab9fa-5d7d-205f-bf9d-25d4f51f89f6 term=2 tally=1 +48:40.580 [INFO] TestAgent.server.raft: election won: tally=1 +48:40.580 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28384 [Leader]" +48:40.581 [INFO] TestAgent.server: cluster leadership acquired +48:40.581 [INFO] TestAgent.server: initializing acls +48:40.592 [INFO] TestAgent.server: initializing acls +48:40.593 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:40.594 [INFO] TestAgent.server: New leader elected: payload=Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6 +48:40.595 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:40.595 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:40.595 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:40.595 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:40.595 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +48:40.596 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6 +48:40.604 [INFO] TestAgent.server: Updating LAN server: server="Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6 (Addr: tcp/127.0.0.1:28384) (DC: dc1)" +48:40.625 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6 +48:40.625 [INFO] TestAgent.server: Updating LAN server: server="Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6 (Addr: tcp/127.0.0.1:28384) (DC: dc1)" +48:40.628 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6.dc1 +48:40.628 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6.dc1 +48:40.632 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6.dc1 area=wan +48:40.632 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6.dc1 area=wan +48:40.642 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:40.662 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:40.662 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:40.662 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:40.662 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:40.662 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:40.662 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6 +48:40.662 [INFO] TestAgent.server: member joined, marking health alive: member=Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6 +48:40.663 [INFO] TestAgent.server: federation state anti-entropy synced +48:40.663 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6 +48:40.663 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-547ab9fa-5d7d-205f-bf9d-25d4f51f89f6 +48:40.960 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:40.961 [INFO] TestAgent: Synced node info +48:40.961 [DEBUG] TestAgent: Node info in sync +48:40.968 [WARN] TestAgent.server.acl: failed to remove bootstrap file: error="remove /tmp/consul-test/TestBootstrapCommand_JSON-TestAgent-agent009966960/acl-bootstrap-reset: no such file or directory" +48:40.969 [INFO] TestAgent.server.acl: ACL bootstrap completed +48:40.970 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/bootstrap from=127.0.0.1:54780 latency=1.529066ms +48:40.972 [INFO] TestAgent: Requesting shutdown +48:40.973 [INFO] TestAgent.server: shutting down server +48:40.973 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:40.973 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:40.973 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:40.973 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:40.973 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:40.973 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:40.973 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:40.973 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:40.973 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:40.973 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:40.973 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:40.973 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:40.973 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:40.973 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:40.973 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:40.973Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:40.973Z [INFO] agent.router.manager: shutting down +48:40.973 [INFO] TestAgent: consul server down +48:40.973 [INFO] TestAgent: shutdown complete +48:40.973 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=tcp +48:40.973 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=udp +48:40.973 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28380 network=tcp +48:41.476 [INFO] TestAgent: Waiting for endpoints to shut down +48:41.476 [INFO] TestAgent: Endpoints down +--- PASS: TestBootstrapCommand_JSON (1.04s) PASS -ok github.com/hashicorp/consul/command/acl/bootstrap 1.313s +ok github.com/hashicorp/consul/command/acl/bootstrap 1.346s ? github.com/hashicorp/consul/command/acl/policy [no test files] === RUN TestPolicyCreateCommand_noTabs === PAUSE TestPolicyCreateCommand_noTabs @@ -74895,162 +75083,161 @@ === RUN TestPolicyCreateCommand_JSON === PAUSE TestPolicyCreateCommand_JSON === CONT TestPolicyCreateCommand_noTabs +--- PASS: TestPolicyCreateCommand_noTabs (0.00s) === CONT TestPolicyCreateCommand_JSON [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestPolicyCreateCommand_noTabs (0.00s) === CONT TestPolicyCreateCommand [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:13:23.461Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.491Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:23.500 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:caa3a1c4-48c6-23c3-370e-b941e791f295 Address:127.0.0.1:21237}]" -13:23.501 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21237 [Follower]" leader= -13:23.513 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-caa3a1c4-48c6-23c3-370e-b941e791f295.dc1 127.0.0.1 -13:23.514 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-caa3a1c4-48c6-23c3-370e-b941e791f295 127.0.0.1 -2021-10-31T23:13:23.514Z [INFO] agent.router: Initializing LAN area manager -13:23.514 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=udp -13:23.515 [INFO] TestAgent.server: Adding LAN server: server="Node-caa3a1c4-48c6-23c3-370e-b941e791f295 (Addr: tcp/127.0.0.1:21237) (DC: dc1)" -13:23.515 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-caa3a1c4-48c6-23c3-370e-b941e791f295.dc1 area=wan -13:23.515 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=tcp -13:23.516 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21233 network=tcp -13:23.516 [INFO] TestAgent: started state syncer -13:23.516 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21238 network=tcp -13:23.569 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:23.570 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21237 [Candidate]" term=2 -13:23.570 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:23.570 [DEBUG] TestAgent.server.raft: vote granted: from=caa3a1c4-48c6-23c3-370e-b941e791f295 term=2 tally=1 -13:23.570 [INFO] TestAgent.server.raft: election won: tally=1 -13:23.570 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21237 [Leader]" -13:23.569 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:23.570 [INFO] TestAgent.server: cluster leadership acquired -13:23.570 [INFO] TestAgent.server: initializing acls -13:23.570 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:23.570 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:23.571 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:23.571 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:23.571 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:23.571 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:23.571 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-caa3a1c4-48c6-23c3-370e-b941e791f295 -13:23.571 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-caa3a1c4-48c6-23c3-370e-b941e791f295.dc1 -13:23.572 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:23.573 [INFO] TestAgent.server: New leader elected: payload=Node-caa3a1c4-48c6-23c3-370e-b941e791f295 -13:23.573 [INFO] TestAgent.server: Updating LAN server: server="Node-caa3a1c4-48c6-23c3-370e-b941e791f295 (Addr: tcp/127.0.0.1:21237) (DC: dc1)" -13:23.573 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-caa3a1c4-48c6-23c3-370e-b941e791f295.dc1 area=wan -13:23.629 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:23.629 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:23.629 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:23.629 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:23.629 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:23.629 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-caa3a1c4-48c6-23c3-370e-b941e791f295 -13:23.629 [INFO] TestAgent.server: member joined, marking health alive: member=Node-caa3a1c4-48c6-23c3-370e-b941e791f295 -13:23.630 [INFO] TestAgent.server: federation state anti-entropy synced -13:23.630 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-caa3a1c4-48c6-23c3-370e-b941e791f295 -13:23.634 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:23.634 [INFO] TestAgent: Synced node info -13:23.634 [DEBUG] TestAgent: Node info in sync -13:23.662 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:55112 latency=2.859488ms -13:23.670 [INFO] TestAgent: Requesting shutdown -13:23.670 [INFO] TestAgent.server: shutting down server -13:23.670 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:23.670 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:23.670 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:23.670 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:23.670 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:23.670 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:23.670 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:23.670 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:23.670 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:23.670 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:23.670 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:23.670 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:23.670 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:23.670 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:23.670 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:23.670Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:23.670Z [INFO] agent.router.manager: shutting down -13:23.671 [INFO] TestAgent: consul server down -13:23.671 [INFO] TestAgent: shutdown complete -13:23.671 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=tcp -13:23.671 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=udp -13:23.671 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21233 network=tcp -13:24.171 [INFO] TestAgent: Waiting for endpoints to shut down -13:24.171 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyCreateCommand_JSON (0.84s) -2021-10-31T23:13:23.469Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.523Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:23.527 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a1e0524c-1f1a-6814-cceb-6e23bb3159e8 Address:127.0.0.1:21244}]" -13:23.528 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a1e0524c-1f1a-6814-cceb-6e23bb3159e8.dc1 127.0.0.1 -13:23.528 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a1e0524c-1f1a-6814-cceb-6e23bb3159e8 127.0.0.1 -2021-10-31T23:13:23.529Z [INFO] agent.router: Initializing LAN area manager -13:23.529 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21239 network=udp -13:23.529 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21244 [Follower]" leader= -13:23.529 [INFO] TestAgent.server: Adding LAN server: server="Node-a1e0524c-1f1a-6814-cceb-6e23bb3159e8 (Addr: tcp/127.0.0.1:21244) (DC: dc1)" -13:23.529 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a1e0524c-1f1a-6814-cceb-6e23bb3159e8.dc1 area=wan -13:23.529 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21239 network=tcp -13:23.530 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21240 network=tcp -13:23.530 [INFO] TestAgent: started state syncer -13:23.530 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21245 network=tcp -13:23.582 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:23.582 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21244 [Candidate]" term=2 -13:23.583 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:23.583 [DEBUG] TestAgent.server.raft: vote granted: from=a1e0524c-1f1a-6814-cceb-6e23bb3159e8 term=2 tally=1 -13:23.583 [INFO] TestAgent.server.raft: election won: tally=1 -13:23.583 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21244 [Leader]" -13:23.583 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=2 mode=2 found=true leader=127.0.0.1:21244 -13:23.583 [INFO] TestAgent.server: cluster leadership acquired -13:23.583 [INFO] TestAgent.server: initializing acls -13:23.583 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:23.583 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:23.597 [INFO] TestAgent.server: New leader elected: payload=Node-a1e0524c-1f1a-6814-cceb-6e23bb3159e8 -13:23.602 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:23.603 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:23.603 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:23.603 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:23.604 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-a1e0524c-1f1a-6814-cceb-6e23bb3159e8 -13:23.604 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-a1e0524c-1f1a-6814-cceb-6e23bb3159e8.dc1 -13:23.605 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:23.605 [INFO] TestAgent.server: Updating LAN server: server="Node-a1e0524c-1f1a-6814-cceb-6e23bb3159e8 (Addr: tcp/127.0.0.1:21244) (DC: dc1)" -13:23.605 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-a1e0524c-1f1a-6814-cceb-6e23bb3159e8.dc1 area=wan -13:23.624 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:23.624 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:23.624 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:23.624 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:23.624 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:23.624 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1e0524c-1f1a-6814-cceb-6e23bb3159e8 -13:23.624 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a1e0524c-1f1a-6814-cceb-6e23bb3159e8 -13:23.625 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1e0524c-1f1a-6814-cceb-6e23bb3159e8 -13:23.625 [INFO] TestAgent.server: federation state anti-entropy synced -13:23.920 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:46408 latency=1.218534ms -13:23.921 [INFO] TestAgent: Requesting shutdown -13:23.921 [INFO] TestAgent.server: shutting down server -13:23.921 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:23.921 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:23.921 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:23.921 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:23.921 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:23.921 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:23.921 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:23.921 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:23.921 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:23.921 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:23.921 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:23.921 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:23.921 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:23.921 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:23.921 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:23.921 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:23.921Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:23.921Z [INFO] agent.router.manager: shutting down -13:23.921 [INFO] TestAgent: consul server down -13:23.921 [INFO] TestAgent: shutdown complete -13:23.921 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21239 network=tcp -13:23.921 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21239 network=udp -13:23.929 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21240 network=tcp -13:24.433 [INFO] TestAgent: Waiting for endpoints to shut down -13:24.433 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyCreateCommand (1.10s) +2022-12-04T05:48:42.761Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:42.787Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:42.798 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ac3ff4ff-f2c3-72f6-1641-56e858114081 Address:127.0.0.1:14097}]" +48:42.799 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ac3ff4ff-f2c3-72f6-1641-56e858114081.dc1 127.0.0.1 +48:42.800 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ac3ff4ff-f2c3-72f6-1641-56e858114081 127.0.0.1 +2022-12-04T05:48:42.800Z [INFO] agent.router: Initializing LAN area manager +48:42.801 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=udp +48:42.801 [INFO] TestAgent.server: Adding LAN server: server="Node-ac3ff4ff-f2c3-72f6-1641-56e858114081 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +48:42.801 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=tcp +48:42.816 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14097 [Follower]" leader= +48:42.817 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14093 network=tcp +48:42.817 [INFO] TestAgent: started state syncer +48:42.817 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14098 network=tcp +48:42.817 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ac3ff4ff-f2c3-72f6-1641-56e858114081.dc1 area=wan +48:42.852 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:42.875 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:42.875 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14097 [Candidate]" term=2 +48:42.876 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:42.876 [DEBUG] TestAgent.server.raft: vote granted: from=ac3ff4ff-f2c3-72f6-1641-56e858114081 term=2 tally=1 +48:42.876 [INFO] TestAgent.server.raft: election won: tally=1 +48:42.876 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14097 [Leader]" +48:42.876 [INFO] TestAgent.server: cluster leadership acquired +48:42.876 [INFO] TestAgent.server: initializing acls +48:42.876 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:42.876 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:42.877 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:42.877 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:42.877 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:42.877 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:42.877 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ac3ff4ff-f2c3-72f6-1641-56e858114081 +48:42.877 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ac3ff4ff-f2c3-72f6-1641-56e858114081.dc1 +48:42.878 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:42.876 [INFO] TestAgent.server: New leader elected: payload=Node-ac3ff4ff-f2c3-72f6-1641-56e858114081 +48:42.878 [INFO] TestAgent.server: Updating LAN server: server="Node-ac3ff4ff-f2c3-72f6-1641-56e858114081 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +48:42.878 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ac3ff4ff-f2c3-72f6-1641-56e858114081.dc1 area=wan +48:42.896 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:42.896 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:42.896 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:42.896 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:42.896 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:42.896 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ac3ff4ff-f2c3-72f6-1641-56e858114081 +48:42.896 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ac3ff4ff-f2c3-72f6-1641-56e858114081 +48:42.898 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ac3ff4ff-f2c3-72f6-1641-56e858114081 +48:42.899 [INFO] TestAgent.server: federation state anti-entropy synced +48:43.044 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:43.058 [INFO] TestAgent: Synced node info +48:43.108 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:60988 latency=2.039387ms +48:43.109 [INFO] TestAgent: Requesting shutdown +48:43.109 [INFO] TestAgent.server: shutting down server +48:43.109 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:43.109 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:43.109 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:43.109 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:43.109 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:43.109 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:43.109 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:43.109 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:43.109 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:43.109 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:43.109 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:43.109 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:43.109 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:43.109 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:43.109 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:43.109Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:43.109Z [INFO] agent.router.manager: shutting down +48:43.109 [INFO] TestAgent: consul server down +48:43.109 [INFO] TestAgent: shutdown complete +48:43.109 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=tcp +48:43.109 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=udp +48:43.109 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14093 network=tcp +48:43.612 [INFO] TestAgent: Waiting for endpoints to shut down +48:43.612 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyCreateCommand (0.96s) +2022-12-04T05:48:42.766Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:42.821Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:42.823 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:23f82777-ee43-c16a-4a0b-df76442fbacc Address:127.0.0.1:14090}]" +48:42.823 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= +48:42.824 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-23f82777-ee43-c16a-4a0b-df76442fbacc.dc1 127.0.0.1 +48:42.833 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-23f82777-ee43-c16a-4a0b-df76442fbacc 127.0.0.1 +2022-12-04T05:48:42.833Z [INFO] agent.router: Initializing LAN area manager +48:42.834 [INFO] TestAgent.server: Adding LAN server: server="Node-23f82777-ee43-c16a-4a0b-df76442fbacc (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +48:42.834 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp +48:42.834 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp +48:42.835 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp +48:42.835 [INFO] TestAgent: started state syncer +48:42.835 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp +48:42.835 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-23f82777-ee43-c16a-4a0b-df76442fbacc.dc1 area=wan +48:42.878 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:42.878 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 +48:42.879 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:42.879 [DEBUG] TestAgent.server.raft: vote granted: from=23f82777-ee43-c16a-4a0b-df76442fbacc term=2 tally=1 +48:42.879 [INFO] TestAgent.server.raft: election won: tally=1 +48:42.879 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" +48:42.879 [INFO] TestAgent.server: cluster leadership acquired +48:42.879 [INFO] TestAgent.server: initializing acls +48:42.879 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:42.879 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:42.880 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:42.880 [INFO] TestAgent.server: New leader elected: payload=Node-23f82777-ee43-c16a-4a0b-df76442fbacc +48:42.880 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:42.880 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:42.880 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:42.880 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-23f82777-ee43-c16a-4a0b-df76442fbacc +48:42.880 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-23f82777-ee43-c16a-4a0b-df76442fbacc.dc1 +48:42.881 [INFO] TestAgent.server: Updating LAN server: server="Node-23f82777-ee43-c16a-4a0b-df76442fbacc (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +48:42.881 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:42.881 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-23f82777-ee43-c16a-4a0b-df76442fbacc.dc1 area=wan +48:42.898 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:42.898 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:42.898 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:42.898 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:42.898 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:42.898 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-23f82777-ee43-c16a-4a0b-df76442fbacc +48:42.898 [INFO] TestAgent.server: member joined, marking health alive: member=Node-23f82777-ee43-c16a-4a0b-df76442fbacc +48:42.898 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-23f82777-ee43-c16a-4a0b-df76442fbacc +48:42.899 [INFO] TestAgent.server: federation state anti-entropy synced +48:43.131 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:43.132 [INFO] TestAgent: Synced node info +48:43.155 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:45794 latency=1.106594ms +48:43.155 [INFO] TestAgent: Requesting shutdown +48:43.155 [INFO] TestAgent.server: shutting down server +48:43.155 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:43.155 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:43.155 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:43.155 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:43.155 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:43.155 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:43.155 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:43.155 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:43.155 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:43.155 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:43.155 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:43.155 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:43.155 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:43.155 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:43.156 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:43.156Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:43.156Z [INFO] agent.router.manager: shutting down +48:43.156 [INFO] TestAgent: consul server down +48:43.156 [INFO] TestAgent: shutdown complete +48:43.156 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp +48:43.157 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp +48:43.157 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp +48:43.660 [INFO] TestAgent: Waiting for endpoints to shut down +48:43.660 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyCreateCommand_JSON (1.01s) PASS -ok github.com/hashicorp/consul/command/acl/policy/create 1.891s +ok github.com/hashicorp/consul/command/acl/policy/create 1.170s === RUN TestPolicyDeleteCommand_noTabs === PAUSE TestPolicyDeleteCommand_noTabs === RUN TestPolicyDeleteCommand @@ -75061,84 +75248,84 @@ [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:13:23.765Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:23.826Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:23.829 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0ee527de-8205-b3ea-f7f1-49c7128dd35e Address:127.0.0.1:22258}]" -13:23.829 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0ee527de-8205-b3ea-f7f1-49c7128dd35e.dc1 127.0.0.1 -13:23.830 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22258 [Follower]" leader= -13:23.834 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0ee527de-8205-b3ea-f7f1-49c7128dd35e 127.0.0.1 -2021-10-31T23:13:23.834Z [INFO] agent.router: Initializing LAN area manager -13:23.834 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0ee527de-8205-b3ea-f7f1-49c7128dd35e.dc1 area=wan -13:23.835 [INFO] TestAgent.server: Adding LAN server: server="Node-0ee527de-8205-b3ea-f7f1-49c7128dd35e (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -13:23.835 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=udp -13:23.835 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=tcp -13:23.835 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22254 network=tcp -13:23.835 [INFO] TestAgent: started state syncer -13:23.836 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22259 network=tcp -13:23.885 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:23.893 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:23.894 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22258 [Candidate]" term=2 -13:23.894 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:23.894 [DEBUG] TestAgent.server.raft: vote granted: from=0ee527de-8205-b3ea-f7f1-49c7128dd35e term=2 tally=1 -13:23.894 [INFO] TestAgent.server.raft: election won: tally=1 -13:23.894 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22258 [Leader]" -13:23.894 [INFO] TestAgent.server: cluster leadership acquired -13:23.894 [INFO] TestAgent.server: initializing acls -13:23.895 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:23.895 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:23.897 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:23.897 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:23.897 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:23.897 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:23.897 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0ee527de-8205-b3ea-f7f1-49c7128dd35e -13:23.897 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0ee527de-8205-b3ea-f7f1-49c7128dd35e.dc1 -13:23.910 [INFO] TestAgent.server: New leader elected: payload=Node-0ee527de-8205-b3ea-f7f1-49c7128dd35e -13:23.911 [INFO] TestAgent.server: Updating LAN server: server="Node-0ee527de-8205-b3ea-f7f1-49c7128dd35e (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -13:23.911 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0ee527de-8205-b3ea-f7f1-49c7128dd35e.dc1 area=wan -13:23.912 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:23.943 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:23.943 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:23.943 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:23.943 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:23.943 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:23.944 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0ee527de-8205-b3ea-f7f1-49c7128dd35e -13:23.944 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0ee527de-8205-b3ea-f7f1-49c7128dd35e -13:23.944 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0ee527de-8205-b3ea-f7f1-49c7128dd35e -13:23.944 [INFO] TestAgent.server: federation state anti-entropy synced -13:24.187 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:49490 latency=1.344203ms -13:24.195 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policy/47f4a3ea-d603-edc8-cf66-0a38626bebed from=127.0.0.1:49492 latency=859.549µs -13:24.198 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policy/47f4a3ea-d603-edc8-cf66-0a38626bebed from=127.0.0.1:49490 error="ACL not found" -13:24.198 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/47f4a3ea-d603-edc8-cf66-0a38626bebed from=127.0.0.1:49490 latency=320.827µs -13:24.199 [INFO] TestAgent: Requesting shutdown -13:24.199 [INFO] TestAgent.server: shutting down server -13:24.199 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:24.199 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:24.199 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:24.199 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:24.199 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:24.199 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:24.199 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:24.199 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:24.199 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:24.199 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:24.199 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:24.199 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:24.199 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:24.199 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:24.199 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:24.199 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:24.199Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:24.199Z [INFO] agent.router.manager: shutting down -13:24.199 [INFO] TestAgent: consul server down -13:24.199 [INFO] TestAgent: shutdown complete -13:24.199 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=tcp -13:24.199 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=udp -13:24.199 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22254 network=tcp -13:24.702 [INFO] TestAgent: Waiting for endpoints to shut down -13:24.702 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyDeleteCommand (1.10s) +2022-12-04T05:48:43.480Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:43.502Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:43.504 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:af45dd77-ef88-e43d-8c5d-0fdd33112f9a Address:127.0.0.1:27363}]" +48:43.505 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= +48:43.505 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-af45dd77-ef88-e43d-8c5d-0fdd33112f9a.dc1 127.0.0.1 +48:43.506 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-af45dd77-ef88-e43d-8c5d-0fdd33112f9a 127.0.0.1 +2022-12-04T05:48:43.506Z [INFO] agent.router: Initializing LAN area manager +48:43.506 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp +48:43.506 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-af45dd77-ef88-e43d-8c5d-0fdd33112f9a.dc1 area=wan +48:43.506 [INFO] TestAgent.server: Adding LAN server: server="Node-af45dd77-ef88-e43d-8c5d-0fdd33112f9a (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +48:43.506 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp +48:43.507 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp +48:43.507 [INFO] TestAgent: started state syncer +48:43.507 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp +48:43.544 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:43.544 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 +48:43.544 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:43.544 [DEBUG] TestAgent.server.raft: vote granted: from=af45dd77-ef88-e43d-8c5d-0fdd33112f9a term=2 tally=1 +48:43.544 [INFO] TestAgent.server.raft: election won: tally=1 +48:43.544 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" +48:43.544 [INFO] TestAgent.server: cluster leadership acquired +48:43.544 [INFO] TestAgent.server: initializing acls +48:43.545 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:43.545 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:43.545 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:43.546 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:43.546 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:43.546 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:43.546 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-af45dd77-ef88-e43d-8c5d-0fdd33112f9a +48:43.546 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-af45dd77-ef88-e43d-8c5d-0fdd33112f9a.dc1 +48:43.547 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:43.548 [INFO] TestAgent.server: New leader elected: payload=Node-af45dd77-ef88-e43d-8c5d-0fdd33112f9a +48:43.548 [INFO] TestAgent.server: Updating LAN server: server="Node-af45dd77-ef88-e43d-8c5d-0fdd33112f9a (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +48:43.548 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-af45dd77-ef88-e43d-8c5d-0fdd33112f9a.dc1 area=wan +48:43.565 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:43.565 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:43.565 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:43.565 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:43.565 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:43.565 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-af45dd77-ef88-e43d-8c5d-0fdd33112f9a +48:43.565 [INFO] TestAgent.server: member joined, marking health alive: member=Node-af45dd77-ef88-e43d-8c5d-0fdd33112f9a +48:43.566 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-af45dd77-ef88-e43d-8c5d-0fdd33112f9a +48:43.566 [INFO] TestAgent.server: federation state anti-entropy synced +48:43.724 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:43.725 [INFO] TestAgent: Synced node info +48:43.913 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:59668 latency=1.179972ms +48:43.924 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/policy/b309bda4-c806-254d-04a3-11ae556dea13 from=127.0.0.1:59670 latency=9.78325ms +48:43.929 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/policy/b309bda4-c806-254d-04a3-11ae556dea13 from=127.0.0.1:59668 error="ACL not found" +48:43.929 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/b309bda4-c806-254d-04a3-11ae556dea13 from=127.0.0.1:59668 latency=483.568µs +48:43.930 [INFO] TestAgent: Requesting shutdown +48:43.930 [INFO] TestAgent.server: shutting down server +48:43.930 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:43.930 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:43.930 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:43.930 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:43.930 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:43.930 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:43.930 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:43.930 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:43.930 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:43.930 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:43.930 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:43.930 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:43.930 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:43.930 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:43.930 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:43.930Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:43.930Z [INFO] agent.router.manager: shutting down +48:43.930 [INFO] TestAgent: consul server down +48:43.930 [INFO] TestAgent: shutdown complete +48:43.930 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp +48:43.930 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp +48:43.931 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp +48:44.432 [INFO] TestAgent: Waiting for endpoints to shut down +48:44.432 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyDeleteCommand (1.06s) PASS -ok github.com/hashicorp/consul/command/acl/policy/delete 1.527s +ok github.com/hashicorp/consul/command/acl/policy/delete 1.244s === RUN TestPolicyListCommand_noTabs === PAUSE TestPolicyListCommand_noTabs === RUN TestPolicyListCommand @@ -75146,169 +75333,167 @@ === RUN TestPolicyListCommand_JSON === PAUSE TestPolicyListCommand_JSON === CONT TestPolicyListCommand_noTabs ---- PASS: TestPolicyListCommand_noTabs (0.01s) -=== CONT TestPolicyListCommand === CONT TestPolicyListCommand_JSON +=== CONT TestPolicyListCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestPolicyListCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:13:34.784Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:34.830Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:34.848 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e3b39f71-1db1-a060-9462-9027f2eea279 Address:127.0.0.1:20223}]" -13:34.853 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20223 [Follower]" leader= -13:34.859 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e3b39f71-1db1-a060-9462-9027f2eea279.dc1 127.0.0.1 -13:34.860 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e3b39f71-1db1-a060-9462-9027f2eea279 127.0.0.1 -2021-10-31T23:13:34.860Z [INFO] agent.router: Initializing LAN area manager -13:34.860 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20218 network=udp -13:34.873 [INFO] TestAgent.server: Adding LAN server: server="Node-e3b39f71-1db1-a060-9462-9027f2eea279 (Addr: tcp/127.0.0.1:20223) (DC: dc1)" -13:34.874 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e3b39f71-1db1-a060-9462-9027f2eea279.dc1 area=wan -13:34.874 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20218 network=tcp -13:34.874 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20219 network=tcp -13:34.875 [INFO] TestAgent: started state syncer -13:34.875 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20224 network=tcp -13:34.914 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:34.914 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20223 [Candidate]" term=2 -13:34.914 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:34.914 [DEBUG] TestAgent.server.raft: vote granted: from=e3b39f71-1db1-a060-9462-9027f2eea279 term=2 tally=1 -13:34.914 [INFO] TestAgent.server.raft: election won: tally=1 -13:34.914 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20223 [Leader]" -13:34.914 [INFO] TestAgent.server: cluster leadership acquired -13:34.915 [INFO] TestAgent.server: initializing acls -13:34.915 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:34.915 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:34.916 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:34.916 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:34.916 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:34.916 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:34.916 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e3b39f71-1db1-a060-9462-9027f2eea279 -13:34.916 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e3b39f71-1db1-a060-9462-9027f2eea279.dc1 -13:34.917 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:34.917 [INFO] TestAgent.server: New leader elected: payload=Node-e3b39f71-1db1-a060-9462-9027f2eea279 -13:34.918 [INFO] TestAgent.server: Updating LAN server: server="Node-e3b39f71-1db1-a060-9462-9027f2eea279 (Addr: tcp/127.0.0.1:20223) (DC: dc1)" -13:34.918 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e3b39f71-1db1-a060-9462-9027f2eea279.dc1 area=wan -13:34.958 [INFO] TestAgent: Synced node info -13:35.000 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:35.000 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:35.000 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:35.000 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:35.000 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:35.000 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e3b39f71-1db1-a060-9462-9027f2eea279 -13:35.000 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e3b39f71-1db1-a060-9462-9027f2eea279 -13:35.000 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e3b39f71-1db1-a060-9462-9027f2eea279 -13:35.000 [INFO] TestAgent.server: federation state anti-entropy synced -13:35.229 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:46624 latency=7.123177ms -13:35.243 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:46624 latency=1.597141ms -13:35.245 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:46624 latency=665.47µs -13:35.248 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:46624 latency=2.798824ms -13:35.255 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:46624 latency=771.298µs -13:35.272 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policies from=127.0.0.1:46628 latency=498.847µs -13:35.273 [INFO] TestAgent: Requesting shutdown -13:35.273 [INFO] TestAgent.server: shutting down server -13:35.273 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:35.273 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:35.273 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:35.273 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:35.273 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:35.273 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:35.273 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:35.273 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:35.273 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:35.273 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:35.273 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:35.273 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:35.273 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:35.273 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:35.273 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:35.273Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:35.273Z [INFO] agent.router.manager: shutting down -13:35.278 [INFO] TestAgent: consul server down -13:35.278 [INFO] TestAgent: shutdown complete -13:35.278 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20218 network=tcp -13:35.278 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20218 network=udp -13:35.278 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20219 network=tcp -13:35.781 [INFO] TestAgent: Waiting for endpoints to shut down -13:35.781 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyListCommand_JSON (1.20s) -2021-10-31T23:13:34.791Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:34.871Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:34.873 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5b23ba9d-2442-08ae-59c0-754b9bb84fe8 Address:127.0.0.1:20216}]" -13:34.874 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20216 [Follower]" leader= -13:34.878 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5b23ba9d-2442-08ae-59c0-754b9bb84fe8.dc1 127.0.0.1 -13:34.879 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5b23ba9d-2442-08ae-59c0-754b9bb84fe8 127.0.0.1 -2021-10-31T23:13:34.879Z [INFO] agent.router: Initializing LAN area manager -13:34.879 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=udp -13:34.879 [INFO] TestAgent.server: Adding LAN server: server="Node-5b23ba9d-2442-08ae-59c0-754b9bb84fe8 (Addr: tcp/127.0.0.1:20216) (DC: dc1)" -13:34.879 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5b23ba9d-2442-08ae-59c0-754b9bb84fe8.dc1 area=wan -13:34.880 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=tcp -13:34.880 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20212 network=tcp -13:34.880 [INFO] TestAgent: started state syncer -13:34.880 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20217 network=tcp -13:34.929 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:34.929 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20216 [Candidate]" term=2 -13:34.930 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:34.930 [DEBUG] TestAgent.server.raft: vote granted: from=5b23ba9d-2442-08ae-59c0-754b9bb84fe8 term=2 tally=1 -13:34.930 [INFO] TestAgent.server.raft: election won: tally=1 -13:34.930 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20216 [Leader]" -13:34.930 [INFO] TestAgent.server: cluster leadership acquired -13:34.930 [INFO] TestAgent.server: initializing acls -13:34.930 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:34.930 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:34.930 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:34.931 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:34.931 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:34.931 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:34.931 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5b23ba9d-2442-08ae-59c0-754b9bb84fe8 -13:34.931 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5b23ba9d-2442-08ae-59c0-754b9bb84fe8.dc1 -13:34.931 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:34.931 [INFO] TestAgent.server: New leader elected: payload=Node-5b23ba9d-2442-08ae-59c0-754b9bb84fe8 -13:34.931 [INFO] TestAgent.server: Updating LAN server: server="Node-5b23ba9d-2442-08ae-59c0-754b9bb84fe8 (Addr: tcp/127.0.0.1:20216) (DC: dc1)" -13:34.932 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5b23ba9d-2442-08ae-59c0-754b9bb84fe8.dc1 area=wan -13:34.987 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:34.987 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:34.987 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:34.987 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:34.987 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:34.987 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5b23ba9d-2442-08ae-59c0-754b9bb84fe8 -13:34.987 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5b23ba9d-2442-08ae-59c0-754b9bb84fe8 -13:34.988 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5b23ba9d-2442-08ae-59c0-754b9bb84fe8 -13:34.988 [INFO] TestAgent.server: federation state anti-entropy synced -13:35.178 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:35.178 [INFO] TestAgent: Synced node info -13:35.270 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:53258 latency=16.3687ms -13:35.273 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:53258 latency=561.349µs -13:35.278 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:53258 latency=5.090221ms -13:35.290 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:53258 latency=4.382019ms -13:35.299 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:53258 latency=1.79092ms -13:35.310 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policies from=127.0.0.1:53262 latency=430.947µs -13:35.310 [INFO] TestAgent: Requesting shutdown -13:35.310 [INFO] TestAgent.server: shutting down server -13:35.310 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:35.310 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:35.310 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:35.310 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:35.310 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:35.310 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:35.310 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:35.310 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:35.310 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:35.310 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:35.310 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:35.310 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:35.310 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:35.310 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:35.310 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:35.311Z [INFO] agent.router.manager: shutting down -13:35.311 [INFO] TestAgent: consul server down -13:35.311 [INFO] TestAgent: shutdown complete -2021-10-31T23:13:35.311Z [INFO] agent.router.manager: shutting down -13:35.311 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=tcp -13:35.311 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=udp -13:35.311 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20212 network=tcp -13:35.813 [INFO] TestAgent: Waiting for endpoints to shut down -13:35.813 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyListCommand (1.24s) +2022-12-04T05:48:49.877Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:49.894Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:49.901 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c79bd2c5-ef5d-3176-9695-cd522c734b24 Address:127.0.0.1:14090}]" +48:49.901 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= +48:49.903 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c79bd2c5-ef5d-3176-9695-cd522c734b24.dc1 127.0.0.1 +48:49.903 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c79bd2c5-ef5d-3176-9695-cd522c734b24 127.0.0.1 +2022-12-04T05:48:49.904Z [INFO] agent.router: Initializing LAN area manager +48:49.904 [INFO] TestAgent.server: Adding LAN server: server="Node-c79bd2c5-ef5d-3176-9695-cd522c734b24 (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +48:49.904 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c79bd2c5-ef5d-3176-9695-cd522c734b24.dc1 area=wan +48:49.904 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp +48:49.904 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp +48:49.905 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp +48:49.905 [INFO] TestAgent: started state syncer +48:49.905 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp +48:49.945 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:49.945 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 +48:49.945 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:49.945 [DEBUG] TestAgent.server.raft: vote granted: from=c79bd2c5-ef5d-3176-9695-cd522c734b24 term=2 tally=1 +48:49.945 [INFO] TestAgent.server.raft: election won: tally=1 +48:49.945 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" +48:49.945 [INFO] TestAgent.server: cluster leadership acquired +48:49.945 [INFO] TestAgent.server: initializing acls +48:49.945 [INFO] TestAgent.server: New leader elected: payload=Node-c79bd2c5-ef5d-3176-9695-cd522c734b24 +48:49.946 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:49.946 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:49.946 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:49.947 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:49.947 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:49.947 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:49.947 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c79bd2c5-ef5d-3176-9695-cd522c734b24 +48:49.947 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c79bd2c5-ef5d-3176-9695-cd522c734b24.dc1 +48:49.948 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:49.948 [INFO] TestAgent.server: Updating LAN server: server="Node-c79bd2c5-ef5d-3176-9695-cd522c734b24 (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +48:49.948 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c79bd2c5-ef5d-3176-9695-cd522c734b24.dc1 area=wan +48:49.962 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:49.962 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:49.962 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:49.962 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:49.962 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:49.962 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c79bd2c5-ef5d-3176-9695-cd522c734b24 +48:49.962 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c79bd2c5-ef5d-3176-9695-cd522c734b24 +48:49.963 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c79bd2c5-ef5d-3176-9695-cd522c734b24 +48:49.963 [INFO] TestAgent.server: federation state anti-entropy synced +48:50.045 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:45796 latency=4.865936ms +48:50.049 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:45796 latency=641.626µs +48:50.049 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:45796 latency=325.924µs +48:50.051 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:45796 latency=889.934µs +48:50.052 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:45796 latency=429.613µs +48:50.053 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policies from=127.0.0.1:45798 latency=369.041µs +48:50.053 [INFO] TestAgent: Requesting shutdown +48:50.053 [INFO] TestAgent.server: shutting down server +48:50.053 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:50.053 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:50.053 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:50.053 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:50.053 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:50.053 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:50.053 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:50.053 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:50.053 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:50.053 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:50.053 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:50.053 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:50.053 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:50.053 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:50.054 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:50.054Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:50.054Z [INFO] agent.router.manager: shutting down +48:50.054 [INFO] TestAgent: consul server down +48:50.054 [INFO] TestAgent: shutdown complete +48:50.054 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp +48:50.054 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp +48:50.054 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp +48:50.554 [INFO] TestAgent: Waiting for endpoints to shut down +48:50.554 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyListCommand (0.72s) +2022-12-04T05:48:49.881Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:49.894Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:49.901 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fae0d672-fca4-1748-3c67-61e2b9126eac Address:127.0.0.1:14097}]" +48:49.901 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14097 [Follower]" leader= +48:49.902 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fae0d672-fca4-1748-3c67-61e2b9126eac.dc1 127.0.0.1 +48:49.904 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fae0d672-fca4-1748-3c67-61e2b9126eac 127.0.0.1 +2022-12-04T05:48:49.904Z [INFO] agent.router: Initializing LAN area manager +48:49.904 [INFO] TestAgent.server: Adding LAN server: server="Node-fae0d672-fca4-1748-3c67-61e2b9126eac (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +48:49.904 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=udp +48:49.904 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fae0d672-fca4-1748-3c67-61e2b9126eac.dc1 area=wan +48:49.904 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=tcp +48:49.904 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14093 network=tcp +48:49.905 [INFO] TestAgent: started state syncer +48:49.905 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14098 network=tcp +48:49.944 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:49.944 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14097 [Candidate]" term=2 +48:49.944 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:49.944 [DEBUG] TestAgent.server.raft: vote granted: from=fae0d672-fca4-1748-3c67-61e2b9126eac term=2 tally=1 +48:49.944 [INFO] TestAgent.server.raft: election won: tally=1 +48:49.944 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14097 [Leader]" +48:49.944 [INFO] TestAgent.server: cluster leadership acquired +48:49.944 [INFO] TestAgent.server: initializing acls +48:49.944 [INFO] TestAgent.server: New leader elected: payload=Node-fae0d672-fca4-1748-3c67-61e2b9126eac +48:49.944 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:49.944 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:49.945 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:49.945 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:49.945 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:49.945 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:49.945 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-fae0d672-fca4-1748-3c67-61e2b9126eac +48:49.945 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-fae0d672-fca4-1748-3c67-61e2b9126eac.dc1 +48:49.945 [INFO] TestAgent.server: Updating LAN server: server="Node-fae0d672-fca4-1748-3c67-61e2b9126eac (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +48:49.945 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-fae0d672-fca4-1748-3c67-61e2b9126eac.dc1 area=wan +48:49.946 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:49.962 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:49.962 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:49.962 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:49.962 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:49.962 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:49.962 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fae0d672-fca4-1748-3c67-61e2b9126eac +48:49.962 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fae0d672-fca4-1748-3c67-61e2b9126eac +48:49.963 [INFO] TestAgent.server: federation state anti-entropy synced +48:49.963 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fae0d672-fca4-1748-3c67-61e2b9126eac +48:50.115 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:60990 latency=961.877µs +48:50.116 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:60990 latency=627.657µs +48:50.117 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:60990 latency=365.897µs +48:50.118 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:60990 latency=284.509µs +48:50.118 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:60990 latency=351.323µs +48:50.119 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policies from=127.0.0.1:60992 latency=208.331µs +48:50.120 [INFO] TestAgent: Requesting shutdown +48:50.120 [INFO] TestAgent.server: shutting down server +48:50.120 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:50.120 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:50.120 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:50.120 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:50.120 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:50.120 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:50.120 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:50.120 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:50.120 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:50.120 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:50.120 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:50.120 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:50.120 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:50.120 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:50.120 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:50.120 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:50.120Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:50.120Z [INFO] agent.router.manager: shutting down +48:50.120 [INFO] TestAgent: consul server down +48:50.120 [INFO] TestAgent: shutdown complete +48:50.120 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=tcp +48:50.120 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=udp +48:50.120 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14093 network=tcp +48:50.624 [INFO] TestAgent: Waiting for endpoints to shut down +48:50.624 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyListCommand_JSON (0.79s) PASS -ok github.com/hashicorp/consul/command/acl/policy/list 1.535s +ok github.com/hashicorp/consul/command/acl/policy/list 0.843s === RUN TestPolicyReadCommand_noTabs === PAUSE TestPolicyReadCommand_noTabs === RUN TestPolicyReadCommand @@ -75316,163 +75501,162 @@ === RUN TestPolicyReadCommand_JSON === PAUSE TestPolicyReadCommand_JSON === CONT TestPolicyReadCommand_noTabs ---- PASS: TestPolicyReadCommand_noTabs (0.00s) === CONT TestPolicyReadCommand_JSON -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestPolicyReadCommand +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestPolicyReadCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:13:48.692Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:48.749Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:48.769 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6c557adf-5eb6-6092-f6b5-3d10dea6e20e Address:127.0.0.1:30426}]" -13:48.770 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6c557adf-5eb6-6092-f6b5-3d10dea6e20e.dc1 127.0.0.1 -13:48.770 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= -13:48.770 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6c557adf-5eb6-6092-f6b5-3d10dea6e20e 127.0.0.1 -2021-10-31T23:13:48.770Z [INFO] agent.router: Initializing LAN area manager -13:48.771 [INFO] TestAgent.server: Adding LAN server: server="Node-6c557adf-5eb6-6092-f6b5-3d10dea6e20e (Addr: tcp/127.0.0.1:30426) (DC: dc1)" -13:48.771 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp -13:48.771 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6c557adf-5eb6-6092-f6b5-3d10dea6e20e.dc1 area=wan -13:48.771 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp -13:48.771 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp -13:48.771 [INFO] TestAgent: started state syncer -13:48.771 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp -13:48.821 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:48.833 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:48.833 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 -13:48.834 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:48.834 [DEBUG] TestAgent.server.raft: vote granted: from=6c557adf-5eb6-6092-f6b5-3d10dea6e20e term=2 tally=1 -13:48.834 [INFO] TestAgent.server.raft: election won: tally=1 -13:48.834 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" -13:48.834 [INFO] TestAgent.server: cluster leadership acquired -13:48.834 [INFO] TestAgent.server: initializing acls -13:48.834 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:48.834 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:48.835 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:48.835 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:48.835 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:48.835 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:48.835 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-6c557adf-5eb6-6092-f6b5-3d10dea6e20e -13:48.835 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-6c557adf-5eb6-6092-f6b5-3d10dea6e20e.dc1 -13:48.836 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:48.836 [INFO] TestAgent.server: New leader elected: payload=Node-6c557adf-5eb6-6092-f6b5-3d10dea6e20e -13:48.836 [INFO] TestAgent.server: Updating LAN server: server="Node-6c557adf-5eb6-6092-f6b5-3d10dea6e20e (Addr: tcp/127.0.0.1:30426) (DC: dc1)" -13:48.836 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-6c557adf-5eb6-6092-f6b5-3d10dea6e20e.dc1 area=wan -13:48.923 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:48.923 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:48.923 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:48.923 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:48.923 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:48.923 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6c557adf-5eb6-6092-f6b5-3d10dea6e20e -13:48.923 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6c557adf-5eb6-6092-f6b5-3d10dea6e20e -13:48.924 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6c557adf-5eb6-6092-f6b5-3d10dea6e20e -13:48.924 [INFO] TestAgent.server: federation state anti-entropy synced -13:48.951 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:48.951 [INFO] TestAgent: Synced node info -13:48.995 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:44526 latency=943.973µs -13:49.002 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/265770b4-2d9d-2342-161e-893ea02727e7 from=127.0.0.1:44528 latency=1.445245ms -13:49.006 [INFO] TestAgent: Requesting shutdown -13:49.006 [INFO] TestAgent.server: shutting down server -13:49.006 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:49.006 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:49.006 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:49.006 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:49.006 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:49.006 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:49.006 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:49.006 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:49.006 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:49.006 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:49.006 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:49.006 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:49.006 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:49.006 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:49.006 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:49.006Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:49.006Z [INFO] agent.router.manager: shutting down -13:49.006 [INFO] TestAgent: consul server down -13:49.006 [INFO] TestAgent: shutdown complete -13:49.006 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp -13:49.006 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp -13:49.006 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp -13:49.507 [INFO] TestAgent: Waiting for endpoints to shut down -13:49.507 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyReadCommand_JSON (1.00s) -2021-10-31T23:13:48.668Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:48.751Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:48.767 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5b657913-536a-747f-2c39-9d7c6261d1b0 Address:127.0.0.1:30433}]" -13:48.767 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5b657913-536a-747f-2c39-9d7c6261d1b0.dc1 127.0.0.1 -13:48.768 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5b657913-536a-747f-2c39-9d7c6261d1b0 127.0.0.1 -2021-10-31T23:13:48.768Z [INFO] agent.router: Initializing LAN area manager -13:48.768 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=udp -13:48.770 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30433 [Follower]" leader= -13:48.770 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=tcp -13:48.770 [INFO] TestAgent.server: Adding LAN server: server="Node-5b657913-536a-747f-2c39-9d7c6261d1b0 (Addr: tcp/127.0.0.1:30433) (DC: dc1)" -13:48.770 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5b657913-536a-747f-2c39-9d7c6261d1b0.dc1 area=wan -13:48.771 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30429 network=tcp -13:48.771 [INFO] TestAgent: started state syncer -13:48.771 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30434 network=tcp -13:48.821 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:48.821 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30433 [Candidate]" term=2 -13:48.822 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:48.822 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:48.822 [DEBUG] TestAgent.server.raft: vote granted: from=5b657913-536a-747f-2c39-9d7c6261d1b0 term=2 tally=1 -13:48.822 [INFO] TestAgent.server.raft: election won: tally=1 -13:48.822 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30433 [Leader]" -13:48.822 [INFO] TestAgent.server: cluster leadership acquired -13:48.822 [INFO] TestAgent.server: initializing acls -13:48.822 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:48.822 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:48.823 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:48.823 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:48.823 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:48.823 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:48.823 [INFO] TestAgent.server: New leader elected: payload=Node-5b657913-536a-747f-2c39-9d7c6261d1b0 -13:48.823 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5b657913-536a-747f-2c39-9d7c6261d1b0 -13:48.823 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5b657913-536a-747f-2c39-9d7c6261d1b0.dc1 -13:48.824 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:48.826 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5b657913-536a-747f-2c39-9d7c6261d1b0.dc1 area=wan -13:48.826 [INFO] TestAgent.server: Updating LAN server: server="Node-5b657913-536a-747f-2c39-9d7c6261d1b0 (Addr: tcp/127.0.0.1:30433) (DC: dc1)" -13:48.884 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:48.885 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:48.885 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:48.885 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:48.885 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:48.890 [INFO] TestAgent.server: federation state anti-entropy synced -13:48.894 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5b657913-536a-747f-2c39-9d7c6261d1b0 -13:48.894 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5b657913-536a-747f-2c39-9d7c6261d1b0 -13:48.895 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5b657913-536a-747f-2c39-9d7c6261d1b0 -13:49.026 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:37746 latency=776.625µs -13:49.028 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/7fc4a344-cef4-2ec0-24d6-51869f4c2fa2 from=127.0.0.1:37748 latency=389.198µs -13:49.028 [INFO] TestAgent: Requesting shutdown -13:49.028 [INFO] TestAgent.server: shutting down server -13:49.028 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:49.028 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:49.028 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:49.028 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:49.028 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:49.028 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:49.028 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:49.028 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:49.028 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:49.028 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:49.028 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:49.028 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:49.028 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:49.028 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:49.028 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:49.028 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:49.028Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:49.029Z [INFO] agent.router.manager: shutting down -13:49.029 [INFO] TestAgent: consul server down -13:49.029 [INFO] TestAgent: shutdown complete -13:49.029 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=tcp -13:49.029 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=udp -13:49.029 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30429 network=tcp -13:49.533 [INFO] TestAgent: Waiting for endpoints to shut down -13:49.533 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyReadCommand (1.02s) +2022-12-04T05:48:51.228Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:51.245Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:51.255 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9fc5c0e1-d5e2-a09a-cef4-93e6fd03e6db Address:127.0.0.1:14090}]" +48:51.255 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= +48:51.256 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9fc5c0e1-d5e2-a09a-cef4-93e6fd03e6db.dc1 127.0.0.1 +48:51.257 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9fc5c0e1-d5e2-a09a-cef4-93e6fd03e6db 127.0.0.1 +2022-12-04T05:48:51.257Z [INFO] agent.router: Initializing LAN area manager +48:51.257 [INFO] TestAgent.server: Adding LAN server: server="Node-9fc5c0e1-d5e2-a09a-cef4-93e6fd03e6db (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +48:51.257 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9fc5c0e1-d5e2-a09a-cef4-93e6fd03e6db.dc1 area=wan +48:51.257 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp +48:51.257 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp +48:51.258 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp +48:51.258 [INFO] TestAgent: started state syncer +48:51.258 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp +48:51.307 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:51.313 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:51.313 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 +48:51.313 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:51.313 [DEBUG] TestAgent.server.raft: vote granted: from=9fc5c0e1-d5e2-a09a-cef4-93e6fd03e6db term=2 tally=1 +48:51.313 [INFO] TestAgent.server.raft: election won: tally=1 +48:51.313 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" +48:51.313 [INFO] TestAgent.server: cluster leadership acquired +48:51.313 [INFO] TestAgent.server: initializing acls +48:51.314 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:51.314 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:51.314 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:51.314 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:51.314 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:51.314 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:51.314 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9fc5c0e1-d5e2-a09a-cef4-93e6fd03e6db +48:51.314 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9fc5c0e1-d5e2-a09a-cef4-93e6fd03e6db.dc1 +48:51.315 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:51.315 [INFO] TestAgent.server: New leader elected: payload=Node-9fc5c0e1-d5e2-a09a-cef4-93e6fd03e6db +48:51.315 [INFO] TestAgent.server: Updating LAN server: server="Node-9fc5c0e1-d5e2-a09a-cef4-93e6fd03e6db (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +48:51.315 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9fc5c0e1-d5e2-a09a-cef4-93e6fd03e6db.dc1 area=wan +48:51.323 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:51.323 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:51.323 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:51.323 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:51.323 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:51.323 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9fc5c0e1-d5e2-a09a-cef4-93e6fd03e6db +48:51.323 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9fc5c0e1-d5e2-a09a-cef4-93e6fd03e6db +48:51.325 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9fc5c0e1-d5e2-a09a-cef4-93e6fd03e6db +48:51.325 [INFO] TestAgent.server: federation state anti-entropy synced +48:51.508 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:45800 latency=1.652ms +48:51.511 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/c3fa6a9f-b3bf-b7ce-9d07-285d64b17536 from=127.0.0.1:45802 latency=382.909µs +48:51.511 [INFO] TestAgent: Requesting shutdown +48:51.511 [INFO] TestAgent.server: shutting down server +48:51.511 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:51.511 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:51.511 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:51.511 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:51.511 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:51.511 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:51.511 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:51.511 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:51.511 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:51.511 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:51.511 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:51.511 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:51.511 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:51.511 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:51.511 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:51.511 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:51.511Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:51.511Z [INFO] agent.router.manager: shutting down +48:51.511 [INFO] TestAgent: consul server down +48:51.511 [INFO] TestAgent: shutdown complete +48:51.511 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp +48:51.512 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp +48:51.512 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp +48:52.012 [INFO] TestAgent: Waiting for endpoints to shut down +48:52.012 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyReadCommand_JSON (0.81s) +2022-12-04T05:48:51.234Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:51.255Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:51.257 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:de7c4a6c-ebbe-a9a3-b74a-9a16977e36b5 Address:127.0.0.1:14097}]" +48:51.257 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14097 [Follower]" leader= +48:51.258 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-de7c4a6c-ebbe-a9a3-b74a-9a16977e36b5.dc1 127.0.0.1 +48:51.258 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-de7c4a6c-ebbe-a9a3-b74a-9a16977e36b5 127.0.0.1 +2022-12-04T05:48:51.258Z [INFO] agent.router: Initializing LAN area manager +48:51.258 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-de7c4a6c-ebbe-a9a3-b74a-9a16977e36b5.dc1 area=wan +48:51.258 [INFO] TestAgent.server: Adding LAN server: server="Node-de7c4a6c-ebbe-a9a3-b74a-9a16977e36b5 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +48:51.259 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=udp +48:51.259 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=tcp +48:51.259 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14093 network=tcp +48:51.259 [INFO] TestAgent: started state syncer +48:51.259 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14098 network=tcp +48:51.303 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:51.303 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14097 [Candidate]" term=2 +48:51.304 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:51.304 [DEBUG] TestAgent.server.raft: vote granted: from=de7c4a6c-ebbe-a9a3-b74a-9a16977e36b5 term=2 tally=1 +48:51.304 [INFO] TestAgent.server.raft: election won: tally=1 +48:51.304 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14097 [Leader]" +48:51.304 [INFO] TestAgent.server: cluster leadership acquired +48:51.304 [INFO] TestAgent.server: initializing acls +48:51.304 [INFO] TestAgent.server: New leader elected: payload=Node-de7c4a6c-ebbe-a9a3-b74a-9a16977e36b5 +48:51.304 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:51.304 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:51.305 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:51.305 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:51.305 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:51.305 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:51.305 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-de7c4a6c-ebbe-a9a3-b74a-9a16977e36b5 +48:51.305 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-de7c4a6c-ebbe-a9a3-b74a-9a16977e36b5.dc1 +48:51.305 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:51.306 [INFO] TestAgent.server: Updating LAN server: server="Node-de7c4a6c-ebbe-a9a3-b74a-9a16977e36b5 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +48:51.307 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-de7c4a6c-ebbe-a9a3-b74a-9a16977e36b5.dc1 area=wan +48:51.322 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:51.322 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:51.322 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:51.322 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:51.322 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:51.322 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-de7c4a6c-ebbe-a9a3-b74a-9a16977e36b5 +48:51.322 [INFO] TestAgent.server: member joined, marking health alive: member=Node-de7c4a6c-ebbe-a9a3-b74a-9a16977e36b5 +48:51.324 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-de7c4a6c-ebbe-a9a3-b74a-9a16977e36b5 +48:51.325 [INFO] TestAgent.server: federation state anti-entropy synced +48:51.504 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:51.504 [INFO] TestAgent: Synced node info +48:51.561 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:60994 latency=772.82µs +48:51.564 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/0a6007d3-7de2-c43a-53af-a4548c2c9b4a from=127.0.0.1:60996 latency=398.475µs +48:51.565 [INFO] TestAgent: Requesting shutdown +48:51.565 [INFO] TestAgent.server: shutting down server +48:51.565 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:51.565 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:51.565 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:51.565 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:51.565 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:51.565 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:51.565 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:51.565 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:51.565 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:51.565 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:51.565 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:51.565 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:51.565 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:51.565 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:51.565 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:51.565Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:51.565Z [INFO] agent.router.manager: shutting down +48:51.565 [INFO] TestAgent: consul server down +48:51.565 [INFO] TestAgent: shutdown complete +48:51.565 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=tcp +48:51.565 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=udp +48:51.565 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14093 network=tcp +48:52.066 [INFO] TestAgent: Waiting for endpoints to shut down +48:52.066 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyReadCommand (0.87s) PASS -ok github.com/hashicorp/consul/command/acl/policy/read 1.429s +ok github.com/hashicorp/consul/command/acl/policy/read 0.921s === RUN TestPolicyUpdateCommand_noTabs === PAUSE TestPolicyUpdateCommand_noTabs === RUN TestPolicyUpdateCommand @@ -75480,203 +75664,202 @@ === RUN TestPolicyUpdateCommand_JSON === PAUSE TestPolicyUpdateCommand_JSON === CONT TestPolicyUpdateCommand_noTabs ---- PASS: TestPolicyUpdateCommand_noTabs (0.00s) === CONT TestPolicyUpdateCommand_JSON -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestPolicyUpdateCommand +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestPolicyUpdateCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:13:49.155Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:49.234Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:49.236 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9bb45743-0ab4-17ab-bb29-d2a9d23bd1d5 Address:127.0.0.1:20223}]" -13:49.237 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20223 [Follower]" leader= -13:49.237 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9bb45743-0ab4-17ab-bb29-d2a9d23bd1d5.dc1 127.0.0.1 -13:49.238 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9bb45743-0ab4-17ab-bb29-d2a9d23bd1d5 127.0.0.1 -2021-10-31T23:13:49.238Z [INFO] agent.router: Initializing LAN area manager -13:49.238 [INFO] TestAgent.server: Adding LAN server: server="Node-9bb45743-0ab4-17ab-bb29-d2a9d23bd1d5 (Addr: tcp/127.0.0.1:20223) (DC: dc1)" -13:49.238 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9bb45743-0ab4-17ab-bb29-d2a9d23bd1d5.dc1 area=wan -13:49.241 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20218 network=udp -13:49.241 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20218 network=tcp -13:49.241 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20219 network=tcp -13:49.241 [INFO] TestAgent: started state syncer -13:49.242 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20224 network=tcp -13:49.288 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:49.293 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:49.293 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20223 [Candidate]" term=2 -13:49.294 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:49.294 [DEBUG] TestAgent.server.raft: vote granted: from=9bb45743-0ab4-17ab-bb29-d2a9d23bd1d5 term=2 tally=1 -13:49.294 [INFO] TestAgent.server.raft: election won: tally=1 -13:49.294 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20223 [Leader]" -13:49.294 [INFO] TestAgent.server: cluster leadership acquired -13:49.294 [INFO] TestAgent.server: initializing acls -13:49.294 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:49.294 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:49.295 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:49.295 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:49.295 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:49.295 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:49.295 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9bb45743-0ab4-17ab-bb29-d2a9d23bd1d5 -13:49.295 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9bb45743-0ab4-17ab-bb29-d2a9d23bd1d5.dc1 -13:49.296 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:49.296 [INFO] TestAgent.server: New leader elected: payload=Node-9bb45743-0ab4-17ab-bb29-d2a9d23bd1d5 -13:49.296 [INFO] TestAgent.server: Updating LAN server: server="Node-9bb45743-0ab4-17ab-bb29-d2a9d23bd1d5 (Addr: tcp/127.0.0.1:20223) (DC: dc1)" -13:49.296 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9bb45743-0ab4-17ab-bb29-d2a9d23bd1d5.dc1 area=wan -13:49.375 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:49.375 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:49.375 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:49.375 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:49.375 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:49.375 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9bb45743-0ab4-17ab-bb29-d2a9d23bd1d5 -13:49.375 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9bb45743-0ab4-17ab-bb29-d2a9d23bd1d5 -13:49.376 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9bb45743-0ab4-17ab-bb29-d2a9d23bd1d5 -13:49.377 [INFO] TestAgent.server: federation state anti-entropy synced -13:49.423 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -13:49.424 [INFO] TestAgent: Synced node info -13:49.424 [DEBUG] TestAgent: Node info in sync -13:49.557 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:46714 latency=2.118528ms -13:49.570 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/bbe82dfb-b950-94b8-0840-469717f9d927 from=127.0.0.1:46720 latency=449.101µs -13:49.572 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy/bbe82dfb-b950-94b8-0840-469717f9d927 from=127.0.0.1:46720 latency=701.99µs -13:49.572 [INFO] TestAgent: Requesting shutdown -13:49.572 [INFO] TestAgent.server: shutting down server -13:49.572 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:49.572 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:49.572 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:49.572 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:49.572 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:49.572 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:49.572 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:49.572 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:49.572 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:49.572 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:49.572 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:49.572 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:49.572 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:49.572 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:49.572 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:49.572Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:49.572Z [INFO] agent.router.manager: shutting down -13:49.572 [INFO] TestAgent: consul server down -13:49.572 [INFO] TestAgent: shutdown complete -13:49.572 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20218 network=tcp -13:49.572 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20218 network=udp -13:49.572 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20219 network=tcp -13:50.074 [INFO] TestAgent: Waiting for endpoints to shut down -13:50.074 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyUpdateCommand (1.11s) -2021-10-31T23:13:49.125Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:49.214Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:49.237 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bfc02b09-c2a6-5367-5b13-6ed9ff7dd4cf Address:127.0.0.1:20216}]" -13:49.237 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bfc02b09-c2a6-5367-5b13-6ed9ff7dd4cf.dc1 127.0.0.1 -13:49.238 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20216 [Follower]" leader= -13:49.247 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bfc02b09-c2a6-5367-5b13-6ed9ff7dd4cf 127.0.0.1 -2021-10-31T23:13:49.247Z [INFO] agent.router: Initializing LAN area manager -13:49.248 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=udp -13:49.248 [INFO] TestAgent.server: Adding LAN server: server="Node-bfc02b09-c2a6-5367-5b13-6ed9ff7dd4cf (Addr: tcp/127.0.0.1:20216) (DC: dc1)" -13:49.248 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bfc02b09-c2a6-5367-5b13-6ed9ff7dd4cf.dc1 area=wan -13:49.248 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=tcp -13:49.248 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20212 network=tcp -13:49.248 [INFO] TestAgent: started state syncer -13:49.248 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20217 network=tcp -13:49.284 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:49.284 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20216 [Candidate]" term=2 -13:49.284 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:49.284 [DEBUG] TestAgent.server.raft: vote granted: from=bfc02b09-c2a6-5367-5b13-6ed9ff7dd4cf term=2 tally=1 -13:49.284 [INFO] TestAgent.server.raft: election won: tally=1 -13:49.284 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20216 [Leader]" -13:49.284 [INFO] TestAgent.server: cluster leadership acquired -13:49.284 [INFO] TestAgent.server: initializing acls -13:49.285 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:49.285 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:49.285 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:49.286 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:49.286 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:49.286 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:49.286 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-bfc02b09-c2a6-5367-5b13-6ed9ff7dd4cf -13:49.286 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-bfc02b09-c2a6-5367-5b13-6ed9ff7dd4cf.dc1 -13:49.287 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:49.288 [INFO] TestAgent.server: New leader elected: payload=Node-bfc02b09-c2a6-5367-5b13-6ed9ff7dd4cf -13:49.288 [INFO] TestAgent.server: Updating LAN server: server="Node-bfc02b09-c2a6-5367-5b13-6ed9ff7dd4cf (Addr: tcp/127.0.0.1:20216) (DC: dc1)" -13:49.288 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-bfc02b09-c2a6-5367-5b13-6ed9ff7dd4cf.dc1 area=wan -13:49.326 [INFO] TestAgent: Synced node info -13:49.386 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:49.386 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:49.387 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:49.387 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:49.387 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:49.387 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bfc02b09-c2a6-5367-5b13-6ed9ff7dd4cf -13:49.387 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bfc02b09-c2a6-5367-5b13-6ed9ff7dd4cf -13:49.387 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bfc02b09-c2a6-5367-5b13-6ed9ff7dd4cf -13:49.388 [INFO] TestAgent.server: federation state anti-entropy synced -13:49.555 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:53348 latency=1.369054ms -13:49.619 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/2a7ece95-e083-9a16-3509-dca6baa7948d from=127.0.0.1:53350 latency=4.380905ms -13:49.644 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy/2a7ece95-e083-9a16-3509-dca6baa7948d from=127.0.0.1:53350 latency=2.056892ms -13:49.646 [INFO] TestAgent: Requesting shutdown -13:49.646 [INFO] TestAgent.server: shutting down server -13:49.646 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:49.646 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:49.646 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:49.646 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:49.646 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:49.646 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:49.646 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:49.646 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:49.646 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:49.646 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:49.646 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:49.646 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:49.646 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:49.646 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:49.646 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:49.646Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:49.646Z [INFO] agent.router.manager: shutting down -13:49.646 [INFO] TestAgent: consul server down -13:49.646 [INFO] TestAgent: shutdown complete -13:49.646 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=tcp -13:49.646 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=udp -13:49.646 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20212 network=tcp -13:50.149 [INFO] TestAgent: Waiting for endpoints to shut down -13:50.149 [INFO] TestAgent: Endpoints down ---- PASS: TestPolicyUpdateCommand_JSON (1.19s) +2022-12-04T05:48:51.954Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:51.969Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:51.972 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c27048ab-004e-1750-3ede-cba496909e9f Address:127.0.0.1:15118}]" +48:51.974 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15118 [Follower]" leader= +48:51.975 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c27048ab-004e-1750-3ede-cba496909e9f.dc1 127.0.0.1 +48:51.976 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c27048ab-004e-1750-3ede-cba496909e9f 127.0.0.1 +2022-12-04T05:48:51.977Z [INFO] agent.router: Initializing LAN area manager +48:51.977 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c27048ab-004e-1750-3ede-cba496909e9f.dc1 area=wan +48:51.977 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=udp +48:51.977 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=tcp +48:51.977 [INFO] TestAgent.server: Adding LAN server: server="Node-c27048ab-004e-1750-3ede-cba496909e9f (Addr: tcp/127.0.0.1:15118) (DC: dc1)" +48:51.978 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15114 network=tcp +48:51.978 [INFO] TestAgent: started state syncer +48:51.978 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15119 network=tcp +48:52.027 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:52.032 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:52.032 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15118 [Candidate]" term=2 +48:52.032 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:52.032 [DEBUG] TestAgent.server.raft: vote granted: from=c27048ab-004e-1750-3ede-cba496909e9f term=2 tally=1 +48:52.032 [INFO] TestAgent.server.raft: election won: tally=1 +48:52.032 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15118 [Leader]" +48:52.032 [INFO] TestAgent.server: cluster leadership acquired +48:52.032 [INFO] TestAgent.server: New leader elected: payload=Node-c27048ab-004e-1750-3ede-cba496909e9f +48:52.032 [INFO] TestAgent.server: initializing acls +48:52.032 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:52.032 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:52.033 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:52.033 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:52.033 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:52.033 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:52.033 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c27048ab-004e-1750-3ede-cba496909e9f +48:52.033 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c27048ab-004e-1750-3ede-cba496909e9f.dc1 +48:52.033 [INFO] TestAgent.server: Updating LAN server: server="Node-c27048ab-004e-1750-3ede-cba496909e9f (Addr: tcp/127.0.0.1:15118) (DC: dc1)" +48:52.034 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c27048ab-004e-1750-3ede-cba496909e9f.dc1 area=wan +48:52.034 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:52.035 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:52.035 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:52.035 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:52.035 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:52.035 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:52.035 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c27048ab-004e-1750-3ede-cba496909e9f +48:52.035 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c27048ab-004e-1750-3ede-cba496909e9f +48:52.036 [INFO] TestAgent.server: federation state anti-entropy synced +48:52.036 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c27048ab-004e-1750-3ede-cba496909e9f +48:52.257 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:50452 latency=6.677612ms +48:52.267 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/fec45113-4967-435c-91b1-3995bbfb8c7e from=127.0.0.1:50454 latency=408.087µs +48:52.270 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy/fec45113-4967-435c-91b1-3995bbfb8c7e from=127.0.0.1:50454 latency=969.479µs +48:52.276 [INFO] TestAgent: Requesting shutdown +48:52.276 [INFO] TestAgent.server: shutting down server +48:52.276 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:52.276 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:52.276 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:52.276 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:52.276 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:52.276 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:52.276 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:52.277 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:52.277 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:52.277 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:52.277 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:52.277 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:52.277 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:52.277 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:52.277 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:52.277 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:52.277Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:52.277Z [INFO] agent.router.manager: shutting down +48:52.277 [INFO] TestAgent: consul server down +48:52.277 [INFO] TestAgent: shutdown complete +48:52.277 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=tcp +48:52.277 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=udp +48:52.277 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15114 network=tcp +48:52.777 [INFO] TestAgent: Waiting for endpoints to shut down +48:52.777 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyUpdateCommand (0.86s) +2022-12-04T05:48:51.955Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:51.971Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:51.975 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:66af4adc-8b0d-2261-ddbf-d9d5124d7758 Address:127.0.0.1:15111}]" +48:51.976 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15111 [Follower]" leader= +48:51.976 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-66af4adc-8b0d-2261-ddbf-d9d5124d7758.dc1 127.0.0.1 +48:51.977 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-66af4adc-8b0d-2261-ddbf-d9d5124d7758 127.0.0.1 +2022-12-04T05:48:51.977Z [INFO] agent.router: Initializing LAN area manager +48:51.977 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-66af4adc-8b0d-2261-ddbf-d9d5124d7758.dc1 area=wan +48:51.977 [INFO] TestAgent.server: Adding LAN server: server="Node-66af4adc-8b0d-2261-ddbf-d9d5124d7758 (Addr: tcp/127.0.0.1:15111) (DC: dc1)" +48:51.978 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=udp +48:51.978 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=tcp +48:51.979 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15107 network=tcp +48:51.979 [INFO] TestAgent: started state syncer +48:51.979 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15112 network=tcp +48:52.017 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:52.017 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15111 [Candidate]" term=2 +48:52.017 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:52.017 [DEBUG] TestAgent.server.raft: vote granted: from=66af4adc-8b0d-2261-ddbf-d9d5124d7758 term=2 tally=1 +48:52.017 [INFO] TestAgent.server.raft: election won: tally=1 +48:52.017 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15111 [Leader]" +48:52.017 [INFO] TestAgent.server: cluster leadership acquired +48:52.017 [INFO] TestAgent.server: initializing acls +48:52.017 [INFO] TestAgent.server: New leader elected: payload=Node-66af4adc-8b0d-2261-ddbf-d9d5124d7758 +48:52.018 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:52.018 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:52.018 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:52.018 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:52.018 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:52.018 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:52.018 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-66af4adc-8b0d-2261-ddbf-d9d5124d7758 +48:52.018 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-66af4adc-8b0d-2261-ddbf-d9d5124d7758.dc1 +48:52.019 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:52.024 [INFO] TestAgent.server: Updating LAN server: server="Node-66af4adc-8b0d-2261-ddbf-d9d5124d7758 (Addr: tcp/127.0.0.1:15111) (DC: dc1)" +48:52.024 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-66af4adc-8b0d-2261-ddbf-d9d5124d7758.dc1 area=wan +48:52.042 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:52.042 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:52.042 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:52.042 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:52.042 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:52.042 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-66af4adc-8b0d-2261-ddbf-d9d5124d7758 +48:52.042 [INFO] TestAgent.server: member joined, marking health alive: member=Node-66af4adc-8b0d-2261-ddbf-d9d5124d7758 +48:52.042 [INFO] TestAgent.server: federation state anti-entropy synced +48:52.042 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-66af4adc-8b0d-2261-ddbf-d9d5124d7758 +48:52.236 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:52.237 [INFO] TestAgent: Synced node info +48:52.385 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:54270 latency=2.085023ms +48:52.387 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/policy/9bdbef27-1b62-a0cb-5f61-782675b7588c from=127.0.0.1:54272 latency=348.019µs +48:52.388 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy/9bdbef27-1b62-a0cb-5f61-782675b7588c from=127.0.0.1:54272 latency=664.235µs +48:52.388 [INFO] TestAgent: Requesting shutdown +48:52.388 [INFO] TestAgent.server: shutting down server +48:52.388 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:52.388 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:52.388 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:52.388 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:52.388 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:52.388 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:52.388 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:52.388 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:52.388 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:52.388 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:52.388 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:52.388 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:52.388 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:52.388 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:52.388 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:52.389Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:52.389Z [INFO] agent.router.manager: shutting down +48:52.389 [INFO] TestAgent: consul server down +48:52.389 [INFO] TestAgent: shutdown complete +48:52.389 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=tcp +48:52.389 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=udp +48:52.389 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15107 network=tcp +48:52.889 [INFO] TestAgent: Waiting for endpoints to shut down +48:52.889 [INFO] TestAgent: Endpoints down +--- PASS: TestPolicyUpdateCommand_JSON (0.97s) PASS -ok github.com/hashicorp/consul/command/acl/policy/update 1.394s +ok github.com/hashicorp/consul/command/acl/policy/update 1.016s === RUN TestFormatRole === RUN TestFormatRole/basic -=== RUN TestFormatRole/basic/json === RUN TestFormatRole/basic/pretty === RUN TestFormatRole/basic/pretty-meta +=== RUN TestFormatRole/basic/json === RUN TestFormatRole/complex === RUN TestFormatRole/complex/pretty === RUN TestFormatRole/complex/pretty-meta === RUN TestFormatRole/complex/json ---- PASS: TestFormatRole (0.01s) - --- PASS: TestFormatRole/basic (0.01s) - --- PASS: TestFormatRole/basic/json (0.00s) +--- PASS: TestFormatRole (0.00s) + --- PASS: TestFormatRole/basic (0.00s) --- PASS: TestFormatRole/basic/pretty (0.00s) - --- PASS: TestFormatRole/basic/pretty-meta (0.01s) + --- PASS: TestFormatRole/basic/pretty-meta (0.00s) + --- PASS: TestFormatRole/basic/json (0.00s) --- PASS: TestFormatRole/complex (0.00s) --- PASS: TestFormatRole/complex/pretty (0.00s) --- PASS: TestFormatRole/complex/pretty-meta (0.00s) --- PASS: TestFormatRole/complex/json (0.00s) === RUN TestFormatTokenList -=== RUN TestFormatTokenList/complex -=== RUN TestFormatTokenList/complex/pretty-meta -=== RUN TestFormatTokenList/complex/json -=== RUN TestFormatTokenList/complex/pretty === RUN TestFormatTokenList/basic === RUN TestFormatTokenList/basic/pretty === RUN TestFormatTokenList/basic/pretty-meta === RUN TestFormatTokenList/basic/json +=== RUN TestFormatTokenList/complex +=== RUN TestFormatTokenList/complex/pretty +=== RUN TestFormatTokenList/complex/pretty-meta +=== RUN TestFormatTokenList/complex/json --- PASS: TestFormatTokenList (0.00s) - --- PASS: TestFormatTokenList/complex (0.00s) - --- PASS: TestFormatTokenList/complex/pretty-meta (0.00s) - --- PASS: TestFormatTokenList/complex/json (0.00s) - --- PASS: TestFormatTokenList/complex/pretty (0.00s) --- PASS: TestFormatTokenList/basic (0.00s) --- PASS: TestFormatTokenList/basic/pretty (0.00s) --- PASS: TestFormatTokenList/basic/pretty-meta (0.00s) --- PASS: TestFormatTokenList/basic/json (0.00s) + --- PASS: TestFormatTokenList/complex (0.00s) + --- PASS: TestFormatTokenList/complex/pretty (0.00s) + --- PASS: TestFormatTokenList/complex/pretty-meta (0.00s) + --- PASS: TestFormatTokenList/complex/json (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/role 0.066s +ok github.com/hashicorp/consul/command/acl/role 0.006s === RUN TestRoleCreateCommand_noTabs === PAUSE TestRoleCreateCommand_noTabs === RUN TestRoleCreateCommand_Pretty @@ -75684,10 +75867,10 @@ === RUN TestRoleCreateCommand_JSON === PAUSE TestRoleCreateCommand_JSON === CONT TestRoleCreateCommand_noTabs ---- PASS: TestRoleCreateCommand_noTabs (0.00s) === CONT TestRoleCreateCommand_JSON -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestRoleCreateCommand_Pretty +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestRoleCreateCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestRoleCreateCommand_Pretty/policy-name @@ -75695,164 +75878,167 @@ === RUN TestRoleCreateCommand_Pretty/service-identity === RUN TestRoleCreateCommand_Pretty/dc-scoped-service-identity === RUN TestRoleCreateCommand_Pretty/node-identity -2021-10-31T23:13:58.990Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:59.046Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:59.080 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:27cada6b-ec93-349a-f01f-8db63428bb6b Address:127.0.0.1:13076}]" -13:59.081 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-27cada6b-ec93-349a-f01f-8db63428bb6b.dc1 127.0.0.1 -13:59.081 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-27cada6b-ec93-349a-f01f-8db63428bb6b 127.0.0.1 -2021-10-31T23:13:59.081Z [INFO] agent.router: Initializing LAN area manager -13:59.081 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=udp -13:59.081 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13076 [Follower]" leader= -13:59.082 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=tcp -13:59.082 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-27cada6b-ec93-349a-f01f-8db63428bb6b.dc1 area=wan -13:59.082 [INFO] TestAgent.server: Adding LAN server: server="Node-27cada6b-ec93-349a-f01f-8db63428bb6b (Addr: tcp/127.0.0.1:13076) (DC: dc1)" -13:59.082 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13072 network=tcp -13:59.082 [INFO] TestAgent: started state syncer -13:59.082 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13077 network=tcp -13:59.129 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:59.130 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13076 [Candidate]" term=2 -13:59.130 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:59.130 [DEBUG] TestAgent.server.raft: vote granted: from=27cada6b-ec93-349a-f01f-8db63428bb6b term=2 tally=1 -13:59.130 [INFO] TestAgent.server.raft: election won: tally=1 -13:59.130 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13076 [Leader]" -13:59.130 [INFO] TestAgent.server: cluster leadership acquired -13:59.130 [INFO] TestAgent.server: initializing acls -13:59.131 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:59.131 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:59.131 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:59.132 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:59.132 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:59.132 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:59.132 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-27cada6b-ec93-349a-f01f-8db63428bb6b -13:59.132 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-27cada6b-ec93-349a-f01f-8db63428bb6b.dc1 -13:59.150 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-27cada6b-ec93-349a-f01f-8db63428bb6b.dc1 area=wan -13:59.150 [INFO] TestAgent.server: New leader elected: payload=Node-27cada6b-ec93-349a-f01f-8db63428bb6b -13:59.150 [INFO] TestAgent.server: Updating LAN server: server="Node-27cada6b-ec93-349a-f01f-8db63428bb6b (Addr: tcp/127.0.0.1:13076) (DC: dc1)" -13:59.151 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:59.201 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:59.201 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:59.201 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:59.201 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:59.201 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:59.201 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-27cada6b-ec93-349a-f01f-8db63428bb6b -13:59.201 [INFO] TestAgent.server: member joined, marking health alive: member=Node-27cada6b-ec93-349a-f01f-8db63428bb6b -13:59.207 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-27cada6b-ec93-349a-f01f-8db63428bb6b -13:59.208 [INFO] TestAgent.server: federation state anti-entropy synced -13:59.332 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:42318 latency=1.085446ms -13:59.336 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:42320 latency=2.229496ms -13:59.342 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:42322 latency=757.863µs -13:59.378 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:42324 latency=8.908109ms -13:59.402 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:42326 latency=789µs -13:59.414 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:42328 latency=825.895µs -13:59.418 [INFO] TestAgent: Requesting shutdown -13:59.418 [INFO] TestAgent.server: shutting down server -13:59.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:59.418 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:59.418 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:59.418 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:59.418 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:59.418 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:59.418 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:59.418 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:59.418 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:59.418 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:59.418 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:59.418 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:59.418 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:59.418 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:59.418 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:59.418 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:59.418Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:59.418Z [INFO] agent.router.manager: shutting down -13:59.418 [INFO] TestAgent: consul server down -13:59.418 [INFO] TestAgent: shutdown complete -13:59.418 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=tcp -13:59.418 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=udp -13:59.418 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13072 network=tcp -13:59.921 [INFO] TestAgent: Waiting for endpoints to shut down -13:59.921 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleCreateCommand_Pretty (1.08s) - --- PASS: TestRoleCreateCommand_Pretty/policy-name (0.01s) - --- PASS: TestRoleCreateCommand_Pretty/policy-id (0.01s) - --- PASS: TestRoleCreateCommand_Pretty/service-identity (0.05s) - --- PASS: TestRoleCreateCommand_Pretty/dc-scoped-service-identity (0.01s) +2022-12-04T05:48:57.093Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:57.109Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:57.111 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b93ae0e8-9ff8-43d6-acac-4b5d1996d5d1 Address:127.0.0.1:30426}]" +48:57.111 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= +48:57.111 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b93ae0e8-9ff8-43d6-acac-4b5d1996d5d1.dc1 127.0.0.1 +48:57.112 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b93ae0e8-9ff8-43d6-acac-4b5d1996d5d1 127.0.0.1 +2022-12-04T05:48:57.112Z [INFO] agent.router: Initializing LAN area manager +48:57.112 [INFO] TestAgent.server: Adding LAN server: server="Node-b93ae0e8-9ff8-43d6-acac-4b5d1996d5d1 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +48:57.112 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b93ae0e8-9ff8-43d6-acac-4b5d1996d5d1.dc1 area=wan +48:57.113 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp +48:57.113 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp +48:57.113 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp +48:57.113 [INFO] TestAgent: started state syncer +48:57.113 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp +48:57.162 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:57.167 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:57.167 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 +48:57.167 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:57.167 [DEBUG] TestAgent.server.raft: vote granted: from=b93ae0e8-9ff8-43d6-acac-4b5d1996d5d1 term=2 tally=1 +48:57.167 [INFO] TestAgent.server.raft: election won: tally=1 +48:57.167 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" +48:57.167 [INFO] TestAgent.server: cluster leadership acquired +48:57.167 [INFO] TestAgent.server: New leader elected: payload=Node-b93ae0e8-9ff8-43d6-acac-4b5d1996d5d1 +48:57.167 [INFO] TestAgent.server: initializing acls +48:57.167 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:57.167 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:57.168 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:57.168 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:57.168 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:57.168 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:57.168 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-b93ae0e8-9ff8-43d6-acac-4b5d1996d5d1 +48:57.168 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-b93ae0e8-9ff8-43d6-acac-4b5d1996d5d1.dc1 +48:57.169 [INFO] TestAgent.server: Updating LAN server: server="Node-b93ae0e8-9ff8-43d6-acac-4b5d1996d5d1 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +48:57.169 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-b93ae0e8-9ff8-43d6-acac-4b5d1996d5d1.dc1 area=wan +48:57.169 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:57.186 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:57.186 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:57.186 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:57.186 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:57.186 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:57.186 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b93ae0e8-9ff8-43d6-acac-4b5d1996d5d1 +48:57.186 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b93ae0e8-9ff8-43d6-acac-4b5d1996d5d1 +48:57.187 [INFO] TestAgent.server: federation state anti-entropy synced +48:57.188 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b93ae0e8-9ff8-43d6-acac-4b5d1996d5d1 +48:57.232 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:57.232 [INFO] TestAgent: Synced node info +48:57.364 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:50362 latency=1.651474ms +48:57.366 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:50364 latency=844.151µs +48:57.381 [INFO] TestAgent: Requesting shutdown +48:57.381 [INFO] TestAgent.server: shutting down server +48:57.381 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:57.381 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:57.381 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:57.381 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:57.381 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:57.381 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:57.381 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:57.381 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:57.381 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:57.381 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:57.381 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:57.381 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:57.381 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:57.381 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:57.381 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:57.381Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:57.381Z [INFO] agent.router.manager: shutting down +48:57.382 [INFO] TestAgent: consul server down +48:57.382 [INFO] TestAgent: shutdown complete +48:57.382 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp +48:57.382 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp +48:57.382 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp +48:57.882 [INFO] TestAgent: Waiting for endpoints to shut down +48:57.882 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleCreateCommand_JSON (0.84s) +2022-12-04T05:48:57.097Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:57.113Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:57.115 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f6c0cc27-fa55-6511-a438-8f6b9749a537 Address:127.0.0.1:30433}]" +48:57.115 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30433 [Follower]" leader= +48:57.115 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f6c0cc27-fa55-6511-a438-8f6b9749a537.dc1 127.0.0.1 +48:57.116 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f6c0cc27-fa55-6511-a438-8f6b9749a537 127.0.0.1 +2022-12-04T05:48:57.116Z [INFO] agent.router: Initializing LAN area manager +48:57.116 [INFO] TestAgent.server: Adding LAN server: server="Node-f6c0cc27-fa55-6511-a438-8f6b9749a537 (Addr: tcp/127.0.0.1:30433) (DC: dc1)" +48:57.117 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f6c0cc27-fa55-6511-a438-8f6b9749a537.dc1 area=wan +48:57.117 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=udp +48:57.117 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=tcp +48:57.118 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30429 network=tcp +48:57.118 [INFO] TestAgent: started state syncer +48:57.118 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30434 network=tcp +48:57.166 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:57.175 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:57.175 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30433 [Candidate]" term=2 +48:57.175 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:57.175 [DEBUG] TestAgent.server.raft: vote granted: from=f6c0cc27-fa55-6511-a438-8f6b9749a537 term=2 tally=1 +48:57.175 [INFO] TestAgent.server.raft: election won: tally=1 +48:57.175 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30433 [Leader]" +48:57.175 [INFO] TestAgent.server: cluster leadership acquired +48:57.175 [INFO] TestAgent.server: New leader elected: payload=Node-f6c0cc27-fa55-6511-a438-8f6b9749a537 +48:57.175 [INFO] TestAgent.server: initializing acls +48:57.176 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:57.176 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:57.176 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:57.176 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:57.176 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:57.176 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:57.176 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f6c0cc27-fa55-6511-a438-8f6b9749a537 +48:57.176 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f6c0cc27-fa55-6511-a438-8f6b9749a537.dc1 +48:57.176 [INFO] TestAgent.server: Updating LAN server: server="Node-f6c0cc27-fa55-6511-a438-8f6b9749a537 (Addr: tcp/127.0.0.1:30433) (DC: dc1)" +48:57.176 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f6c0cc27-fa55-6511-a438-8f6b9749a537.dc1 area=wan +48:57.177 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:57.186 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:57.186 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:57.186 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:57.186 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:57.186 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:57.186 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f6c0cc27-fa55-6511-a438-8f6b9749a537 +48:57.186 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f6c0cc27-fa55-6511-a438-8f6b9749a537 +48:57.187 [INFO] TestAgent.server: federation state anti-entropy synced +48:57.187 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f6c0cc27-fa55-6511-a438-8f6b9749a537 +48:57.368 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +48:57.380 [INFO] TestAgent: Synced node info +48:57.496 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:33360 latency=1.022731ms +48:57.499 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:33362 latency=1.293753ms +48:57.501 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:33364 latency=702.411µs +48:57.503 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:33366 latency=561.079µs +48:57.505 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:33370 latency=560.414µs +48:57.511 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:33372 latency=627.522µs +48:57.512 [INFO] TestAgent: Requesting shutdown +48:57.512 [INFO] TestAgent.server: shutting down server +48:57.512 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:57.512 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:57.512 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:57.512 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:57.512 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:57.512 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:57.512 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:57.512 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:57.512 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:57.512 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:57.512 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:57.512 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:57.512 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:57.512 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:57.512 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:57.512Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:57.512Z [INFO] agent.router.manager: shutting down +48:57.512 [INFO] TestAgent: consul server down +48:57.512 [INFO] TestAgent: shutdown complete +48:57.512 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=tcp +48:57.512 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=udp +48:57.512 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30429 network=tcp +48:58.013 [INFO] TestAgent: Waiting for endpoints to shut down +48:58.013 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleCreateCommand_Pretty (0.97s) + --- PASS: TestRoleCreateCommand_Pretty/policy-name (0.00s) + --- PASS: TestRoleCreateCommand_Pretty/policy-id (0.00s) + --- PASS: TestRoleCreateCommand_Pretty/service-identity (0.00s) + --- PASS: TestRoleCreateCommand_Pretty/dc-scoped-service-identity (0.00s) --- PASS: TestRoleCreateCommand_Pretty/node-identity (0.01s) -2021-10-31T23:13:58.999Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:13:59.046Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -13:59.069 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5c1b4361-8508-f15e-1dee-ff5990ed0cf4 Address:127.0.0.1:13069}]" -13:59.070 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5c1b4361-8508-f15e-1dee-ff5990ed0cf4.dc1 127.0.0.1 -13:59.071 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5c1b4361-8508-f15e-1dee-ff5990ed0cf4 127.0.0.1 -2021-10-31T23:13:59.071Z [INFO] agent.router: Initializing LAN area manager -13:59.071 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= -13:59.071 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp -13:59.071 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5c1b4361-8508-f15e-1dee-ff5990ed0cf4.dc1 area=wan -13:59.071 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp -13:59.072 [INFO] TestAgent.server: Adding LAN server: server="Node-5c1b4361-8508-f15e-1dee-ff5990ed0cf4 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" -13:59.072 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp -13:59.072 [INFO] TestAgent: started state syncer -13:59.072 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp -13:59.125 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -13:59.145 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -13:59.145 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 -13:59.146 [DEBUG] TestAgent.server.raft: votes: needed=1 -13:59.146 [DEBUG] TestAgent.server.raft: vote granted: from=5c1b4361-8508-f15e-1dee-ff5990ed0cf4 term=2 tally=1 -13:59.146 [INFO] TestAgent.server.raft: election won: tally=1 -13:59.146 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" -13:59.149 [INFO] TestAgent.server: cluster leadership acquired -13:59.150 [INFO] TestAgent.server: New leader elected: payload=Node-5c1b4361-8508-f15e-1dee-ff5990ed0cf4 -13:59.150 [INFO] TestAgent.server: initializing acls -13:59.151 [INFO] TestAgent.server: Created ACL 'global-management' policy -13:59.151 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -13:59.151 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -13:59.152 [INFO] TestAgent.server: Created ACL anonymous token from configuration -13:59.152 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -13:59.152 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -13:59.152 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5c1b4361-8508-f15e-1dee-ff5990ed0cf4 -13:59.152 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5c1b4361-8508-f15e-1dee-ff5990ed0cf4.dc1 -13:59.153 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -13:59.153 [INFO] TestAgent.server: Updating LAN server: server="Node-5c1b4361-8508-f15e-1dee-ff5990ed0cf4 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" -13:59.153 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5c1b4361-8508-f15e-1dee-ff5990ed0cf4.dc1 area=wan -13:59.206 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -13:59.206 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -13:59.206 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -13:59.206 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -13:59.206 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -13:59.206 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5c1b4361-8508-f15e-1dee-ff5990ed0cf4 -13:59.206 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5c1b4361-8508-f15e-1dee-ff5990ed0cf4 -13:59.207 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5c1b4361-8508-f15e-1dee-ff5990ed0cf4 -13:59.208 [INFO] TestAgent.server: federation state anti-entropy synced -13:59.472 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:51830 latency=2.169298ms -13:59.474 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:51832 latency=826.623µs -13:59.478 [INFO] TestAgent: Requesting shutdown -13:59.478 [INFO] TestAgent.server: shutting down server -13:59.478 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -13:59.478 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -13:59.478 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -13:59.478 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -13:59.478 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -13:59.478 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -13:59.478 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -13:59.478 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -13:59.478 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -13:59.478 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -13:59.478 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -13:59.478 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -13:59.478 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -13:59.478 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -13:59.478 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -13:59.478 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:13:59.478Z [INFO] agent.router.manager: shutting down -2021-10-31T23:13:59.478Z [INFO] agent.router.manager: shutting down -13:59.478 [INFO] TestAgent: consul server down -13:59.478 [INFO] TestAgent: shutdown complete -13:59.478 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp -13:59.478 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp -13:59.478 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp -13:59.981 [INFO] TestAgent: Waiting for endpoints to shut down -13:59.981 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleCreateCommand_JSON (1.14s) PASS -ok github.com/hashicorp/consul/command/acl/role/create 2.052s +ok github.com/hashicorp/consul/command/acl/role/create 1.048s === RUN TestRoleDeleteCommand_noTabs === PAUSE TestRoleDeleteCommand_noTabs === RUN TestRoleDeleteCommand @@ -75866,90 +76052,90 @@ === RUN TestRoleDeleteCommand/id_or_name_required === RUN TestRoleDeleteCommand/delete_works === RUN TestRoleDeleteCommand/delete_works_via_prefixes -2021-10-31T23:14:06.695Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:06.743Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:06.754 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d66b6d53-9127-c53d-092c-3ac37afcac4b Address:127.0.0.1:17153}]" -14:06.755 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d66b6d53-9127-c53d-092c-3ac37afcac4b.dc1 127.0.0.1 -14:06.756 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d66b6d53-9127-c53d-092c-3ac37afcac4b 127.0.0.1 -2021-10-31T23:14:06.756Z [INFO] agent.router: Initializing LAN area manager -14:06.756 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17148 network=udp -14:06.756 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17153 [Follower]" leader= -14:06.756 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d66b6d53-9127-c53d-092c-3ac37afcac4b.dc1 area=wan -14:06.756 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17148 network=tcp -14:06.756 [INFO] TestAgent.server: Adding LAN server: server="Node-d66b6d53-9127-c53d-092c-3ac37afcac4b (Addr: tcp/127.0.0.1:17153) (DC: dc1)" -14:06.757 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17149 network=tcp -14:06.757 [INFO] TestAgent: started state syncer -14:06.757 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17154 network=tcp -14:06.808 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -14:06.813 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:06.814 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17153 [Candidate]" term=2 -14:06.814 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:06.814 [DEBUG] TestAgent.server.raft: vote granted: from=d66b6d53-9127-c53d-092c-3ac37afcac4b term=2 tally=1 -14:06.814 [INFO] TestAgent.server.raft: election won: tally=1 -14:06.814 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17153 [Leader]" -14:06.814 [INFO] TestAgent.server: cluster leadership acquired -14:06.814 [INFO] TestAgent.server: initializing acls -14:06.814 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:06.814 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:06.815 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:06.815 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:06.815 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:06.815 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:06.815 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d66b6d53-9127-c53d-092c-3ac37afcac4b -14:06.816 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d66b6d53-9127-c53d-092c-3ac37afcac4b.dc1 -14:06.816 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:06.818 [INFO] TestAgent.server: New leader elected: payload=Node-d66b6d53-9127-c53d-092c-3ac37afcac4b -14:06.818 [INFO] TestAgent.server: Updating LAN server: server="Node-d66b6d53-9127-c53d-092c-3ac37afcac4b (Addr: tcp/127.0.0.1:17153) (DC: dc1)" -14:06.818 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d66b6d53-9127-c53d-092c-3ac37afcac4b.dc1 area=wan -14:06.867 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:06.867 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:06.867 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:06.867 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:06.867 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:06.867 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d66b6d53-9127-c53d-092c-3ac37afcac4b -14:06.867 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d66b6d53-9127-c53d-092c-3ac37afcac4b -14:06.868 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d66b6d53-9127-c53d-092c-3ac37afcac4b -14:06.868 [INFO] TestAgent.server: federation state anti-entropy synced -14:07.061 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:48578 latency=1.437582ms -14:07.084 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role/98add0f8-d432-b483-1849-115a05f77cf5 from=127.0.0.1:48580 latency=1.510535ms -14:07.090 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/98add0f8-d432-b483-1849-115a05f77cf5 from=127.0.0.1:48578 latency=251.99µs -14:07.098 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:48578 latency=7.557914ms -14:07.103 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:48582 latency=420.47µs -14:07.115 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role/99af7454-46d3-04cf-5998-6a4caecd3e2b from=127.0.0.1:48582 latency=1.667142ms -14:07.118 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/99af7454-46d3-04cf-5998-6a4caecd3e2b from=127.0.0.1:48578 latency=117.799µs -14:07.118 [INFO] TestAgent: Requesting shutdown -14:07.118 [INFO] TestAgent.server: shutting down server -14:07.118 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:07.118 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:07.118 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:07.118 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:07.118 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:07.118 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:07.118 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:07.118 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:07.118 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:07.118 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -14:07.118 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:07.118 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:07.118 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:07.118 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:07.119 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:07.119 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:07.119Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:07.119Z [INFO] agent.router.manager: shutting down -14:07.119 [INFO] TestAgent: consul server down -14:07.119 [INFO] TestAgent: shutdown complete -14:07.119 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17148 network=tcp -14:07.119 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17148 network=udp -14:07.119 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17149 network=tcp -14:07.621 [INFO] TestAgent: Waiting for endpoints to shut down -14:07.621 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleDeleteCommand (1.02s) +2022-12-04T05:48:58.072Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:58.090Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:58.092 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7575f98a-5011-2ff7-84ba-61e8c663c111 Address:127.0.0.1:15111}]" +48:58.092 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15111 [Follower]" leader= +48:58.093 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7575f98a-5011-2ff7-84ba-61e8c663c111.dc1 127.0.0.1 +48:58.093 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7575f98a-5011-2ff7-84ba-61e8c663c111 127.0.0.1 +2022-12-04T05:48:58.094Z [INFO] agent.router: Initializing LAN area manager +48:58.094 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7575f98a-5011-2ff7-84ba-61e8c663c111.dc1 area=wan +48:58.094 [INFO] TestAgent.server: Adding LAN server: server="Node-7575f98a-5011-2ff7-84ba-61e8c663c111 (Addr: tcp/127.0.0.1:15111) (DC: dc1)" +48:58.094 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=tcp +48:58.094 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=udp +48:58.094 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15107 network=tcp +48:58.095 [INFO] TestAgent: started state syncer +48:58.095 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15112 network=tcp +48:58.144 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:58.159 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:58.159 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15111 [Candidate]" term=2 +48:58.159 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:58.159 [DEBUG] TestAgent.server.raft: vote granted: from=7575f98a-5011-2ff7-84ba-61e8c663c111 term=2 tally=1 +48:58.159 [INFO] TestAgent.server.raft: election won: tally=1 +48:58.159 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15111 [Leader]" +48:58.159 [INFO] TestAgent.server: cluster leadership acquired +48:58.159 [INFO] TestAgent.server: initializing acls +48:58.159 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:58.159 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:58.159 [INFO] TestAgent.server: New leader elected: payload=Node-7575f98a-5011-2ff7-84ba-61e8c663c111 +48:58.160 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:58.160 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:58.160 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:58.160 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:58.160 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7575f98a-5011-2ff7-84ba-61e8c663c111 +48:58.160 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7575f98a-5011-2ff7-84ba-61e8c663c111.dc1 +48:58.161 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:58.164 [INFO] TestAgent.server: Updating LAN server: server="Node-7575f98a-5011-2ff7-84ba-61e8c663c111 (Addr: tcp/127.0.0.1:15111) (DC: dc1)" +48:58.164 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7575f98a-5011-2ff7-84ba-61e8c663c111.dc1 area=wan +48:58.177 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:58.177 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:58.177 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:58.177 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:58.177 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:58.177 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7575f98a-5011-2ff7-84ba-61e8c663c111 +48:58.177 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7575f98a-5011-2ff7-84ba-61e8c663c111 +48:58.177 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7575f98a-5011-2ff7-84ba-61e8c663c111 +48:58.178 [INFO] TestAgent.server: federation state anti-entropy synced +48:58.270 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54274 latency=1.368105ms +48:58.273 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role/d9bb494d-0631-c367-9faf-df115ab2ca9f from=127.0.0.1:54276 latency=1.567747ms +48:58.274 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/d9bb494d-0631-c367-9faf-df115ab2ca9f from=127.0.0.1:54274 latency=107.859µs +48:58.274 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54274 latency=456.309µs +48:58.276 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:54278 latency=255.006µs +48:58.276 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/role/a143959e-cb9b-c3d1-2d7a-6a2ed3251e04 from=127.0.0.1:54278 latency=346.077µs +48:58.276 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/a143959e-cb9b-c3d1-2d7a-6a2ed3251e04 from=127.0.0.1:54274 latency=40.729µs +48:58.277 [INFO] TestAgent: Requesting shutdown +48:58.277 [INFO] TestAgent.server: shutting down server +48:58.277 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:58.277 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:58.277 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:58.277 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:58.277 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:58.277 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:58.277 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:58.277 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:58.277 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:58.277 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:58.277 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:58.277 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:58.277 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:58.277 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:58.277 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:58.277 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:58.277Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:58.277Z [INFO] agent.router.manager: shutting down +48:58.277 [INFO] TestAgent: consul server down +48:58.277 [INFO] TestAgent: shutdown complete +48:58.277 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=tcp +48:58.277 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=udp +48:58.277 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15107 network=tcp +48:58.777 [INFO] TestAgent: Waiting for endpoints to shut down +48:58.777 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleDeleteCommand (0.75s) --- PASS: TestRoleDeleteCommand/id_or_name_required (0.00s) - --- PASS: TestRoleDeleteCommand/delete_works (0.03s) - --- PASS: TestRoleDeleteCommand/delete_works_via_prefixes (0.03s) + --- PASS: TestRoleDeleteCommand/delete_works (0.01s) + --- PASS: TestRoleDeleteCommand/delete_works_via_prefixes (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/role/delete 1.187s +ok github.com/hashicorp/consul/command/acl/role/delete 0.808s === RUN TestRoleListCommand_noTabs === PAUSE TestRoleListCommand_noTabs === RUN TestRoleListCommand @@ -75963,165 +76149,163 @@ === CONT TestRoleListCommand [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:14:08.569Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:08.603Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:08.611 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9836938c-8b33-ba73-bf7b-3d4946a29e9c Address:127.0.0.1:14090}]" -14:08.612 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9836938c-8b33-ba73-bf7b-3d4946a29e9c.dc1 127.0.0.1 -14:08.612 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9836938c-8b33-ba73-bf7b-3d4946a29e9c 127.0.0.1 -2021-10-31T23:14:08.612Z [INFO] agent.router: Initializing LAN area manager -14:08.613 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp -14:08.613 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= -14:08.613 [INFO] TestAgent.server: Adding LAN server: server="Node-9836938c-8b33-ba73-bf7b-3d4946a29e9c (Addr: tcp/127.0.0.1:14090) (DC: dc1)" -14:08.613 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9836938c-8b33-ba73-bf7b-3d4946a29e9c.dc1 area=wan -14:08.613 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp -14:08.614 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp -14:08.614 [INFO] TestAgent: started state syncer -14:08.614 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp -14:08.663 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -14:08.669 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:08.669 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 -14:08.670 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:08.670 [DEBUG] TestAgent.server.raft: vote granted: from=9836938c-8b33-ba73-bf7b-3d4946a29e9c term=2 tally=1 -14:08.670 [INFO] TestAgent.server.raft: election won: tally=1 -14:08.670 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" -14:08.670 [INFO] TestAgent.server: cluster leadership acquired -14:08.670 [INFO] TestAgent.server: initializing acls -14:08.670 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:08.670 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:08.670 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:08.671 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:08.671 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:08.671 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:08.671 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9836938c-8b33-ba73-bf7b-3d4946a29e9c -14:08.671 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9836938c-8b33-ba73-bf7b-3d4946a29e9c.dc1 -14:08.671 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:08.671 [INFO] TestAgent.server: New leader elected: payload=Node-9836938c-8b33-ba73-bf7b-3d4946a29e9c -14:08.671 [INFO] TestAgent.server: Updating LAN server: server="Node-9836938c-8b33-ba73-bf7b-3d4946a29e9c (Addr: tcp/127.0.0.1:14090) (DC: dc1)" -14:08.671 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9836938c-8b33-ba73-bf7b-3d4946a29e9c.dc1 area=wan -14:08.683 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:08.683 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:08.683 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:08.683 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:08.683 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:08.683 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9836938c-8b33-ba73-bf7b-3d4946a29e9c -14:08.683 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9836938c-8b33-ba73-bf7b-3d4946a29e9c -14:08.683 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9836938c-8b33-ba73-bf7b-3d4946a29e9c -14:08.684 [INFO] TestAgent.server: federation state anti-entropy synced -14:08.737 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:49382 latency=1.778213ms -14:08.739 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:49382 latency=772.531µs -14:08.759 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:49382 latency=9.111531ms -14:08.767 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:49382 latency=779.359µs -14:08.775 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:49382 latency=720.362µs -14:08.777 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:49384 latency=311.723µs -14:08.777 [INFO] TestAgent: Requesting shutdown -14:08.778 [INFO] TestAgent.server: shutting down server -14:08.778 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:08.778 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:08.778 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:08.778 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:08.778 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:08.778 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:08.778 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:08.778 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:08.778 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:08.778 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -14:08.778 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:08.778 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:08.778 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:08.778 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:08.778 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:08.778 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:08.778Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:08.778Z [INFO] agent.router.manager: shutting down -14:08.778 [INFO] TestAgent: consul server down -14:08.778 [INFO] TestAgent: shutdown complete -14:08.778 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp -14:08.778 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp -14:08.778 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp -14:09.281 [INFO] TestAgent: Waiting for endpoints to shut down -14:09.281 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleListCommand_JSON (0.78s) -2021-10-31T23:14:08.600Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:08.619Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:08.621 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9e6327ef-551d-c823-51ae-c495095adda1 Address:127.0.0.1:14097}]" -14:08.621 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14097 [Follower]" leader= -14:08.622 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9e6327ef-551d-c823-51ae-c495095adda1.dc1 127.0.0.1 -14:08.623 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9e6327ef-551d-c823-51ae-c495095adda1 127.0.0.1 -2021-10-31T23:14:08.623Z [INFO] agent.router: Initializing LAN area manager -14:08.628 [INFO] TestAgent.server: Adding LAN server: server="Node-9e6327ef-551d-c823-51ae-c495095adda1 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" -14:08.628 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9e6327ef-551d-c823-51ae-c495095adda1.dc1 area=wan -14:08.628 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=tcp -14:08.634 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=udp -14:08.635 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14093 network=tcp -14:08.635 [INFO] TestAgent: started state syncer -14:08.635 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14098 network=tcp -14:08.661 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:08.661 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14097 [Candidate]" term=2 -14:08.661 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:08.661 [DEBUG] TestAgent.server.raft: vote granted: from=9e6327ef-551d-c823-51ae-c495095adda1 term=2 tally=1 -14:08.661 [INFO] TestAgent.server.raft: election won: tally=1 -14:08.661 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14097 [Leader]" -14:08.661 [INFO] TestAgent.server: cluster leadership acquired -14:08.661 [INFO] TestAgent.server: initializing acls -14:08.662 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:08.662 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:08.662 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:08.662 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:08.662 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:08.662 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:08.662 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-9e6327ef-551d-c823-51ae-c495095adda1 -14:08.663 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-9e6327ef-551d-c823-51ae-c495095adda1.dc1 -14:08.663 [INFO] TestAgent.server: New leader elected: payload=Node-9e6327ef-551d-c823-51ae-c495095adda1 -14:08.663 [INFO] TestAgent.server: Updating LAN server: server="Node-9e6327ef-551d-c823-51ae-c495095adda1 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" -14:08.663 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-9e6327ef-551d-c823-51ae-c495095adda1.dc1 area=wan -14:08.663 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:08.685 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:08.685 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:08.685 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:08.685 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:08.685 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:08.685 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9e6327ef-551d-c823-51ae-c495095adda1 -14:08.685 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9e6327ef-551d-c823-51ae-c495095adda1 -14:08.686 [INFO] TestAgent.server: federation state anti-entropy synced -14:08.686 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9e6327ef-551d-c823-51ae-c495095adda1 -14:08.710 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -14:08.711 [INFO] TestAgent: Synced node info -14:08.711 [DEBUG] TestAgent: Node info in sync -14:09.017 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54818 latency=1.702858ms -14:09.022 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54818 latency=4.905637ms -14:09.030 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54818 latency=778.327µs -14:09.038 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54818 latency=660.582µs -14:09.043 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54818 latency=631.25µs -14:09.050 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:54820 latency=434.744µs -14:09.051 [INFO] TestAgent: Requesting shutdown -14:09.051 [INFO] TestAgent.server: shutting down server -14:09.051 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:09.051 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:09.051 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:09.051 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:09.051 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:09.051 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:09.051 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:09.051 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:09.051 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:09.051 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:09.051 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:09.051 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:09.051 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:09.051 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:09.051 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:09.051Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:09.051Z [INFO] agent.router.manager: shutting down -14:09.051 [INFO] TestAgent: consul server down -14:09.051 [INFO] TestAgent: shutdown complete -14:09.051 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=tcp -14:09.051 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=udp -14:09.052 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14093 network=tcp -14:09.553 [INFO] TestAgent: Waiting for endpoints to shut down -14:09.553 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleListCommand (1.05s) +2022-12-04T05:48:58.839Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:58.856Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:58.858 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4d77ac0c-45d2-aaf4-271a-4c31162e8d6d Address:127.0.0.1:13069}]" +48:58.858 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= +48:58.859 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4d77ac0c-45d2-aaf4-271a-4c31162e8d6d.dc1 127.0.0.1 +48:58.859 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4d77ac0c-45d2-aaf4-271a-4c31162e8d6d 127.0.0.1 +2022-12-04T05:48:58.859Z [INFO] agent.router: Initializing LAN area manager +48:58.860 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp +48:58.860 [INFO] TestAgent.server: Adding LAN server: server="Node-4d77ac0c-45d2-aaf4-271a-4c31162e8d6d (Addr: tcp/127.0.0.1:13069) (DC: dc1)" +48:58.860 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4d77ac0c-45d2-aaf4-271a-4c31162e8d6d.dc1 area=wan +48:58.860 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp +48:58.868 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp +48:58.868 [INFO] TestAgent: started state syncer +48:58.870 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp +48:58.902 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:58.902 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 +48:58.903 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:58.903 [DEBUG] TestAgent.server.raft: vote granted: from=4d77ac0c-45d2-aaf4-271a-4c31162e8d6d term=2 tally=1 +48:58.903 [INFO] TestAgent.server.raft: election won: tally=1 +48:58.903 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" +48:58.903 [INFO] TestAgent.server: cluster leadership acquired +48:58.903 [INFO] TestAgent.server: initializing acls +48:58.903 [INFO] TestAgent.server: New leader elected: payload=Node-4d77ac0c-45d2-aaf4-271a-4c31162e8d6d +48:58.903 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:58.903 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:58.904 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:58.908 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:58.908 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:58.908 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:58.908 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-4d77ac0c-45d2-aaf4-271a-4c31162e8d6d +48:58.909 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-4d77ac0c-45d2-aaf4-271a-4c31162e8d6d.dc1 +48:58.909 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:58.910 [INFO] TestAgent.server: Updating LAN server: server="Node-4d77ac0c-45d2-aaf4-271a-4c31162e8d6d (Addr: tcp/127.0.0.1:13069) (DC: dc1)" +48:58.910 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-4d77ac0c-45d2-aaf4-271a-4c31162e8d6d.dc1 area=wan +48:58.927 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:58.927 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:58.927 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:58.927 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:58.927 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:58.927 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4d77ac0c-45d2-aaf4-271a-4c31162e8d6d +48:58.927 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4d77ac0c-45d2-aaf4-271a-4c31162e8d6d +48:58.928 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4d77ac0c-45d2-aaf4-271a-4c31162e8d6d +48:58.928 [INFO] TestAgent.server: federation state anti-entropy synced +48:59.034 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54418 latency=4.811686ms +48:59.036 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54418 latency=633.431µs +48:59.043 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54418 latency=6.998949ms +48:59.047 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54418 latency=2.022745ms +48:59.052 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:54418 latency=674.508µs +48:59.057 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:54420 latency=457.503µs +48:59.058 [INFO] TestAgent: Requesting shutdown +48:59.058 [INFO] TestAgent.server: shutting down server +48:59.058 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:59.058 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:59.058 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:59.058 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:59.058 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:59.058 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:59.058 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:59.058 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:59.058 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:59.058 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:59.058 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:59.058 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:59.058 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:59.058 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:59.058 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:59.058 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:59.058Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:59.058Z [INFO] agent.router.manager: shutting down +48:59.058 [INFO] TestAgent: consul server down +48:59.058 [INFO] TestAgent: shutdown complete +48:59.058 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp +48:59.058 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp +48:59.058 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp +48:59.559 [INFO] TestAgent: Waiting for endpoints to shut down +48:59.559 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleListCommand_JSON (0.79s) +2022-12-04T05:48:58.838Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:48:58.855Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +48:58.857 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c523b2b6-13b6-b293-4a4a-1c41723d55c5 Address:127.0.0.1:13076}]" +48:58.857 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13076 [Follower]" leader= +48:58.858 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c523b2b6-13b6-b293-4a4a-1c41723d55c5.dc1 127.0.0.1 +48:58.858 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c523b2b6-13b6-b293-4a4a-1c41723d55c5 127.0.0.1 +2022-12-04T05:48:58.858Z [INFO] agent.router: Initializing LAN area manager +48:58.858 [INFO] TestAgent.server: Adding LAN server: server="Node-c523b2b6-13b6-b293-4a4a-1c41723d55c5 (Addr: tcp/127.0.0.1:13076) (DC: dc1)" +48:58.858 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c523b2b6-13b6-b293-4a4a-1c41723d55c5.dc1 area=wan +48:58.859 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=udp +48:58.859 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=tcp +48:58.859 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13072 network=tcp +48:58.859 [INFO] TestAgent: started state syncer +48:58.859 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13077 network=tcp +48:58.910 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +48:58.914 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +48:58.914 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13076 [Candidate]" term=2 +48:58.914 [DEBUG] TestAgent.server.raft: votes: needed=1 +48:58.914 [DEBUG] TestAgent.server.raft: vote granted: from=c523b2b6-13b6-b293-4a4a-1c41723d55c5 term=2 tally=1 +48:58.914 [INFO] TestAgent.server.raft: election won: tally=1 +48:58.914 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13076 [Leader]" +48:58.915 [INFO] TestAgent.server: cluster leadership acquired +48:58.915 [INFO] TestAgent.server: initializing acls +48:58.915 [INFO] TestAgent.server: Created ACL 'global-management' policy +48:58.915 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +48:58.915 [INFO] TestAgent.server: New leader elected: payload=Node-c523b2b6-13b6-b293-4a4a-1c41723d55c5 +48:58.915 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +48:58.915 [INFO] TestAgent.server: Created ACL anonymous token from configuration +48:58.915 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +48:58.915 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +48:58.916 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c523b2b6-13b6-b293-4a4a-1c41723d55c5 +48:58.916 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c523b2b6-13b6-b293-4a4a-1c41723d55c5.dc1 +48:58.916 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +48:58.916 [INFO] TestAgent.server: Updating LAN server: server="Node-c523b2b6-13b6-b293-4a4a-1c41723d55c5 (Addr: tcp/127.0.0.1:13076) (DC: dc1)" +48:58.916 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c523b2b6-13b6-b293-4a4a-1c41723d55c5.dc1 area=wan +48:58.929 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +48:58.929 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +48:58.929 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +48:58.929 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +48:58.929 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +48:58.929 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c523b2b6-13b6-b293-4a4a-1c41723d55c5 +48:58.929 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c523b2b6-13b6-b293-4a4a-1c41723d55c5 +48:58.930 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c523b2b6-13b6-b293-4a4a-1c41723d55c5 +48:58.930 [INFO] TestAgent.server: federation state anti-entropy synced +48:59.036 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:59714 latency=870.386µs +48:59.050 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:59714 latency=5.660866ms +48:59.052 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:59714 latency=629.145µs +48:59.057 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:59714 latency=658.103µs +48:59.060 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:59714 latency=581.748µs +48:59.061 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:59716 latency=339.805µs +48:59.061 [INFO] TestAgent: Requesting shutdown +48:59.061 [INFO] TestAgent.server: shutting down server +48:59.061 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +48:59.061 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +48:59.061 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +48:59.061 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +48:59.061 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +48:59.061 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +48:59.061 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +48:59.061 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +48:59.061 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +48:59.061 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +48:59.061 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +48:59.061 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +48:59.061 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +48:59.061 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +48:59.061 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +48:59.061 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:48:59.062Z [INFO] agent.router.manager: shutting down +2022-12-04T05:48:59.062Z [INFO] agent.router.manager: shutting down +48:59.062 [INFO] TestAgent: consul server down +48:59.062 [INFO] TestAgent: shutdown complete +48:59.062 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=tcp +48:59.062 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=udp +48:59.062 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13072 network=tcp +48:59.562 [INFO] TestAgent: Waiting for endpoints to shut down +48:59.562 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleListCommand (0.79s) PASS -ok github.com/hashicorp/consul/command/acl/role/list 1.249s +ok github.com/hashicorp/consul/command/acl/role/list 0.891s === RUN TestRoleReadCommand_noTabs === PAUSE TestRoleReadCommand_noTabs === RUN TestRoleReadCommand @@ -76129,183 +76313,181 @@ === RUN TestRoleReadCommand_JSON === PAUSE TestRoleReadCommand_JSON === CONT TestRoleReadCommand_noTabs ---- PASS: TestRoleReadCommand_noTabs (0.00s) === CONT TestRoleReadCommand_JSON [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestRoleReadCommand_noTabs (0.00s) === CONT TestRoleReadCommand [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range +=== RUN TestRoleReadCommand_JSON/read_by_id === RUN TestRoleReadCommand/id_or_name_required === RUN TestRoleReadCommand/read_by_id_not_found === RUN TestRoleReadCommand/read_by_name_not_found === RUN TestRoleReadCommand/read_by_id === RUN TestRoleReadCommand/read_by_id_prefix === RUN TestRoleReadCommand/read_by_name -=== RUN TestRoleReadCommand_JSON/read_by_id -2021-10-31T23:14:12.742Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:12.812Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:12.815 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f78a6983-eb3a-c619-c9ef-bfd05b7b45ff Address:127.0.0.1:23286}]" -14:12.815 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23286 [Follower]" leader= -14:12.815 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f78a6983-eb3a-c619-c9ef-bfd05b7b45ff.dc1 127.0.0.1 -14:12.816 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f78a6983-eb3a-c619-c9ef-bfd05b7b45ff 127.0.0.1 -2021-10-31T23:14:12.816Z [INFO] agent.router: Initializing LAN area manager -14:12.816 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23281 network=udp -14:12.816 [INFO] TestAgent.server: Adding LAN server: server="Node-f78a6983-eb3a-c619-c9ef-bfd05b7b45ff (Addr: tcp/127.0.0.1:23286) (DC: dc1)" -14:12.816 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f78a6983-eb3a-c619-c9ef-bfd05b7b45ff.dc1 area=wan -14:12.817 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23281 network=tcp -14:12.817 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23282 network=tcp -14:12.817 [INFO] TestAgent: started state syncer -14:12.819 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23287 network=tcp -14:12.867 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -14:12.869 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:12.869 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23286 [Candidate]" term=2 -14:12.870 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:12.870 [DEBUG] TestAgent.server.raft: vote granted: from=f78a6983-eb3a-c619-c9ef-bfd05b7b45ff term=2 tally=1 -14:12.870 [INFO] TestAgent.server.raft: election won: tally=1 -14:12.870 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23286 [Leader]" -14:12.870 [INFO] TestAgent.server: cluster leadership acquired -14:12.870 [INFO] TestAgent.server: initializing acls -14:12.870 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:12.870 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:12.871 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:12.871 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:12.871 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:12.871 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:12.871 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f78a6983-eb3a-c619-c9ef-bfd05b7b45ff -14:12.871 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f78a6983-eb3a-c619-c9ef-bfd05b7b45ff.dc1 -14:12.872 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:12.873 [INFO] TestAgent.server: New leader elected: payload=Node-f78a6983-eb3a-c619-c9ef-bfd05b7b45ff -14:12.874 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f78a6983-eb3a-c619-c9ef-bfd05b7b45ff.dc1 area=wan -14:12.874 [INFO] TestAgent.server: Updating LAN server: server="Node-f78a6983-eb3a-c619-c9ef-bfd05b7b45ff (Addr: tcp/127.0.0.1:23286) (DC: dc1)" -14:12.940 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:12.940 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:12.940 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:12.940 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:12.940 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:12.940 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f78a6983-eb3a-c619-c9ef-bfd05b7b45ff -14:12.940 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f78a6983-eb3a-c619-c9ef-bfd05b7b45ff -14:12.941 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f78a6983-eb3a-c619-c9ef-bfd05b7b45ff -14:12.941 [INFO] TestAgent.server: federation state anti-entropy synced -14:12.974 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/473ab180-eb39-1595-70bf-f534c8222936 from=127.0.0.1:60340 latency=566.66µs -14:12.975 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/name/blah from=127.0.0.1:60342 latency=87.151µs -14:12.977 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:60344 latency=780.433µs -14:12.991 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/8f967096-7060-3adb-47a9-1b2b75149153 from=127.0.0.1:60346 latency=1.113001ms -14:12.998 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:60344 latency=688.194µs -14:13.003 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:60348 latency=363.433µs -14:13.010 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/bbc024d9-bcfe-fc9e-ca82-e76876244229 from=127.0.0.1:60348 latency=321.795µs -14:13.011 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:60344 latency=508.18µs -14:13.013 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/name/test-role-by-name from=127.0.0.1:60350 latency=413.872µs -14:13.018 [INFO] TestAgent: Requesting shutdown -14:13.018 [INFO] TestAgent.server: shutting down server -14:13.018 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:13.018 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:13.018 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:13.018 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:13.018 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:13.018 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:13.018 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:13.018 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:13.018 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:13.018 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -14:13.018 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:13.018 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:13.018 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:13.018 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:13.018 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:13.018 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:13.018Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:13.018Z [INFO] agent.router.manager: shutting down -14:13.018 [INFO] TestAgent: consul server down -14:13.018 [INFO] TestAgent: shutdown complete -14:13.018 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23281 network=tcp -14:13.019 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23281 network=udp -14:13.019 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23282 network=tcp -14:13.521 [INFO] TestAgent: Waiting for endpoints to shut down -14:13.521 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleReadCommand (0.92s) - --- PASS: TestRoleReadCommand/id_or_name_required (0.00s) - --- PASS: TestRoleReadCommand/read_by_id_not_found (0.03s) +2022-12-04T05:49:04.607Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:04.628Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:04.631 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ebcf2762-f23c-36d4-b703-36b356940de0 Address:127.0.0.1:21237}]" +49:04.631 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ebcf2762-f23c-36d4-b703-36b356940de0.dc1 127.0.0.1 +49:04.632 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ebcf2762-f23c-36d4-b703-36b356940de0 127.0.0.1 +2022-12-04T05:49:04.632Z [INFO] agent.router: Initializing LAN area manager +49:04.632 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21237 [Follower]" leader= +49:04.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=udp +49:04.633 [INFO] TestAgent.server: Adding LAN server: server="Node-ebcf2762-f23c-36d4-b703-36b356940de0 (Addr: tcp/127.0.0.1:21237) (DC: dc1)" +49:04.633 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ebcf2762-f23c-36d4-b703-36b356940de0.dc1 area=wan +49:04.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=tcp +49:04.639 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21233 network=tcp +49:04.639 [INFO] TestAgent: started state syncer +49:04.640 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21238 network=tcp +49:04.672 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:04.672 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21237 [Candidate]" term=2 +49:04.672 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:04.672 [DEBUG] TestAgent.server.raft: vote granted: from=ebcf2762-f23c-36d4-b703-36b356940de0 term=2 tally=1 +49:04.672 [INFO] TestAgent.server.raft: election won: tally=1 +49:04.672 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21237 [Leader]" +49:04.672 [INFO] TestAgent.server: cluster leadership acquired +49:04.672 [INFO] TestAgent.server: initializing acls +49:04.673 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:04.673 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:04.673 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:04.673 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:04.673 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:04.673 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:04.673 [INFO] TestAgent.server: New leader elected: payload=Node-ebcf2762-f23c-36d4-b703-36b356940de0 +49:04.673 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ebcf2762-f23c-36d4-b703-36b356940de0 +49:04.674 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ebcf2762-f23c-36d4-b703-36b356940de0.dc1 +49:04.674 [INFO] TestAgent.server: Updating LAN server: server="Node-ebcf2762-f23c-36d4-b703-36b356940de0 (Addr: tcp/127.0.0.1:21237) (DC: dc1)" +49:04.674 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ebcf2762-f23c-36d4-b703-36b356940de0.dc1 area=wan +49:04.674 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:04.690 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:04.690 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:04.690 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:04.690 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:04.690 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:04.690 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ebcf2762-f23c-36d4-b703-36b356940de0 +49:04.690 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ebcf2762-f23c-36d4-b703-36b356940de0 +49:04.691 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ebcf2762-f23c-36d4-b703-36b356940de0 +49:04.692 [INFO] TestAgent.server: federation state anti-entropy synced +49:04.711 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:51952 latency=1.216639ms +49:04.712 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/cd4e5042-d69a-9c31-e821-f8827a5ae20e from=127.0.0.1:51954 latency=384.774µs +49:04.713 [INFO] TestAgent: Requesting shutdown +49:04.713 [INFO] TestAgent.server: shutting down server +49:04.713 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:04.713 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:04.713 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:04.713 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:04.713 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:04.713 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:04.713 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:04.713 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:04.713 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:04.713 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:04.713 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:04.713 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:04.713 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:04.713 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:04.713 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:04.713 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:04.713Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:04.713Z [INFO] agent.router.manager: shutting down +49:04.714 [INFO] TestAgent: consul server down +49:04.714 [INFO] TestAgent: shutdown complete +49:04.714 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=tcp +49:04.714 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=udp +49:04.714 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21233 network=tcp +49:05.216 [INFO] TestAgent: Waiting for endpoints to shut down +49:05.216 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleReadCommand_JSON (0.66s) + --- PASS: TestRoleReadCommand_JSON/read_by_id (0.00s) +2022-12-04T05:49:04.606Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:04.629Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:04.631 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0a70e973-ef4b-5a8d-ad9c-11408c1a8bb0 Address:127.0.0.1:21244}]" +49:04.631 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0a70e973-ef4b-5a8d-ad9c-11408c1a8bb0.dc1 127.0.0.1 +49:04.632 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21244 [Follower]" leader= +49:04.632 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0a70e973-ef4b-5a8d-ad9c-11408c1a8bb0 127.0.0.1 +2022-12-04T05:49:04.632Z [INFO] agent.router: Initializing LAN area manager +49:04.633 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0a70e973-ef4b-5a8d-ad9c-11408c1a8bb0.dc1 area=wan +49:04.633 [INFO] TestAgent.server: Adding LAN server: server="Node-0a70e973-ef4b-5a8d-ad9c-11408c1a8bb0 (Addr: tcp/127.0.0.1:21244) (DC: dc1)" +49:04.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21239 network=udp +49:04.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21239 network=tcp +49:04.640 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21240 network=tcp +49:04.640 [INFO] TestAgent: started state syncer +49:04.640 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21245 network=tcp +49:04.679 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:04.679 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21244 [Candidate]" term=2 +49:04.679 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:04.679 [DEBUG] TestAgent.server.raft: vote granted: from=0a70e973-ef4b-5a8d-ad9c-11408c1a8bb0 term=2 tally=1 +49:04.679 [INFO] TestAgent.server.raft: election won: tally=1 +49:04.679 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21244 [Leader]" +49:04.679 [INFO] TestAgent.server: cluster leadership acquired +49:04.679 [INFO] TestAgent.server: New leader elected: payload=Node-0a70e973-ef4b-5a8d-ad9c-11408c1a8bb0 +49:04.679 [INFO] TestAgent.server: initializing acls +49:04.680 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:04.680 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:04.680 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:04.680 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:04.680 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:04.680 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:04.681 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0a70e973-ef4b-5a8d-ad9c-11408c1a8bb0 +49:04.681 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0a70e973-ef4b-5a8d-ad9c-11408c1a8bb0.dc1 +49:04.681 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:04.681 [INFO] TestAgent.server: Updating LAN server: server="Node-0a70e973-ef4b-5a8d-ad9c-11408c1a8bb0 (Addr: tcp/127.0.0.1:21244) (DC: dc1)" +49:04.681 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0a70e973-ef4b-5a8d-ad9c-11408c1a8bb0.dc1 area=wan +49:04.691 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:04.691 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:04.691 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:04.691 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:04.691 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:04.691 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a70e973-ef4b-5a8d-ad9c-11408c1a8bb0 +49:04.691 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0a70e973-ef4b-5a8d-ad9c-11408c1a8bb0 +49:04.692 [INFO] TestAgent.server: federation state anti-entropy synced +49:04.692 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0a70e973-ef4b-5a8d-ad9c-11408c1a8bb0 +49:04.742 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/621b80cf-d367-8986-1876-0ee7c092a615 from=127.0.0.1:35460 latency=515.545µs +49:04.743 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/name/blah from=127.0.0.1:35462 latency=164.732µs +49:04.745 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:35464 latency=679.651µs +49:04.747 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/301a52e4-3b37-6362-87b9-37c9fdc23320 from=127.0.0.1:35466 latency=788.127µs +49:04.748 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:35464 latency=548.994µs +49:04.749 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/roles from=127.0.0.1:35468 latency=263.343µs +49:04.750 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/046a3dde-dce1-ab0b-aa2e-0c519acf9f06 from=127.0.0.1:35468 latency=220.306µs +49:04.751 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:35464 latency=441.308µs +49:04.752 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/name/test-role-by-name from=127.0.0.1:35470 latency=201.565µs +49:04.752 [INFO] TestAgent: Requesting shutdown +49:04.752 [INFO] TestAgent.server: shutting down server +49:04.752 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:04.752 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:04.752 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:04.752 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:04.752 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:04.752 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:04.752 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:04.752 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:04.752 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:04.752 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:04.752 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:04.752 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:04.752 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:04.752 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:04.752 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:04.752 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:04.753Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:04.753Z [INFO] agent.router.manager: shutting down +49:04.753 [INFO] TestAgent: consul server down +49:04.753 [INFO] TestAgent: shutdown complete +49:04.753 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21239 network=tcp +49:04.753 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21239 network=udp +49:04.753 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21240 network=tcp +49:05.256 [INFO] TestAgent: Waiting for endpoints to shut down +49:05.256 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleReadCommand (0.70s) + --- PASS: TestRoleReadCommand/id_or_name_required (0.01s) + --- PASS: TestRoleReadCommand/read_by_id_not_found (0.00s) --- PASS: TestRoleReadCommand/read_by_name_not_found (0.00s) - --- PASS: TestRoleReadCommand/read_by_id (0.02s) - --- PASS: TestRoleReadCommand/read_by_id_prefix (0.02s) - --- PASS: TestRoleReadCommand/read_by_name (0.01s) -2021-10-31T23:14:12.744Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:12.795Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:12.814 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2aa547b8-1c65-afb9-9eb7-cb52c19f5592 Address:127.0.0.1:23279}]" -14:12.815 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2aa547b8-1c65-afb9-9eb7-cb52c19f5592.dc1 127.0.0.1 -14:12.815 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23279 [Follower]" leader= -14:12.815 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2aa547b8-1c65-afb9-9eb7-cb52c19f5592 127.0.0.1 -2021-10-31T23:14:12.815Z [INFO] agent.router: Initializing LAN area manager -14:12.816 [INFO] TestAgent.server: Adding LAN server: server="Node-2aa547b8-1c65-afb9-9eb7-cb52c19f5592 (Addr: tcp/127.0.0.1:23279) (DC: dc1)" -14:12.816 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2aa547b8-1c65-afb9-9eb7-cb52c19f5592.dc1 area=wan -14:12.816 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=udp -14:12.816 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=tcp -14:12.817 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23275 network=tcp -14:12.817 [INFO] TestAgent: started state syncer -14:12.817 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23280 network=tcp -14:12.867 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -14:12.885 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:12.885 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23279 [Candidate]" term=2 -14:12.886 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:12.886 [DEBUG] TestAgent.server.raft: vote granted: from=2aa547b8-1c65-afb9-9eb7-cb52c19f5592 term=2 tally=1 -14:12.886 [INFO] TestAgent.server.raft: election won: tally=1 -14:12.886 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23279 [Leader]" -14:12.886 [INFO] TestAgent.server: cluster leadership acquired -14:12.886 [INFO] TestAgent.server: initializing acls -14:12.886 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:12.886 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:12.887 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:12.887 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:12.887 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:12.887 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:12.887 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-2aa547b8-1c65-afb9-9eb7-cb52c19f5592 -14:12.887 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-2aa547b8-1c65-afb9-9eb7-cb52c19f5592.dc1 -14:12.888 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:12.888 [INFO] TestAgent.server: New leader elected: payload=Node-2aa547b8-1c65-afb9-9eb7-cb52c19f5592 -14:12.888 [INFO] TestAgent.server: Updating LAN server: server="Node-2aa547b8-1c65-afb9-9eb7-cb52c19f5592 (Addr: tcp/127.0.0.1:23279) (DC: dc1)" -14:12.888 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-2aa547b8-1c65-afb9-9eb7-cb52c19f5592.dc1 area=wan -14:12.947 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:12.947 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:12.947 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:12.947 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:12.947 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:12.947 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2aa547b8-1c65-afb9-9eb7-cb52c19f5592 -14:12.947 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2aa547b8-1c65-afb9-9eb7-cb52c19f5592 -14:12.948 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2aa547b8-1c65-afb9-9eb7-cb52c19f5592 -14:12.948 [INFO] TestAgent.server: federation state anti-entropy synced -14:13.110 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:42606 latency=1.407012ms -14:13.127 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/f34477ce-aa2d-d887-bd5f-ece1194a7ace from=127.0.0.1:42608 latency=1.387427ms -14:13.128 [INFO] TestAgent: Requesting shutdown -14:13.128 [INFO] TestAgent.server: shutting down server -14:13.128 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:13.128 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:13.128 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:13.128 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:13.128 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:13.128 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:13.128 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:13.128 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:13.128 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:13.128 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -14:13.128 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:13.128 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:13.128 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:13.128 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:13.128 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:13.128 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:13.128Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:13.128Z [INFO] agent.router.manager: shutting down -14:13.128 [INFO] TestAgent: consul server down -14:13.128 [INFO] TestAgent: shutdown complete -14:13.128 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=tcp -14:13.128 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=udp -14:13.128 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23275 network=tcp -14:13.629 [INFO] TestAgent: Waiting for endpoints to shut down -14:13.629 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleReadCommand_JSON (1.03s) - --- PASS: TestRoleReadCommand_JSON/read_by_id (0.02s) + --- PASS: TestRoleReadCommand/read_by_id (0.00s) + --- PASS: TestRoleReadCommand/read_by_id_prefix (0.00s) + --- PASS: TestRoleReadCommand/read_by_name (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/role/read 1.778s +ok github.com/hashicorp/consul/command/acl/role/read 0.770s === RUN TestRoleUpdateCommand_noTabs === PAUSE TestRoleUpdateCommand_noTabs === RUN TestRoleUpdateCommand @@ -76318,448 +76500,454 @@ --- PASS: TestRoleUpdateCommand_noTabs (0.00s) === CONT TestRoleUpdateCommand_noMerge [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestRoleUpdateCommand_JSON === CONT TestRoleUpdateCommand +=== CONT TestRoleUpdateCommand_JSON [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -=== RUN TestRoleUpdateCommand_JSON/update_a_role_that_does_not_exist -=== RUN TestRoleUpdateCommand_JSON/update_with_policy_by_name === RUN TestRoleUpdateCommand_noMerge/update_a_role_that_does_not_exist === RUN TestRoleUpdateCommand_noMerge/update_with_policy_by_name === RUN TestRoleUpdateCommand_noMerge/update_with_policy_by_id === RUN TestRoleUpdateCommand_noMerge/update_with_service_identity +=== RUN TestRoleUpdateCommand_noMerge/update_with_service_identity_scoped_to_2_DCs === RUN TestRoleUpdateCommand/update_a_role_that_does_not_exist === RUN TestRoleUpdateCommand/update_with_policy_by_name === RUN TestRoleUpdateCommand/update_with_policy_by_id -=== RUN TestRoleUpdateCommand_noMerge/update_with_service_identity_scoped_to_2_DCs === RUN TestRoleUpdateCommand/update_with_service_identity === RUN TestRoleUpdateCommand/update_with_service_identity_scoped_to_2_DCs === RUN TestRoleUpdateCommand/update_with_node_identity -2021-10-31T23:14:19.455Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:19.527Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:19.529 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:55925d52-7ef1-6379-4b28-96eb3566f835 Address:127.0.0.1:30433}]" -14:19.529 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30433 [Follower]" leader= -14:19.543 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-55925d52-7ef1-6379-4b28-96eb3566f835.dc1 127.0.0.1 -14:19.544 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-55925d52-7ef1-6379-4b28-96eb3566f835 127.0.0.1 -2021-10-31T23:14:19.544Z [INFO] agent.router: Initializing LAN area manager -14:19.544 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=udp -14:19.544 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-55925d52-7ef1-6379-4b28-96eb3566f835.dc1 area=wan -14:19.544 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=tcp -14:19.544 [INFO] TestAgent.server: Adding LAN server: server="Node-55925d52-7ef1-6379-4b28-96eb3566f835 (Addr: tcp/127.0.0.1:30433) (DC: dc1)" -14:19.556 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30429 network=tcp -14:19.557 [INFO] TestAgent: started state syncer -14:19.557 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30434 network=tcp -14:19.607 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:19.607 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30433 [Candidate]" term=2 -14:19.607 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:19.607 [DEBUG] TestAgent.server.raft: vote granted: from=55925d52-7ef1-6379-4b28-96eb3566f835 term=2 tally=1 -14:19.607 [INFO] TestAgent.server.raft: election won: tally=1 -14:19.607 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30433 [Leader]" -14:19.613 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -14:19.613 [INFO] TestAgent.server: cluster leadership acquired -14:19.614 [INFO] TestAgent.server: initializing acls -14:19.614 [INFO] TestAgent.server: New leader elected: payload=Node-55925d52-7ef1-6379-4b28-96eb3566f835 -14:19.614 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:19.614 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:19.615 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:19.615 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:19.615 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:19.615 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:19.615 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-55925d52-7ef1-6379-4b28-96eb3566f835 -14:19.615 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-55925d52-7ef1-6379-4b28-96eb3566f835.dc1 -14:19.616 [INFO] TestAgent.server: Updating LAN server: server="Node-55925d52-7ef1-6379-4b28-96eb3566f835 (Addr: tcp/127.0.0.1:30433) (DC: dc1)" -14:19.616 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-55925d52-7ef1-6379-4b28-96eb3566f835.dc1 area=wan -14:19.616 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:19.675 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:19.675 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:19.675 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:19.675 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:19.675 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:19.675 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-55925d52-7ef1-6379-4b28-96eb3566f835 -14:19.675 [INFO] TestAgent.server: member joined, marking health alive: member=Node-55925d52-7ef1-6379-4b28-96eb3566f835 -14:19.675 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-55925d52-7ef1-6379-4b28-96eb3566f835 -14:19.675 [INFO] TestAgent.server: federation state anti-entropy synced -14:19.699 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:37812 latency=8.398744ms -14:19.712 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:37812 latency=1.323478ms -14:19.718 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/daf6b191-87d9-e547-3175-40ec5d94098b from=127.0.0.1:37814 latency=273.564µs -14:19.728 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/1b0fb8f8-b4eb-4b8c-19d9-53dc13bb2f3d from=127.0.0.1:37816 latency=519.838µs -14:19.730 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/1b0fb8f8-b4eb-4b8c-19d9-53dc13bb2f3d from=127.0.0.1:37816 latency=763.063µs -14:19.730 [INFO] TestAgent: Requesting shutdown -14:19.730 [INFO] TestAgent.server: shutting down server -14:19.730 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:19.730 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:19.730 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:19.730 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:19.730 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:19.730 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:19.730 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:19.730 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:19.730 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:19.730 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -14:19.731 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:19.731 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:19.731 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:19.731 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:19.731 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:19.731 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:19.731Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:19.731Z [INFO] agent.router.manager: shutting down -14:19.731 [INFO] TestAgent: consul server down -14:19.731 [INFO] TestAgent: shutdown complete -14:19.731 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=tcp -14:19.731 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=udp -14:19.731 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30429 network=tcp -14:20.233 [INFO] TestAgent: Waiting for endpoints to shut down -14:20.233 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleUpdateCommand_JSON (0.94s) - --- PASS: TestRoleUpdateCommand_JSON/update_a_role_that_does_not_exist (0.01s) - --- PASS: TestRoleUpdateCommand_JSON/update_with_policy_by_name (0.01s) -2021-10-31T23:14:19.467Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:19.587Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:19.600 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:427af7b6-93e0-8af4-b1e0-47b993dac4b7 Address:127.0.0.1:30426}]" -14:19.601 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-427af7b6-93e0-8af4-b1e0-47b993dac4b7.dc1 127.0.0.1 -14:19.601 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-427af7b6-93e0-8af4-b1e0-47b993dac4b7 127.0.0.1 -2021-10-31T23:14:19.601Z [INFO] agent.router: Initializing LAN area manager -14:19.601 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= -14:19.602 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-427af7b6-93e0-8af4-b1e0-47b993dac4b7.dc1 area=wan -14:19.602 [INFO] TestAgent.server: Adding LAN server: server="Node-427af7b6-93e0-8af4-b1e0-47b993dac4b7 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" -14:19.607 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp -14:19.608 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp -14:19.609 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp -14:19.609 [INFO] TestAgent: started state syncer -14:19.609 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp -14:19.651 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:19.651 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 -14:19.652 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:19.652 [DEBUG] TestAgent.server.raft: vote granted: from=427af7b6-93e0-8af4-b1e0-47b993dac4b7 term=2 tally=1 -14:19.652 [INFO] TestAgent.server.raft: election won: tally=1 -14:19.652 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" -14:19.652 [INFO] TestAgent.server: cluster leadership acquired -14:19.652 [INFO] TestAgent.server: initializing acls -14:19.652 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:19.652 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:19.652 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:19.653 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:19.653 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:19.653 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:19.653 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-427af7b6-93e0-8af4-b1e0-47b993dac4b7 -14:19.653 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-427af7b6-93e0-8af4-b1e0-47b993dac4b7.dc1 -14:19.654 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:19.654 [INFO] TestAgent.server: New leader elected: payload=Node-427af7b6-93e0-8af4-b1e0-47b993dac4b7 -14:19.654 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-427af7b6-93e0-8af4-b1e0-47b993dac4b7.dc1 area=wan -14:19.654 [INFO] TestAgent.server: Updating LAN server: server="Node-427af7b6-93e0-8af4-b1e0-47b993dac4b7 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" -14:19.673 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:19.673 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:19.673 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:19.673 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:19.673 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:19.673 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-427af7b6-93e0-8af4-b1e0-47b993dac4b7 -14:19.673 [INFO] TestAgent.server: member joined, marking health alive: member=Node-427af7b6-93e0-8af4-b1e0-47b993dac4b7 -14:19.674 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-427af7b6-93e0-8af4-b1e0-47b993dac4b7 -14:19.674 [INFO] TestAgent.server: federation state anti-entropy synced -14:19.959 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:44602 latency=1.181538ms -14:19.967 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:44602 latency=718.294µs -14:19.968 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:44602 latency=436.459µs -14:19.978 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/4778e87c-7fb9-cdcd-4931-69686b0100cc from=127.0.0.1:44604 latency=247.189µs -14:19.986 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:44602 latency=881.064µs -14:19.990 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -14:19.991 [INFO] TestAgent: Synced node info -14:19.991 [DEBUG] TestAgent: Node info in sync -14:19.991 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/0c8eda62-2d0d-c0f9-3bd8-406de6cd4b00 from=127.0.0.1:44606 latency=269.124µs -14:19.997 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/0c8eda62-2d0d-c0f9-3bd8-406de6cd4b00 from=127.0.0.1:44606 latency=576.526µs -14:19.997 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/0c8eda62-2d0d-c0f9-3bd8-406de6cd4b00 from=127.0.0.1:44602 latency=149.492µs -14:20.001 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:44602 latency=1.121908ms -14:20.007 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/ce35e901-1788-daf4-e580-6493b5d5f44c from=127.0.0.1:44608 latency=992.405µs -14:20.008 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/ce35e901-1788-daf4-e580-6493b5d5f44c from=127.0.0.1:44608 latency=551.667µs -14:20.009 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/ce35e901-1788-daf4-e580-6493b5d5f44c from=127.0.0.1:44602 latency=149.964µs -14:20.015 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:44602 latency=969.914µs -14:20.043 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/ae8174eb-c0a4-b9ac-1f96-7e112d98b9a3 from=127.0.0.1:44616 latency=569.707µs -14:20.045 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/ae8174eb-c0a4-b9ac-1f96-7e112d98b9a3 from=127.0.0.1:44616 latency=789.595µs -14:20.046 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/ae8174eb-c0a4-b9ac-1f96-7e112d98b9a3 from=127.0.0.1:44602 latency=297.698µs -14:20.051 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:44602 latency=820.337µs -14:20.066 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/63ac65f2-1379-7abe-a3b8-6e825b298e7c from=127.0.0.1:44620 latency=371.106µs -14:20.067 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/63ac65f2-1379-7abe-a3b8-6e825b298e7c from=127.0.0.1:44620 latency=557.526µs -14:20.070 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/63ac65f2-1379-7abe-a3b8-6e825b298e7c from=127.0.0.1:44602 latency=354.763µs -14:20.070 [INFO] TestAgent: Requesting shutdown -14:20.071 [INFO] TestAgent.server: shutting down server -14:20.071 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:20.071 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:20.071 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:20.071 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:20.071 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:20.071 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:20.071 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:20.071 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:20.071 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:20.071 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:20.071 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:20.071 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:20.071 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:20.071 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:20.071 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:20.071Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:20.071Z [INFO] agent.router.manager: shutting down -14:20.071 [INFO] TestAgent: consul server down -14:20.071 [INFO] TestAgent: shutdown complete -14:20.071 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp -14:20.071 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp -14:20.071 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp -14:20.571 [INFO] TestAgent: Waiting for endpoints to shut down -14:20.571 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleUpdateCommand_noMerge (1.28s) - --- PASS: TestRoleUpdateCommand_noMerge/update_a_role_that_does_not_exist (0.01s) - --- PASS: TestRoleUpdateCommand_noMerge/update_with_policy_by_name (0.02s) +=== RUN TestRoleUpdateCommand_JSON/update_a_role_that_does_not_exist +=== RUN TestRoleUpdateCommand_JSON/update_with_policy_by_name +2022-12-04T05:49:05.907Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:05.960Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:05.962 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:91c967af-4802-fbaf-4a6f-3aa6852e3681 Address:127.0.0.1:19195}]" +49:05.962 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-91c967af-4802-fbaf-4a6f-3aa6852e3681.dc1 127.0.0.1 +49:05.962 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= +49:05.963 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-91c967af-4802-fbaf-4a6f-3aa6852e3681 127.0.0.1 +2022-12-04T05:49:05.963Z [INFO] agent.router: Initializing LAN area manager +49:05.963 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp +49:05.963 [INFO] TestAgent.server: Adding LAN server: server="Node-91c967af-4802-fbaf-4a6f-3aa6852e3681 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +49:05.963 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-91c967af-4802-fbaf-4a6f-3aa6852e3681.dc1 area=wan +49:05.963 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp +49:05.964 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp +49:05.964 [INFO] TestAgent: started state syncer +49:05.964 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp +49:06.008 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:06.008 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 +49:06.008 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:06.008 [DEBUG] TestAgent.server.raft: vote granted: from=91c967af-4802-fbaf-4a6f-3aa6852e3681 term=2 tally=1 +49:06.008 [INFO] TestAgent.server.raft: election won: tally=1 +49:06.008 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" +49:06.008 [INFO] TestAgent.server: cluster leadership acquired +49:06.008 [INFO] TestAgent.server: initializing acls +49:06.009 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:06.009 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:06.009 [INFO] TestAgent.server: New leader elected: payload=Node-91c967af-4802-fbaf-4a6f-3aa6852e3681 +49:06.020 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:06.020 [INFO] TestAgent.server: initializing acls +49:06.020 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:06.021 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:06.021 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:06.021 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:06.021 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode +49:06.021 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-91c967af-4802-fbaf-4a6f-3aa6852e3681 +49:06.021 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-91c967af-4802-fbaf-4a6f-3aa6852e3681.dc1 +49:06.021 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:06.022 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-91c967af-4802-fbaf-4a6f-3aa6852e3681.dc1 area=wan +49:06.022 [INFO] TestAgent.server: Updating LAN server: server="Node-91c967af-4802-fbaf-4a6f-3aa6852e3681 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +49:06.022 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-91c967af-4802-fbaf-4a6f-3aa6852e3681 +49:06.022 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-91c967af-4802-fbaf-4a6f-3aa6852e3681.dc1 +49:06.023 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:06.023 [INFO] TestAgent.server: Updating LAN server: server="Node-91c967af-4802-fbaf-4a6f-3aa6852e3681 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +49:06.023 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-91c967af-4802-fbaf-4a6f-3aa6852e3681.dc1 area=wan +49:06.028 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:06.028 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:06.028 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:06.028 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:06.028 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:06.028 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-91c967af-4802-fbaf-4a6f-3aa6852e3681 +49:06.028 [INFO] TestAgent.server: member joined, marking health alive: member=Node-91c967af-4802-fbaf-4a6f-3aa6852e3681 +49:06.029 [INFO] TestAgent.server: federation state anti-entropy synced +49:06.030 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-91c967af-4802-fbaf-4a6f-3aa6852e3681 +49:06.030 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-91c967af-4802-fbaf-4a6f-3aa6852e3681 +49:06.043 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:40354 latency=2.028986ms +49:06.045 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:40354 latency=677.123µs +49:06.047 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:40354 latency=751.527µs +49:06.048 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/e58ea8e8-66b3-cd78-871a-5d817b5ffb5f from=127.0.0.1:40356 latency=173.224µs +49:06.051 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:40354 latency=981.499µs +49:06.066 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/3500a067-b0ed-8f67-95c9-0b2c1585fcce from=127.0.0.1:40358 latency=7.772868ms +49:06.081 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/3500a067-b0ed-8f67-95c9-0b2c1585fcce from=127.0.0.1:40358 latency=1.090817ms +49:06.090 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/3500a067-b0ed-8f67-95c9-0b2c1585fcce from=127.0.0.1:40354 latency=1.599068ms +49:06.097 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:40354 latency=707.656µs +49:06.105 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/87f2defa-953b-8da3-1366-f5c7e8aa74b6 from=127.0.0.1:40360 latency=462.197µs +49:06.106 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/87f2defa-953b-8da3-1366-f5c7e8aa74b6 from=127.0.0.1:40360 latency=749.239µs +49:06.107 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/87f2defa-953b-8da3-1366-f5c7e8aa74b6 from=127.0.0.1:40354 latency=158.901µs +49:06.108 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:40354 latency=451.251µs +49:06.117 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/6317da8a-2b32-cd68-1068-dd3b12cb874d from=127.0.0.1:40362 latency=485.99µs +49:06.123 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/6317da8a-2b32-cd68-1068-dd3b12cb874d from=127.0.0.1:40362 latency=881.153µs +49:06.124 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/6317da8a-2b32-cd68-1068-dd3b12cb874d from=127.0.0.1:40354 latency=497.448µs +49:06.125 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:40354 latency=640.95µs +49:06.128 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/47d2894d-b581-6352-0c55-81f6d76f6f38 from=127.0.0.1:40364 latency=291.465µs +49:06.130 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/47d2894d-b581-6352-0c55-81f6d76f6f38 from=127.0.0.1:40364 latency=711.626µs +49:06.131 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/47d2894d-b581-6352-0c55-81f6d76f6f38 from=127.0.0.1:40354 latency=182.334µs +49:06.131 [INFO] TestAgent: Requesting shutdown +49:06.131 [INFO] TestAgent.server: shutting down server +49:06.131 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:06.131 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:06.131 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:06.131 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:06.131 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:06.131 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:06.131 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:06.131 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:06.131 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:06.131 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:06.131 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:06.131 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:06.131 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:49:06.131Z [INFO] agent.router.manager: shutting down +49:06.131 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:06.131Z [INFO] agent.router.manager: shutting down +49:06.131 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:06.131 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:06.131 [INFO] TestAgent: consul server down +49:06.131 [INFO] TestAgent: shutdown complete +49:06.131 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp +49:06.131 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp +49:06.131 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp +49:06.632 [INFO] TestAgent: Waiting for endpoints to shut down +49:06.632 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleUpdateCommand_noMerge (0.79s) + --- PASS: TestRoleUpdateCommand_noMerge/update_a_role_that_does_not_exist (0.00s) + --- PASS: TestRoleUpdateCommand_noMerge/update_with_policy_by_name (0.05s) --- PASS: TestRoleUpdateCommand_noMerge/update_with_policy_by_id (0.01s) - --- PASS: TestRoleUpdateCommand_noMerge/update_with_service_identity (0.04s) - --- PASS: TestRoleUpdateCommand_noMerge/update_with_service_identity_scoped_to_2_DCs (0.02s) -2021-10-31T23:14:19.464Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:19.557Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:19.575 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7cb7614e-5f3c-9a41-5de8-c62b45fb9ff3 Address:127.0.0.1:30440}]" -14:19.575 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7cb7614e-5f3c-9a41-5de8-c62b45fb9ff3.dc1 127.0.0.1 -14:19.576 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7cb7614e-5f3c-9a41-5de8-c62b45fb9ff3 127.0.0.1 -2021-10-31T23:14:19.576Z [INFO] agent.router: Initializing LAN area manager -14:19.576 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30435 network=udp -14:19.576 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30440 [Follower]" leader= -14:19.602 [INFO] TestAgent.server: Adding LAN server: server="Node-7cb7614e-5f3c-9a41-5de8-c62b45fb9ff3 (Addr: tcp/127.0.0.1:30440) (DC: dc1)" -14:19.602 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7cb7614e-5f3c-9a41-5de8-c62b45fb9ff3.dc1 area=wan -14:19.602 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30435 network=tcp -14:19.609 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30436 network=tcp -14:19.610 [INFO] TestAgent: started state syncer -14:19.610 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30441 network=tcp -14:19.616 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:19.616 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30440 [Candidate]" term=2 -14:19.616 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:19.616 [DEBUG] TestAgent.server.raft: vote granted: from=7cb7614e-5f3c-9a41-5de8-c62b45fb9ff3 term=2 tally=1 -14:19.616 [INFO] TestAgent.server.raft: election won: tally=1 -14:19.616 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30440 [Leader]" -14:19.617 [INFO] TestAgent.server: cluster leadership acquired -14:19.617 [INFO] TestAgent.server: initializing acls -14:19.617 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:19.617 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:19.617 [INFO] TestAgent.server: New leader elected: payload=Node-7cb7614e-5f3c-9a41-5de8-c62b45fb9ff3 -14:19.618 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:19.618 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:19.618 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:19.618 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:19.618 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7cb7614e-5f3c-9a41-5de8-c62b45fb9ff3 -14:19.618 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7cb7614e-5f3c-9a41-5de8-c62b45fb9ff3.dc1 -14:19.619 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:19.619 [INFO] TestAgent.server: Updating LAN server: server="Node-7cb7614e-5f3c-9a41-5de8-c62b45fb9ff3 (Addr: tcp/127.0.0.1:30440) (DC: dc1)" -14:19.619 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7cb7614e-5f3c-9a41-5de8-c62b45fb9ff3.dc1 area=wan -14:19.671 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:19.671 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:19.671 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:19.671 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:19.671 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:19.671 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7cb7614e-5f3c-9a41-5de8-c62b45fb9ff3 -14:19.671 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7cb7614e-5f3c-9a41-5de8-c62b45fb9ff3 -14:19.672 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7cb7614e-5f3c-9a41-5de8-c62b45fb9ff3 -14:19.672 [INFO] TestAgent.server: federation state anti-entropy synced -14:19.705 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -14:19.705 [INFO] TestAgent: Synced node info -14:19.705 [DEBUG] TestAgent: Node info in sync -14:19.922 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -14:19.922 [DEBUG] TestAgent: Node info in sync -14:20.007 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:52732 latency=880.91µs -14:20.008 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:52732 latency=354.376µs -14:20.009 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:52732 latency=439.496µs -14:20.027 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/8b28bbb2-451b-0679-edc5-06d3d1caf01a from=127.0.0.1:52734 latency=218.217µs -14:20.032 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/6519149b-152c-93e2-f3e7-01fae7c950ad from=127.0.0.1:52736 latency=388.043µs -14:20.038 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/6519149b-152c-93e2-f3e7-01fae7c950ad from=127.0.0.1:52736 latency=5.253176ms -14:20.044 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/6519149b-152c-93e2-f3e7-01fae7c950ad from=127.0.0.1:52732 latency=407.507µs -14:20.047 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/6519149b-152c-93e2-f3e7-01fae7c950ad from=127.0.0.1:52740 latency=315.242µs -14:20.063 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/6519149b-152c-93e2-f3e7-01fae7c950ad from=127.0.0.1:52740 latency=648.211µs -14:20.067 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/6519149b-152c-93e2-f3e7-01fae7c950ad from=127.0.0.1:52732 latency=536.33µs -14:20.070 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/6519149b-152c-93e2-f3e7-01fae7c950ad from=127.0.0.1:52744 latency=483.086µs -14:20.072 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/6519149b-152c-93e2-f3e7-01fae7c950ad from=127.0.0.1:52744 latency=591.409µs -14:20.073 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/6519149b-152c-93e2-f3e7-01fae7c950ad from=127.0.0.1:52732 latency=353.541µs -14:20.086 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/6519149b-152c-93e2-f3e7-01fae7c950ad from=127.0.0.1:52746 latency=1.480399ms -14:20.095 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/6519149b-152c-93e2-f3e7-01fae7c950ad from=127.0.0.1:52746 latency=1.077664ms -14:20.102 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/6519149b-152c-93e2-f3e7-01fae7c950ad from=127.0.0.1:52732 latency=307.839µs -14:20.108 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/6519149b-152c-93e2-f3e7-01fae7c950ad from=127.0.0.1:52748 latency=438.122µs -14:20.109 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/6519149b-152c-93e2-f3e7-01fae7c950ad from=127.0.0.1:52748 latency=679.595µs -14:20.114 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/6519149b-152c-93e2-f3e7-01fae7c950ad from=127.0.0.1:52732 latency=4.674748ms -14:20.118 [INFO] TestAgent: Requesting shutdown -14:20.118 [INFO] TestAgent.server: shutting down server -14:20.118 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:20.118 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:20.118 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:20.118 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:20.118 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:20.118 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:20.118 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:20.118 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:20.118 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:20.118 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:20.118 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:20.118 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:20.118 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:20.118 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:20.118 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:20.118Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:20.118Z [INFO] agent.router.manager: shutting down -14:20.118 [INFO] TestAgent: consul server down -14:20.118 [INFO] TestAgent: shutdown complete -14:20.118 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30435 network=tcp -14:20.118 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30435 network=udp -14:20.118 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30436 network=tcp -14:20.621 [INFO] TestAgent: Waiting for endpoints to shut down -14:20.621 [INFO] TestAgent: Endpoints down ---- PASS: TestRoleUpdateCommand (1.33s) - --- PASS: TestRoleUpdateCommand/update_a_role_that_does_not_exist (0.02s) - --- PASS: TestRoleUpdateCommand/update_with_policy_by_name (0.01s) - --- PASS: TestRoleUpdateCommand/update_with_policy_by_id (0.02s) - --- PASS: TestRoleUpdateCommand/update_with_service_identity (0.01s) - --- PASS: TestRoleUpdateCommand/update_with_service_identity_scoped_to_2_DCs (0.03s) - --- PASS: TestRoleUpdateCommand/update_with_node_identity (0.01s) + --- PASS: TestRoleUpdateCommand_noMerge/update_with_service_identity (0.02s) + --- PASS: TestRoleUpdateCommand_noMerge/update_with_service_identity_scoped_to_2_DCs (0.01s) +2022-12-04T05:49:05.909Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:05.949Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:05.952 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dc453129-9a09-00e2-938d-25e16390b82e Address:127.0.0.1:19202}]" +49:05.952 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19202 [Follower]" leader= +49:05.952 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dc453129-9a09-00e2-938d-25e16390b82e.dc1 127.0.0.1 +49:05.955 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dc453129-9a09-00e2-938d-25e16390b82e 127.0.0.1 +2022-12-04T05:49:05.955Z [INFO] agent.router: Initializing LAN area manager +49:05.958 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dc453129-9a09-00e2-938d-25e16390b82e.dc1 area=wan +49:05.958 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=tcp +49:05.958 [INFO] TestAgent.server: Adding LAN server: server="Node-dc453129-9a09-00e2-938d-25e16390b82e (Addr: tcp/127.0.0.1:19202) (DC: dc1)" +49:05.958 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=udp +49:05.959 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19198 network=tcp +49:05.960 [INFO] TestAgent: started state syncer +49:05.960 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19203 network=tcp +49:05.994 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:05.994 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19202 [Candidate]" term=2 +49:05.994 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:05.994 [DEBUG] TestAgent.server.raft: vote granted: from=dc453129-9a09-00e2-938d-25e16390b82e term=2 tally=1 +49:05.994 [INFO] TestAgent.server.raft: election won: tally=1 +49:05.994 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19202 [Leader]" +49:05.995 [INFO] TestAgent.server: cluster leadership acquired +49:05.995 [INFO] TestAgent.server: initializing acls +49:05.995 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:05.995 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:05.995 [INFO] TestAgent.server: New leader elected: payload=Node-dc453129-9a09-00e2-938d-25e16390b82e +49:05.995 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:05.996 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:05.996 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:05.996 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:05.997 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-dc453129-9a09-00e2-938d-25e16390b82e +49:05.997 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-dc453129-9a09-00e2-938d-25e16390b82e.dc1 +49:05.997 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:06.000 [INFO] TestAgent.server: Updating LAN server: server="Node-dc453129-9a09-00e2-938d-25e16390b82e (Addr: tcp/127.0.0.1:19202) (DC: dc1)" +49:06.000 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-dc453129-9a09-00e2-938d-25e16390b82e.dc1 area=wan +49:06.028 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:06.028 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:06.028 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:06.028 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:06.028 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:06.028 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dc453129-9a09-00e2-938d-25e16390b82e +49:06.028 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dc453129-9a09-00e2-938d-25e16390b82e +49:06.029 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dc453129-9a09-00e2-938d-25e16390b82e +49:06.030 [INFO] TestAgent.server: federation state anti-entropy synced +49:06.096 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:06.097 [INFO] TestAgent: Synced node info +49:06.097 [DEBUG] TestAgent: Node info in sync +49:06.376 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:51390 latency=1.021827ms +49:06.377 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:51390 latency=488.318µs +49:06.377 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:51390 latency=456.305µs +49:06.380 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/10dd9b21-958c-9ed9-6ffd-75864452bb52 from=127.0.0.1:51392 latency=144.155µs +49:06.385 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/fc00261d-d39d-fc9d-f09b-7b98101a9b29 from=127.0.0.1:51394 latency=281.317µs +49:06.386 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/fc00261d-d39d-fc9d-f09b-7b98101a9b29 from=127.0.0.1:51394 latency=576.754µs +49:06.386 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/fc00261d-d39d-fc9d-f09b-7b98101a9b29 from=127.0.0.1:51390 latency=145.009µs +49:06.388 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/fc00261d-d39d-fc9d-f09b-7b98101a9b29 from=127.0.0.1:51396 latency=232.984µs +49:06.390 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/fc00261d-d39d-fc9d-f09b-7b98101a9b29 from=127.0.0.1:51396 latency=660.11µs +49:06.390 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/fc00261d-d39d-fc9d-f09b-7b98101a9b29 from=127.0.0.1:51390 latency=206.684µs +49:06.392 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/fc00261d-d39d-fc9d-f09b-7b98101a9b29 from=127.0.0.1:51398 latency=266.345µs +49:06.393 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/fc00261d-d39d-fc9d-f09b-7b98101a9b29 from=127.0.0.1:51398 latency=535.477µs +49:06.394 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/fc00261d-d39d-fc9d-f09b-7b98101a9b29 from=127.0.0.1:51390 latency=174.539µs +49:06.395 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/fc00261d-d39d-fc9d-f09b-7b98101a9b29 from=127.0.0.1:51400 latency=238.595µs +49:06.397 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/fc00261d-d39d-fc9d-f09b-7b98101a9b29 from=127.0.0.1:51400 latency=685.791µs +49:06.398 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/fc00261d-d39d-fc9d-f09b-7b98101a9b29 from=127.0.0.1:51390 latency=239.565µs +49:06.399 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/fc00261d-d39d-fc9d-f09b-7b98101a9b29 from=127.0.0.1:51404 latency=264.133µs +49:06.400 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/fc00261d-d39d-fc9d-f09b-7b98101a9b29 from=127.0.0.1:51404 latency=738.496µs +49:06.401 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/fc00261d-d39d-fc9d-f09b-7b98101a9b29 from=127.0.0.1:51390 latency=187.201µs +49:06.402 [INFO] TestAgent: Requesting shutdown +49:06.402 [INFO] TestAgent.server: shutting down server +49:06.402 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:06.402 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:06.402 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:06.402 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:06.402 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:06.402 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:06.402 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:06.402 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:06.402 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:06.402 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:06.402 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:06.402 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:06.402 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:06.402 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:06.402 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:06.402Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:06.402Z [INFO] agent.router.manager: shutting down +49:06.402 [INFO] TestAgent: consul server down +49:06.402 [INFO] TestAgent: shutdown complete +49:06.402 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=tcp +49:06.402 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=udp +49:06.402 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19198 network=tcp +49:06.904 [INFO] TestAgent: Waiting for endpoints to shut down +49:06.904 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleUpdateCommand (1.06s) + --- PASS: TestRoleUpdateCommand/update_a_role_that_does_not_exist (0.01s) + --- PASS: TestRoleUpdateCommand/update_with_policy_by_name (0.00s) + --- PASS: TestRoleUpdateCommand/update_with_policy_by_id (0.00s) + --- PASS: TestRoleUpdateCommand/update_with_service_identity (0.00s) + --- PASS: TestRoleUpdateCommand/update_with_service_identity_scoped_to_2_DCs (0.00s) + --- PASS: TestRoleUpdateCommand/update_with_node_identity (0.00s) +2022-12-04T05:49:05.912Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:05.962Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:05.965 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:29e454ed-d42d-620f-7357-1a0b7d0051eb Address:127.0.0.1:19209}]" +49:05.965 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-29e454ed-d42d-620f-7357-1a0b7d0051eb.dc1 127.0.0.1 +49:05.966 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-29e454ed-d42d-620f-7357-1a0b7d0051eb 127.0.0.1 +2022-12-04T05:49:05.966Z [INFO] agent.router: Initializing LAN area manager +49:05.966 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19209 [Follower]" leader= +49:05.966 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19204 network=udp +49:05.966 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19204 network=tcp +49:05.966 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-29e454ed-d42d-620f-7357-1a0b7d0051eb.dc1 area=wan +49:05.967 [INFO] TestAgent.server: Adding LAN server: server="Node-29e454ed-d42d-620f-7357-1a0b7d0051eb (Addr: tcp/127.0.0.1:19209) (DC: dc1)" +49:05.967 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19205 network=tcp +49:05.967 [INFO] TestAgent: started state syncer +49:05.967 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19210 network=tcp +49:06.021 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:06.031 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:06.031 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19209 [Candidate]" term=2 +49:06.031 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:06.031 [DEBUG] TestAgent.server.raft: vote granted: from=29e454ed-d42d-620f-7357-1a0b7d0051eb term=2 tally=1 +49:06.031 [INFO] TestAgent.server.raft: election won: tally=1 +49:06.031 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19209 [Leader]" +49:06.031 [INFO] TestAgent.server: cluster leadership acquired +49:06.031 [INFO] TestAgent.server: initializing acls +49:06.032 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:06.032 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:06.032 [INFO] TestAgent.server: New leader elected: payload=Node-29e454ed-d42d-620f-7357-1a0b7d0051eb +49:06.032 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:06.032 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:06.032 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:06.032 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:06.032 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-29e454ed-d42d-620f-7357-1a0b7d0051eb +49:06.033 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-29e454ed-d42d-620f-7357-1a0b7d0051eb.dc1 +49:06.033 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:06.035 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:06.035 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:06.035 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:06.035 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:06.035 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:06.035 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-29e454ed-d42d-620f-7357-1a0b7d0051eb +49:06.035 [INFO] TestAgent.server: member joined, marking health alive: member=Node-29e454ed-d42d-620f-7357-1a0b7d0051eb +49:06.035 [INFO] TestAgent.server: Updating LAN server: server="Node-29e454ed-d42d-620f-7357-1a0b7d0051eb (Addr: tcp/127.0.0.1:19209) (DC: dc1)" +49:06.035 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-29e454ed-d42d-620f-7357-1a0b7d0051eb +49:06.035 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-29e454ed-d42d-620f-7357-1a0b7d0051eb.dc1 area=wan +49:06.036 [INFO] TestAgent.server: federation state anti-entropy synced +49:06.280 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:06.280 [INFO] TestAgent: Synced node info +49:06.404 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:48276 latency=1.142699ms +49:06.405 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role from=127.0.0.1:48276 latency=729.586µs +49:06.406 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/617db117-ca7c-5ff7-1bb3-602190ce89e1 from=127.0.0.1:48278 latency=121.874µs +49:06.408 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/role/90f728b8-e7ff-75eb-738a-969d9d6fce41 from=127.0.0.1:48280 latency=261.035µs +49:06.413 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/role/90f728b8-e7ff-75eb-738a-969d9d6fce41 from=127.0.0.1:48280 latency=1.090691ms +49:06.414 [INFO] TestAgent: Requesting shutdown +49:06.414 [INFO] TestAgent.server: shutting down server +49:06.414 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:06.414 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:06.415 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:06.415 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:06.415 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:06.415 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:06.415 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:06.415 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:06.415 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:06.415 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:06.415 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:06.415 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:06.415 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:06.415 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:06.415 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:06.415Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:06.415Z [INFO] agent.router.manager: shutting down +49:06.415 [INFO] TestAgent: consul server down +49:06.415 [INFO] TestAgent: shutdown complete +49:06.415 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19204 network=tcp +49:06.415 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19204 network=udp +49:06.415 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19205 network=tcp +49:06.916 [INFO] TestAgent: Waiting for endpoints to shut down +49:06.916 [INFO] TestAgent: Endpoints down +--- PASS: TestRoleUpdateCommand_JSON (1.07s) + --- PASS: TestRoleUpdateCommand_JSON/update_a_role_that_does_not_exist (0.00s) + --- PASS: TestRoleUpdateCommand_JSON/update_with_policy_by_name (0.01s) PASS -ok github.com/hashicorp/consul/command/acl/role/update 2.072s +ok github.com/hashicorp/consul/command/acl/role/update 1.247s === RUN TestRulesTranslateCommand_noTabs === PAUSE TestRulesTranslateCommand_noTabs === RUN TestRulesTranslateCommand === PAUSE TestRulesTranslateCommand === CONT TestRulesTranslateCommand_noTabs ---- PASS: TestRulesTranslateCommand_noTabs (0.00s) === CONT TestRulesTranslateCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestRulesTranslateCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestRulesTranslateCommand/file === RUN TestRulesTranslateCommand/stdin === RUN TestRulesTranslateCommand/arg === RUN TestRulesTranslateCommand/exclusive-options -2021-10-31T23:14:27.282Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:27.341Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:27.348 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:77b603f8-549d-a566-10d7-654567c2a5ba Address:127.0.0.1:30426}]" -14:27.349 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-77b603f8-549d-a566-10d7-654567c2a5ba.dc1 127.0.0.1 -14:27.349 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-77b603f8-549d-a566-10d7-654567c2a5ba 127.0.0.1 -2021-10-31T23:14:27.349Z [INFO] agent.router: Initializing LAN area manager -14:27.350 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= -14:27.350 [INFO] TestAgent.server: Adding LAN server: server="Node-77b603f8-549d-a566-10d7-654567c2a5ba (Addr: tcp/127.0.0.1:30426) (DC: dc1)" -14:27.350 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-77b603f8-549d-a566-10d7-654567c2a5ba.dc1 area=wan -14:27.350 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp -14:27.354 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp -14:27.354 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp -14:27.354 [INFO] TestAgent: started state syncer -14:27.355 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp -14:27.402 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -14:27.417 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:27.417 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 -14:27.418 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:27.418 [DEBUG] TestAgent.server.raft: vote granted: from=77b603f8-549d-a566-10d7-654567c2a5ba term=2 tally=1 -14:27.418 [INFO] TestAgent.server.raft: election won: tally=1 -14:27.418 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" -14:27.418 [INFO] TestAgent.server: cluster leadership acquired -14:27.418 [INFO] TestAgent.server: initializing acls -14:27.418 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:27.418 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:27.419 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:27.419 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:27.419 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:27.419 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:27.419 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-77b603f8-549d-a566-10d7-654567c2a5ba -14:27.419 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-77b603f8-549d-a566-10d7-654567c2a5ba.dc1 -14:27.420 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:27.421 [INFO] TestAgent.server: New leader elected: payload=Node-77b603f8-549d-a566-10d7-654567c2a5ba -14:27.422 [INFO] TestAgent.server: Updating LAN server: server="Node-77b603f8-549d-a566-10d7-654567c2a5ba (Addr: tcp/127.0.0.1:30426) (DC: dc1)" -14:27.422 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-77b603f8-549d-a566-10d7-654567c2a5ba.dc1 area=wan -14:27.466 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:27.466 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:27.466 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:27.466 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:27.466 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:27.466 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-77b603f8-549d-a566-10d7-654567c2a5ba -14:27.466 [INFO] TestAgent.server: member joined, marking health alive: member=Node-77b603f8-549d-a566-10d7-654567c2a5ba -14:27.467 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-77b603f8-549d-a566-10d7-654567c2a5ba -14:27.468 [INFO] TestAgent.server: federation state anti-entropy synced -14:27.475 [INFO] TestAgent: Requesting shutdown -14:27.475 [INFO] TestAgent.server: shutting down server -14:27.475 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:27.475 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:27.475 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:27.475 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:27.475 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:27.475 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:27.475 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:27.475 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:27.475 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:27.475 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -14:27.475 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:27.475 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:27.475 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:27.475 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:27.475 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:27.475 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:27.475Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:27.475Z [INFO] agent.router.manager: shutting down -14:27.475 [INFO] TestAgent: consul server down -14:27.475 [INFO] TestAgent: shutdown complete -14:27.475 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp -14:27.475 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp -14:27.475 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp -14:27.977 [INFO] TestAgent: Waiting for endpoints to shut down -14:27.982 [INFO] TestAgent: Endpoints down ---- PASS: TestRulesTranslateCommand (0.83s) +2022-12-04T05:49:06.199Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:06.213Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:06.215 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:92f44a8c-7371-ded9-bb8e-8e711fddda19 Address:127.0.0.1:13069}]" +49:06.215 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= +49:06.215 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-92f44a8c-7371-ded9-bb8e-8e711fddda19.dc1 127.0.0.1 +49:06.216 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-92f44a8c-7371-ded9-bb8e-8e711fddda19 127.0.0.1 +2022-12-04T05:49:06.216Z [INFO] agent.router: Initializing LAN area manager +49:06.216 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-92f44a8c-7371-ded9-bb8e-8e711fddda19.dc1 area=wan +49:06.216 [INFO] TestAgent.server: Adding LAN server: server="Node-92f44a8c-7371-ded9-bb8e-8e711fddda19 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" +49:06.216 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp +49:06.216 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp +49:06.217 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp +49:06.217 [INFO] TestAgent: started state syncer +49:06.217 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp +49:06.266 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:06.273 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:06.273 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 +49:06.273 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:06.273 [DEBUG] TestAgent.server.raft: vote granted: from=92f44a8c-7371-ded9-bb8e-8e711fddda19 term=2 tally=1 +49:06.273 [INFO] TestAgent.server.raft: election won: tally=1 +49:06.273 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" +49:06.273 [INFO] TestAgent.server: cluster leadership acquired +49:06.274 [INFO] TestAgent.server: initializing acls +49:06.274 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:06.274 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:06.274 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:06.275 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:06.275 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:06.275 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:06.275 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-92f44a8c-7371-ded9-bb8e-8e711fddda19 +49:06.275 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-92f44a8c-7371-ded9-bb8e-8e711fddda19.dc1 +49:06.275 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:06.276 [INFO] TestAgent.server: New leader elected: payload=Node-92f44a8c-7371-ded9-bb8e-8e711fddda19 +49:06.276 [INFO] TestAgent.server: Updating LAN server: server="Node-92f44a8c-7371-ded9-bb8e-8e711fddda19 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" +49:06.276 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-92f44a8c-7371-ded9-bb8e-8e711fddda19.dc1 area=wan +49:06.292 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:06.292 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:06.292 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:06.292 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:06.292 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:06.292 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-92f44a8c-7371-ded9-bb8e-8e711fddda19 +49:06.292 [INFO] TestAgent.server: member joined, marking health alive: member=Node-92f44a8c-7371-ded9-bb8e-8e711fddda19 +49:06.293 [INFO] TestAgent.server: federation state anti-entropy synced +49:06.294 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-92f44a8c-7371-ded9-bb8e-8e711fddda19 +49:06.301 [INFO] TestAgent: Requesting shutdown +49:06.301 [INFO] TestAgent.server: shutting down server +49:06.301 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:06.301 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:06.301 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:06.301 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:06.301 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:06.301 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:06.301 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:06.301 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:06.301 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:06.301 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:06.301 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:06.301 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:06.301 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:06.301 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:06.301 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:06.301 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:06.302Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:06.302Z [INFO] agent.router.manager: shutting down +49:06.302 [INFO] TestAgent: consul server down +49:06.302 [INFO] TestAgent: shutdown complete +49:06.302 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp +49:06.302 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp +49:06.302 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp +49:06.804 [INFO] TestAgent: Waiting for endpoints to shut down +49:06.804 [INFO] TestAgent: Endpoints down +--- PASS: TestRulesTranslateCommand (0.65s) --- PASS: TestRulesTranslateCommand/file (0.00s) --- PASS: TestRulesTranslateCommand/stdin (0.00s) --- PASS: TestRulesTranslateCommand/arg (0.00s) --- PASS: TestRulesTranslateCommand/exclusive-options (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/rules 1.880s +ok github.com/hashicorp/consul/command/acl/rules 0.754s === RUN TestFormatToken === RUN TestFormatToken/legacy === RUN TestFormatToken/legacy/pretty === RUN TestFormatToken/legacy/pretty-meta === RUN TestFormatToken/legacy/json === RUN TestFormatToken/basic +=== RUN TestFormatToken/basic/json === RUN TestFormatToken/basic/pretty === RUN TestFormatToken/basic/pretty-meta -=== RUN TestFormatToken/basic/json === RUN TestFormatToken/complex +=== RUN TestFormatToken/complex/json === RUN TestFormatToken/complex/pretty === RUN TestFormatToken/complex/pretty-meta -=== RUN TestFormatToken/complex/json ---- PASS: TestFormatToken (0.02s) +--- PASS: TestFormatToken (0.00s) --- PASS: TestFormatToken/legacy (0.00s) --- PASS: TestFormatToken/legacy/pretty (0.00s) --- PASS: TestFormatToken/legacy/pretty-meta (0.00s) --- PASS: TestFormatToken/legacy/json (0.00s) - --- PASS: TestFormatToken/basic (0.01s) + --- PASS: TestFormatToken/basic (0.00s) + --- PASS: TestFormatToken/basic/json (0.00s) --- PASS: TestFormatToken/basic/pretty (0.00s) --- PASS: TestFormatToken/basic/pretty-meta (0.00s) - --- PASS: TestFormatToken/basic/json (0.00s) --- PASS: TestFormatToken/complex (0.00s) + --- PASS: TestFormatToken/complex/json (0.00s) --- PASS: TestFormatToken/complex/pretty (0.00s) --- PASS: TestFormatToken/complex/pretty-meta (0.00s) - --- PASS: TestFormatToken/complex/json (0.00s) === RUN TestFormatTokenList +=== RUN TestFormatTokenList/basic +=== RUN TestFormatTokenList/basic/pretty +=== RUN TestFormatTokenList/basic/pretty-meta +=== RUN TestFormatTokenList/basic/json === RUN TestFormatTokenList/legacy +=== RUN TestFormatTokenList/legacy/pretty === RUN TestFormatTokenList/legacy/pretty-meta === RUN TestFormatTokenList/legacy/json -=== RUN TestFormatTokenList/legacy/pretty === RUN TestFormatTokenList/complex === RUN TestFormatTokenList/complex/pretty === RUN TestFormatTokenList/complex/pretty-meta === RUN TestFormatTokenList/complex/json -=== RUN TestFormatTokenList/basic -=== RUN TestFormatTokenList/basic/pretty-meta -=== RUN TestFormatTokenList/basic/json -=== RUN TestFormatTokenList/basic/pretty ---- PASS: TestFormatTokenList (0.03s) - --- PASS: TestFormatTokenList/legacy (0.02s) - --- PASS: TestFormatTokenList/legacy/pretty-meta (0.01s) - --- PASS: TestFormatTokenList/legacy/json (0.00s) - --- PASS: TestFormatTokenList/legacy/pretty (0.01s) - --- PASS: TestFormatTokenList/complex (0.01s) - --- PASS: TestFormatTokenList/complex/pretty (0.00s) - --- PASS: TestFormatTokenList/complex/pretty-meta (0.01s) - --- PASS: TestFormatTokenList/complex/json (0.00s) +--- PASS: TestFormatTokenList (0.00s) --- PASS: TestFormatTokenList/basic (0.00s) + --- PASS: TestFormatTokenList/basic/pretty (0.00s) --- PASS: TestFormatTokenList/basic/pretty-meta (0.00s) --- PASS: TestFormatTokenList/basic/json (0.00s) - --- PASS: TestFormatTokenList/basic/pretty (0.00s) + --- PASS: TestFormatTokenList/legacy (0.00s) + --- PASS: TestFormatTokenList/legacy/pretty (0.00s) + --- PASS: TestFormatTokenList/legacy/pretty-meta (0.00s) + --- PASS: TestFormatTokenList/legacy/json (0.00s) + --- PASS: TestFormatTokenList/complex (0.00s) + --- PASS: TestFormatTokenList/complex/pretty (0.00s) + --- PASS: TestFormatTokenList/complex/pretty-meta (0.00s) + --- PASS: TestFormatTokenList/complex/json (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/token 0.079s +ok github.com/hashicorp/consul/command/acl/token 0.009s === RUN TestTokenCloneCommand_noTabs === PAUSE TestTokenCloneCommand_noTabs === RUN TestTokenCloneCommand_Pretty @@ -76767,185 +76955,178 @@ === RUN TestTokenCloneCommand_JSON === PAUSE TestTokenCloneCommand_JSON === CONT TestTokenCloneCommand_noTabs ---- PASS: TestTokenCloneCommand_noTabs (0.00s) === CONT TestTokenCloneCommand_JSON === CONT TestTokenCloneCommand_Pretty [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestTokenCloneCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestTokenCloneCommand_Pretty/Description === RUN TestTokenCloneCommand_Pretty/Without_Description === RUN TestTokenCloneCommand_JSON/Description === RUN TestTokenCloneCommand_JSON/Without_Description -2021-10-31T23:14:29.544Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:29.634Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:29.636 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7dea28b9-0bf7-6d92-2aff-e9d47aa59936 Address:127.0.0.1:10013}]" -14:29.637 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936.dc1 127.0.0.1 -14:29.637 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936 127.0.0.1 -2021-10-31T23:14:29.637Z [INFO] agent.router: Initializing LAN area manager -14:29.637 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10013 [Follower]" leader= -14:29.638 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10008 network=udp -14:29.638 [INFO] TestAgent.server: Adding LAN server: server="Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936 (Addr: tcp/127.0.0.1:10013) (DC: dc1)" -14:29.638 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936.dc1 area=wan -14:29.638 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10008 network=tcp -14:29.639 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10009 network=tcp -14:29.639 [INFO] TestAgent: started state syncer -14:29.639 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10014 network=tcp -14:29.685 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:29.685 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10013 [Candidate]" term=2 -14:29.686 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:29.686 [DEBUG] TestAgent.server.raft: vote granted: from=7dea28b9-0bf7-6d92-2aff-e9d47aa59936 term=2 tally=1 -14:29.686 [INFO] TestAgent.server.raft: election won: tally=1 -14:29.686 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10013 [Leader]" -14:29.686 [INFO] TestAgent.server: cluster leadership acquired -14:29.686 [INFO] TestAgent.server: initializing acls -14:29.686 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:29.686 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:29.687 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:29.701 [INFO] TestAgent.server: New leader elected: payload=Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936 -14:29.701 [INFO] TestAgent.server: initializing acls -14:29.701 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:29.702 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:29.702 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:29.702 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:29.703 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936 -14:29.703 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936.dc1 -14:29.703 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:29.704 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:29.704 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -14:29.704 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936 -14:29.704 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936.dc1 -14:29.704 [INFO] TestAgent.server: Updating LAN server: server="Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936 (Addr: tcp/127.0.0.1:10013) (DC: dc1)" -14:29.704 [INFO] TestAgent.server: Updating LAN server: server="Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936 (Addr: tcp/127.0.0.1:10013) (DC: dc1)" -14:29.704 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936.dc1 area=wan -14:29.704 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936.dc1 area=wan -14:29.755 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:29.755 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:29.755 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:29.755 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:29.755 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:29.755 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936 -14:29.755 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936 -14:29.756 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936 -14:29.756 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7dea28b9-0bf7-6d92-2aff-e9d47aa59936 -14:29.757 [INFO] TestAgent.server: federation state anti-entropy synced -14:29.787 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:48584 latency=1.171864ms -14:29.790 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:48584 latency=1.275526ms -14:29.796 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/fbe588c3-8f5f-38b5-a0ee-d82328b75ab9/clone from=127.0.0.1:48586 latency=942.567µs -14:29.802 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/e7af3855-9943-1dc2-35b7-46c7ce428e6b from=127.0.0.1:48584 latency=735.007µs -14:29.806 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/fbe588c3-8f5f-38b5-a0ee-d82328b75ab9/clone from=127.0.0.1:48588 latency=1.35757ms -14:29.814 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/56a35c9d-c848-2764-5946-56d244a1e043 from=127.0.0.1:48584 latency=463.849µs -14:29.815 [INFO] TestAgent: Requesting shutdown -14:29.815 [INFO] TestAgent.server: shutting down server -14:29.815 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:29.815 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:29.815 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:29.815 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:29.815 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:29.815 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:29.815 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:29.815 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:29.815 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:29.815 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -14:29.815 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:29.815 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:29.815 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:29.815 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:29.815 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:29.815 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:29.815Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:29.815Z [INFO] agent.router.manager: shutting down -14:29.815 [INFO] TestAgent: consul server down -14:29.815 [INFO] TestAgent: shutdown complete -14:29.815 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10008 network=tcp -14:29.815 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10008 network=udp -14:29.815 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10009 network=tcp -14:30.317 [INFO] TestAgent: Waiting for endpoints to shut down -14:30.317 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenCloneCommand_Pretty (0.95s) - --- PASS: TestTokenCloneCommand_Pretty/Description (0.01s) - --- PASS: TestTokenCloneCommand_Pretty/Without_Description (0.01s) -2021-10-31T23:14:29.574Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:29.598Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:29.620 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0b29d097-963e-7bb7-590d-d41ee6c9af17 Address:127.0.0.1:10006}]" -14:29.621 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0b29d097-963e-7bb7-590d-d41ee6c9af17.dc1 127.0.0.1 -14:29.622 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10006 [Follower]" leader= -14:29.626 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0b29d097-963e-7bb7-590d-d41ee6c9af17 127.0.0.1 -2021-10-31T23:14:29.626Z [INFO] agent.router: Initializing LAN area manager -14:29.627 [INFO] TestAgent.server: Adding LAN server: server="Node-0b29d097-963e-7bb7-590d-d41ee6c9af17 (Addr: tcp/127.0.0.1:10006) (DC: dc1)" -14:29.627 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0b29d097-963e-7bb7-590d-d41ee6c9af17.dc1 area=wan -14:29.627 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=udp -14:29.627 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=tcp -14:29.627 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10002 network=tcp -14:29.627 [INFO] TestAgent: started state syncer -14:29.628 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10007 network=tcp -14:29.678 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -14:29.678 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:29.678 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10006 [Candidate]" term=2 -14:29.678 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:29.678 [DEBUG] TestAgent.server.raft: vote granted: from=0b29d097-963e-7bb7-590d-d41ee6c9af17 term=2 tally=1 -14:29.678 [INFO] TestAgent.server.raft: election won: tally=1 -14:29.678 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10006 [Leader]" -14:29.678 [INFO] TestAgent.server: cluster leadership acquired -14:29.678 [INFO] TestAgent.server: initializing acls -14:29.679 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:29.679 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:29.679 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:29.679 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:29.679 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:29.679 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:29.679 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0b29d097-963e-7bb7-590d-d41ee6c9af17 -14:29.680 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0b29d097-963e-7bb7-590d-d41ee6c9af17.dc1 -14:29.680 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:29.681 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0b29d097-963e-7bb7-590d-d41ee6c9af17.dc1 area=wan -14:29.681 [INFO] TestAgent.server: New leader elected: payload=Node-0b29d097-963e-7bb7-590d-d41ee6c9af17 -14:29.682 [INFO] TestAgent.server: Updating LAN server: server="Node-0b29d097-963e-7bb7-590d-d41ee6c9af17 (Addr: tcp/127.0.0.1:10006) (DC: dc1)" -14:29.739 [INFO] TestAgent: Synced node info -14:29.739 [DEBUG] TestAgent: Node info in sync -14:29.759 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:29.759 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:29.759 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:29.759 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:29.759 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:29.759 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0b29d097-963e-7bb7-590d-d41ee6c9af17 -14:29.759 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0b29d097-963e-7bb7-590d-d41ee6c9af17 -14:29.760 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0b29d097-963e-7bb7-590d-d41ee6c9af17 -14:29.760 [INFO] TestAgent.server: federation state anti-entropy synced -14:29.835 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:55806 latency=1.193745ms -14:29.836 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:55806 latency=666.633µs -14:29.839 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/86e58097-28b8-149d-d37d-55df92b5c060/clone from=127.0.0.1:55808 latency=784.553µs -14:29.841 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/86e58097-28b8-149d-d37d-55df92b5c060/clone from=127.0.0.1:55810 latency=710.591µs -14:29.841 [INFO] TestAgent: Requesting shutdown -14:29.841 [INFO] TestAgent.server: shutting down server -14:29.841 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:29.841 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:29.841 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:29.841 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:29.841 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:29.841 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:29.841 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:29.841 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:29.842 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:29.842 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:29.842 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:29.842 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:29.845 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:29.845 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:29.846 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:29.846Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:29.846Z [INFO] agent.router.manager: shutting down -14:29.846 [INFO] TestAgent: consul server down -14:29.846 [INFO] TestAgent: shutdown complete -14:29.846 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=tcp -14:29.846 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=udp -14:29.846 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10002 network=tcp -14:30.349 [INFO] TestAgent: Waiting for endpoints to shut down -14:30.349 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenCloneCommand_JSON (0.97s) - --- PASS: TestTokenCloneCommand_JSON/Description (0.00s) - --- PASS: TestTokenCloneCommand_JSON/Without_Description (0.00s) +2022-12-04T05:49:13.081Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:13.099Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:13.110 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0d5805d8-ee2f-bc95-edfb-587ba39c7399 Address:127.0.0.1:30433}]" +49:13.110 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30433 [Follower]" leader= +49:13.111 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0d5805d8-ee2f-bc95-edfb-587ba39c7399.dc1 127.0.0.1 +49:13.125 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0d5805d8-ee2f-bc95-edfb-587ba39c7399 127.0.0.1 +2022-12-04T05:49:13.125Z [INFO] agent.router: Initializing LAN area manager +49:13.125 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=udp +49:13.125 [INFO] TestAgent.server: Adding LAN server: server="Node-0d5805d8-ee2f-bc95-edfb-587ba39c7399 (Addr: tcp/127.0.0.1:30433) (DC: dc1)" +49:13.125 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0d5805d8-ee2f-bc95-edfb-587ba39c7399.dc1 area=wan +49:13.125 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=tcp +49:13.126 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30429 network=tcp +49:13.126 [INFO] TestAgent: started state syncer +49:13.126 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30434 network=tcp +49:13.176 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:13.176 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:13.176 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30433 [Candidate]" term=2 +49:13.176 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:13.176 [DEBUG] TestAgent.server.raft: vote granted: from=0d5805d8-ee2f-bc95-edfb-587ba39c7399 term=2 tally=1 +49:13.176 [INFO] TestAgent.server.raft: election won: tally=1 +49:13.176 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30433 [Leader]" +49:13.176 [INFO] TestAgent.server: cluster leadership acquired +49:13.177 [INFO] TestAgent.server: initializing acls +49:13.177 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:13.177 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:13.177 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:13.177 [INFO] TestAgent.server: New leader elected: payload=Node-0d5805d8-ee2f-bc95-edfb-587ba39c7399 +49:13.178 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:13.178 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:13.178 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:13.178 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0d5805d8-ee2f-bc95-edfb-587ba39c7399 +49:13.178 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0d5805d8-ee2f-bc95-edfb-587ba39c7399.dc1 +49:13.179 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:13.180 [INFO] TestAgent.server: Updating LAN server: server="Node-0d5805d8-ee2f-bc95-edfb-587ba39c7399 (Addr: tcp/127.0.0.1:30433) (DC: dc1)" +49:13.180 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0d5805d8-ee2f-bc95-edfb-587ba39c7399.dc1 area=wan +49:13.252 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:13.252 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:13.252 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:13.252 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:13.252 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:13.252 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d5805d8-ee2f-bc95-edfb-587ba39c7399 +49:13.252 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0d5805d8-ee2f-bc95-edfb-587ba39c7399 +49:13.253 [INFO] TestAgent.server: federation state anti-entropy synced +49:13.269 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d5805d8-ee2f-bc95-edfb-587ba39c7399 +49:13.347 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:33374 latency=1.680733ms +49:13.353 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:33374 latency=5.290185ms +49:13.358 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/acd95c9e-0ed2-28e7-a0bd-9a8300bea60e/clone from=127.0.0.1:33378 latency=881.8µs +49:13.360 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/759e6132-23bf-4e29-f8eb-728acaf6842b from=127.0.0.1:33374 latency=218.185µs +49:13.365 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/acd95c9e-0ed2-28e7-a0bd-9a8300bea60e/clone from=127.0.0.1:33380 latency=750.141µs +49:13.377 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/723e5c06-35ad-13f0-bd22-fbd2bcb4ddce from=127.0.0.1:33374 latency=382.793µs +49:13.378 [INFO] TestAgent: Requesting shutdown +49:13.378 [INFO] TestAgent.server: shutting down server +49:13.378 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:13.378 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:13.378 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:13.378 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:13.378 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:13.378 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:13.378 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:13.378 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:13.378 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:13.378 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:13.378 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:13.378 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:13.378 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:13.378 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:13.378 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:13.378 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:13.378Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:13.378Z [INFO] agent.router.manager: shutting down +49:13.378 [INFO] TestAgent: consul server down +49:13.378 [INFO] TestAgent: shutdown complete +49:13.378 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=tcp +49:13.378 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=udp +49:13.378 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30429 network=tcp +49:13.888 [INFO] TestAgent: Waiting for endpoints to shut down +49:13.888 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenCloneCommand_Pretty (0.87s) + --- PASS: TestTokenCloneCommand_Pretty/Description (0.00s) + --- PASS: TestTokenCloneCommand_Pretty/Without_Description (0.02s) +2022-12-04T05:49:13.084Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:13.112Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:13.113 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f8f197cd-a080-2945-a267-5b54d8ae1971 Address:127.0.0.1:30426}]" +49:13.114 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f8f197cd-a080-2945-a267-5b54d8ae1971.dc1 127.0.0.1 +49:13.114 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f8f197cd-a080-2945-a267-5b54d8ae1971 127.0.0.1 +2022-12-04T05:49:13.114Z [INFO] agent.router: Initializing LAN area manager +49:13.115 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp +49:13.115 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= +49:13.115 [INFO] TestAgent.server: Adding LAN server: server="Node-f8f197cd-a080-2945-a267-5b54d8ae1971 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +49:13.115 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f8f197cd-a080-2945-a267-5b54d8ae1971.dc1 area=wan +49:13.115 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp +49:13.116 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp +49:13.116 [INFO] TestAgent: started state syncer +49:13.116 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp +49:13.168 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:13.184 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:13.184 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 +49:13.184 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:13.184 [DEBUG] TestAgent.server.raft: vote granted: from=f8f197cd-a080-2945-a267-5b54d8ae1971 term=2 tally=1 +49:13.184 [INFO] TestAgent.server.raft: election won: tally=1 +49:13.184 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" +49:13.184 [INFO] TestAgent.server: cluster leadership acquired +49:13.184 [INFO] TestAgent.server: initializing acls +49:13.185 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:13.185 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:13.185 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:13.185 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:13.185 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:13.185 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:13.186 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-f8f197cd-a080-2945-a267-5b54d8ae1971 +49:13.186 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-f8f197cd-a080-2945-a267-5b54d8ae1971.dc1 +49:13.186 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:13.186 [INFO] TestAgent.server: New leader elected: payload=Node-f8f197cd-a080-2945-a267-5b54d8ae1971 +49:13.186 [INFO] TestAgent.server: Updating LAN server: server="Node-f8f197cd-a080-2945-a267-5b54d8ae1971 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +49:13.186 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-f8f197cd-a080-2945-a267-5b54d8ae1971.dc1 area=wan +49:13.249 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:13.249 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:13.249 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:13.249 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:13.249 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:13.249 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f8f197cd-a080-2945-a267-5b54d8ae1971 +49:13.249 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f8f197cd-a080-2945-a267-5b54d8ae1971 +49:13.251 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f8f197cd-a080-2945-a267-5b54d8ae1971 +49:13.253 [INFO] TestAgent.server: federation state anti-entropy synced +49:13.436 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:13.437 [INFO] TestAgent: Synced node info +49:13.437 [DEBUG] TestAgent: Node info in sync +49:13.502 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:50366 latency=1.034433ms +49:13.503 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:50366 latency=535.334µs +49:13.510 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/613a5ba3-1365-af91-c105-55a3efe2bf25/clone from=127.0.0.1:50368 latency=751.786µs +49:13.533 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/613a5ba3-1365-af91-c105-55a3efe2bf25/clone from=127.0.0.1:50370 latency=891.002µs +49:13.533 [INFO] TestAgent: Requesting shutdown +49:13.534 [INFO] TestAgent.server: shutting down server +49:13.534 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:13.534 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:13.534 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:13.534 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:13.534 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:13.534 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:13.534 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:13.534 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:13.534 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:13.534 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:13.534 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:13.534 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:13.534 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:13.534 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:13.534 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:13.534Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:13.534Z [INFO] agent.router.manager: shutting down +49:13.534 [INFO] TestAgent: consul server down +49:13.534 [INFO] TestAgent: shutdown complete +49:13.534 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp +49:13.534 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp +49:13.534 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp +49:14.034 [INFO] TestAgent: Waiting for endpoints to shut down +49:14.034 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenCloneCommand_JSON (1.02s) + --- PASS: TestTokenCloneCommand_JSON/Description (0.01s) + --- PASS: TestTokenCloneCommand_JSON/Without_Description (0.02s) PASS -ok github.com/hashicorp/consul/command/acl/token/clone 1.817s +ok github.com/hashicorp/consul/command/acl/token/clone 1.089s === RUN TestTokenCreateCommand_noTabs === PAUSE TestTokenCreateCommand_noTabs === RUN TestTokenCreateCommand_Pretty @@ -76963,252 +77144,256 @@ === RUN TestTokenCreateCommand_Pretty/policy-id === RUN TestTokenCreateCommand_Pretty/node-identity === RUN TestTokenCreateCommand_Pretty/predefined-ids -2021-10-31T23:14:31.699Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:31.756Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:31.758 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d4a95fd9-08bc-b993-a1b0-91b848150c7f Address:127.0.0.1:14097}]" -14:31.759 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14097 [Follower]" leader= -14:31.759 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 -14:31.760 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 -2021-10-31T23:14:31.760Z [INFO] agent.router: Initializing LAN area manager -14:31.760 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=udp -14:31.760 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:14097) (DC: dc1)" -14:31.760 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan -14:31.760 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=tcp -14:31.777 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14093 network=tcp -14:31.777 [INFO] TestAgent: started state syncer -14:31.777 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14098 network=tcp -14:31.813 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:31.813 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14097 [Candidate]" term=2 -14:31.814 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:31.814 [DEBUG] TestAgent.server.raft: vote granted: from=d4a95fd9-08bc-b993-a1b0-91b848150c7f term=2 tally=1 -14:31.814 [INFO] TestAgent.server.raft: election won: tally=1 -14:31.814 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14097 [Leader]" -14:31.814 [INFO] TestAgent.server: cluster leadership acquired -14:31.814 [INFO] TestAgent.server: initializing acls -14:31.814 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:31.814 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:31.815 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:31.815 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:31.815 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:31.815 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:31.815 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: test-node -14:31.815 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: test-node.dc1 -14:31.816 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:31.817 [INFO] TestAgent.server: New leader elected: payload=test-node -14:31.817 [INFO] TestAgent.server: Updating LAN server: server="test-node (Addr: tcp/127.0.0.1:14097) (DC: dc1)" -14:31.818 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=test-node.dc1 area=wan -14:31.867 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:31.867 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:31.867 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:31.867 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:31.867 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:31.867 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -14:31.867 [INFO] TestAgent.server: member joined, marking health alive: member=test-node -14:31.867 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node -14:31.868 [INFO] TestAgent.server: federation state anti-entropy synced -14:31.956 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:54884 latency=1.207116ms -14:31.967 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:54886 latency=1.284178ms -14:31.989 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:54888 latency=11.911896ms -14:32.000 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:54890 latency=2.116343ms -14:32.003 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:54892 latency=549.763µs -14:32.005 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:54894 latency=598.992µs -14:32.005 [INFO] TestAgent: Requesting shutdown -14:32.006 [INFO] TestAgent.server: shutting down server -14:32.006 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:32.006 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:32.006 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:32.006 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:32.006 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:32.006 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:32.006 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:32.006 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:32.006 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:32.006 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -14:32.006 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:32.006 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:32.006 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:32.006 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:32.006 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:32.006 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:32.006Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:32.006Z [INFO] agent.router.manager: shutting down -14:32.006 [INFO] TestAgent: consul server down -14:32.006 [INFO] TestAgent: shutdown complete -14:32.006 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=tcp -14:32.006 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=udp -14:32.006 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14093 network=tcp -14:32.509 [INFO] TestAgent: Waiting for endpoints to shut down -14:32.509 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenCreateCommand_Pretty (0.92s) - --- PASS: TestTokenCreateCommand_Pretty/policy-name (0.01s) - --- PASS: TestTokenCreateCommand_Pretty/policy-id (0.02s) +2022-12-04T05:49:14.207Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.229Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.231 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c2222bcd-d191-7b26-0f25-a0d8ac1a61b1 Address:127.0.0.1:30426}]" +49:14.232 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c2222bcd-d191-7b26-0f25-a0d8ac1a61b1.dc1 127.0.0.1 +49:14.232 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= +49:14.233 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c2222bcd-d191-7b26-0f25-a0d8ac1a61b1 127.0.0.1 +2022-12-04T05:49:14.233Z [INFO] agent.router: Initializing LAN area manager +49:14.233 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c2222bcd-d191-7b26-0f25-a0d8ac1a61b1.dc1 area=wan +49:14.233 [INFO] TestAgent.server: Adding LAN server: server="Node-c2222bcd-d191-7b26-0f25-a0d8ac1a61b1 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +49:14.233 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp +49:14.233 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp +49:14.234 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp +49:14.234 [INFO] TestAgent: started state syncer +49:14.234 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp +49:14.284 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:14.296 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.296 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 +49:14.296 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.296 [DEBUG] TestAgent.server.raft: vote granted: from=c2222bcd-d191-7b26-0f25-a0d8ac1a61b1 term=2 tally=1 +49:14.296 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.296 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" +49:14.297 [INFO] TestAgent.server: cluster leadership acquired +49:14.297 [INFO] TestAgent.server: initializing acls +49:14.297 [INFO] TestAgent.server: New leader elected: payload=Node-c2222bcd-d191-7b26-0f25-a0d8ac1a61b1 +49:14.297 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:14.297 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:14.298 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:14.311 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:14.311 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:14.311 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:14.311 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-c2222bcd-d191-7b26-0f25-a0d8ac1a61b1 +49:14.311 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-c2222bcd-d191-7b26-0f25-a0d8ac1a61b1.dc1 +49:14.312 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-c2222bcd-d191-7b26-0f25-a0d8ac1a61b1.dc1 area=wan +49:14.312 [INFO] TestAgent.server: Updating LAN server: server="Node-c2222bcd-d191-7b26-0f25-a0d8ac1a61b1 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +49:14.312 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.324 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.324 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.324 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.324 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.324 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.324 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c2222bcd-d191-7b26-0f25-a0d8ac1a61b1 +49:14.324 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c2222bcd-d191-7b26-0f25-a0d8ac1a61b1 +49:14.325 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.325 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c2222bcd-d191-7b26-0f25-a0d8ac1a61b1 +49:14.473 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:14.474 [INFO] TestAgent: Synced node info +49:14.665 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:50372 latency=1.816166ms +49:14.667 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:50374 latency=880.147µs +49:14.668 [INFO] TestAgent: Requesting shutdown +49:14.668 [INFO] TestAgent.server: shutting down server +49:14.668 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:14.668 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:14.668 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.668 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.668 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.668 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.668 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.668 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.668 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.668 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:14.668 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:14.668 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.668 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:14.668 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.668 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:14.668Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.668Z [INFO] agent.router.manager: shutting down +49:14.668 [INFO] TestAgent: consul server down +49:14.668 [INFO] TestAgent: shutdown complete +49:14.668 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp +49:14.668 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp +49:14.668 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp +49:15.168 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.168 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenCreateCommand_JSON (1.00s) +2022-12-04T05:49:14.208Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.235Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.246 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:08c5bd5e-2f19-aa7b-0770-367eefdccd02 Address:127.0.0.1:30433}]" +49:14.246 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30433 [Follower]" leader= +49:14.246 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: test-node.dc1 127.0.0.1 +49:14.246 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: test-node 127.0.0.1 +2022-12-04T05:49:14.246Z [INFO] agent.router: Initializing LAN area manager +49:14.247 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=udp +49:14.247 [INFO] TestAgent.server: Adding LAN server: server="test-node (Addr: tcp/127.0.0.1:30433) (DC: dc1)" +49:14.247 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=tcp +49:14.247 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=test-node.dc1 area=wan +49:14.247 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30429 network=tcp +49:14.247 [INFO] TestAgent: started state syncer +49:14.247 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30434 network=tcp +49:14.297 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:14.300 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.300 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30433 [Candidate]" term=2 +49:14.300 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.301 [DEBUG] TestAgent.server.raft: vote granted: from=08c5bd5e-2f19-aa7b-0770-367eefdccd02 term=2 tally=1 +49:14.301 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.301 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30433 [Leader]" +49:14.301 [INFO] TestAgent.server: cluster leadership acquired +49:14.301 [INFO] TestAgent.server: New leader elected: payload=test-node +49:14.301 [INFO] TestAgent.server: initializing acls +49:14.302 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:14.302 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:14.303 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:14.303 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:14.303 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:14.303 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:14.303 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: test-node +49:14.303 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: test-node.dc1 +49:14.303 [INFO] TestAgent.server: Updating LAN server: server="test-node (Addr: tcp/127.0.0.1:30433) (DC: dc1)" +49:14.304 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=test-node.dc1 area=wan +49:14.304 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.333 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.333 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.333 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.333 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.333 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.333 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:14.333 [INFO] TestAgent.server: member joined, marking health alive: member=test-node +49:14.333 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.333 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=test-node +49:14.553 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:14.553 [INFO] TestAgent: Synced node info +49:14.553 [DEBUG] TestAgent: Node info in sync +49:14.666 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:33382 latency=914.887µs +49:14.668 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:33384 latency=527.718µs +49:14.670 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:33386 latency=567.863µs +49:14.677 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:33388 latency=726.431µs +49:14.679 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:33390 latency=503.644µs +49:14.681 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:33392 latency=639.907µs +49:14.682 [INFO] TestAgent: Requesting shutdown +49:14.682 [INFO] TestAgent.server: shutting down server +49:14.682 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:14.682 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:14.682 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.682 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.682 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.682 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.682 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.682 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.682 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.682 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:14.682 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:14.682 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:14.682 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.682 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:14.682 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +2022-12-04T05:49:14.682Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.682Z [INFO] agent.router.manager: shutting down +49:14.682 [INFO] TestAgent: consul server down +49:14.682 [INFO] TestAgent: shutdown complete +49:14.682 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=tcp +49:14.682 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=udp +49:14.682 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30429 network=tcp +49:15.183 [INFO] TestAgent: Waiting for endpoints to shut down +49:15.183 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenCreateCommand_Pretty (1.02s) + --- PASS: TestTokenCreateCommand_Pretty/policy-name (0.00s) + --- PASS: TestTokenCreateCommand_Pretty/policy-id (0.00s) --- PASS: TestTokenCreateCommand_Pretty/node-identity (0.01s) --- PASS: TestTokenCreateCommand_Pretty/predefined-ids (0.00s) -2021-10-31T23:14:31.702Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:31.748Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:31.774 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ff6efe74-44ad-c2fe-6370-65f1ec912e0e Address:127.0.0.1:14090}]" -14:31.775 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ff6efe74-44ad-c2fe-6370-65f1ec912e0e.dc1 127.0.0.1 -14:31.777 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= -14:31.778 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ff6efe74-44ad-c2fe-6370-65f1ec912e0e 127.0.0.1 -2021-10-31T23:14:31.778Z [INFO] agent.router: Initializing LAN area manager -14:31.779 [INFO] TestAgent.server: Adding LAN server: server="Node-ff6efe74-44ad-c2fe-6370-65f1ec912e0e (Addr: tcp/127.0.0.1:14090) (DC: dc1)" -14:31.779 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ff6efe74-44ad-c2fe-6370-65f1ec912e0e.dc1 area=wan -14:31.780 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp -14:31.794 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp -14:31.794 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp -14:31.794 [INFO] TestAgent: started state syncer -14:31.794 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp -14:31.837 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:31.838 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 -14:31.838 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:31.838 [DEBUG] TestAgent.server.raft: vote granted: from=ff6efe74-44ad-c2fe-6370-65f1ec912e0e term=2 tally=1 -14:31.838 [INFO] TestAgent.server.raft: election won: tally=1 -14:31.838 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" -14:31.838 [INFO] TestAgent.server: cluster leadership acquired -14:31.838 [INFO] TestAgent.server: initializing acls -14:31.838 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:31.838 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:31.839 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:31.839 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:31.839 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:31.839 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:31.839 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ff6efe74-44ad-c2fe-6370-65f1ec912e0e -14:31.839 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ff6efe74-44ad-c2fe-6370-65f1ec912e0e.dc1 -14:31.840 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:31.840 [INFO] TestAgent.server: New leader elected: payload=Node-ff6efe74-44ad-c2fe-6370-65f1ec912e0e -14:31.840 [INFO] TestAgent.server: Updating LAN server: server="Node-ff6efe74-44ad-c2fe-6370-65f1ec912e0e (Addr: tcp/127.0.0.1:14090) (DC: dc1)" -14:31.840 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ff6efe74-44ad-c2fe-6370-65f1ec912e0e.dc1 area=wan -14:31.863 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:31.863 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:31.863 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:31.863 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:31.863 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:31.863 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ff6efe74-44ad-c2fe-6370-65f1ec912e0e -14:31.863 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ff6efe74-44ad-c2fe-6370-65f1ec912e0e -14:31.864 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ff6efe74-44ad-c2fe-6370-65f1ec912e0e -14:31.864 [INFO] TestAgent.server: federation state anti-entropy synced -14:32.043 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -14:32.043 [INFO] TestAgent: Synced node info -14:32.207 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:49464 latency=1.003977ms -14:32.210 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:49466 latency=1.973442ms -14:32.211 [INFO] TestAgent: Requesting shutdown -14:32.211 [INFO] TestAgent.server: shutting down server -14:32.211 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:32.211 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:32.211 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:32.211 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:32.211 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:32.211 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:32.211 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:32.211 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:32.211 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:32.211 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:32.211 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:32.211 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:32.211 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:32.211 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:32.211 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:32.211Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:32.211Z [INFO] agent.router.manager: shutting down -14:32.211 [INFO] TestAgent: consul server down -14:32.211 [INFO] TestAgent: shutdown complete -14:32.211 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp -14:32.211 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp -14:32.211 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp -14:32.713 [INFO] TestAgent: Waiting for endpoints to shut down -14:32.713 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenCreateCommand_JSON (1.12s) PASS -ok github.com/hashicorp/consul/command/acl/token/create 1.313s +ok github.com/hashicorp/consul/command/acl/token/create 1.069s === RUN TestTokenDeleteCommand_noTabs === PAUSE TestTokenDeleteCommand_noTabs === RUN TestTokenDeleteCommand === PAUSE TestTokenDeleteCommand === CONT TestTokenDeleteCommand_noTabs ---- PASS: TestTokenDeleteCommand_noTabs (0.00s) === CONT TestTokenDeleteCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestTokenDeleteCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:14:39.570Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:39.644Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:39.651 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:afecc6a0-ba5d-ba0f-4810-27ddc06d905d Address:127.0.0.1:22258}]" -14:39.651 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-afecc6a0-ba5d-ba0f-4810-27ddc06d905d.dc1 127.0.0.1 -14:39.652 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-afecc6a0-ba5d-ba0f-4810-27ddc06d905d 127.0.0.1 -2021-10-31T23:14:39.652Z [INFO] agent.router: Initializing LAN area manager -14:39.652 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=udp -14:39.652 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22258 [Follower]" leader= -14:39.652 [INFO] TestAgent.server: Adding LAN server: server="Node-afecc6a0-ba5d-ba0f-4810-27ddc06d905d (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -14:39.652 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-afecc6a0-ba5d-ba0f-4810-27ddc06d905d.dc1 area=wan -14:39.653 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=tcp -14:39.653 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22254 network=tcp -14:39.653 [INFO] TestAgent: started state syncer -14:39.653 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22259 network=tcp -14:39.706 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -14:39.721 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:39.721 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22258 [Candidate]" term=2 -14:39.722 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:39.722 [DEBUG] TestAgent.server.raft: vote granted: from=afecc6a0-ba5d-ba0f-4810-27ddc06d905d term=2 tally=1 -14:39.722 [INFO] TestAgent.server.raft: election won: tally=1 -14:39.722 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22258 [Leader]" -14:39.722 [INFO] TestAgent.server: cluster leadership acquired -14:39.722 [INFO] TestAgent.server: initializing acls -14:39.722 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:39.722 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:39.723 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:39.723 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:39.723 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:39.723 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:39.723 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-afecc6a0-ba5d-ba0f-4810-27ddc06d905d -14:39.723 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-afecc6a0-ba5d-ba0f-4810-27ddc06d905d.dc1 -14:39.724 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:39.726 [INFO] TestAgent.server: New leader elected: payload=Node-afecc6a0-ba5d-ba0f-4810-27ddc06d905d -14:39.726 [INFO] TestAgent.server: Updating LAN server: server="Node-afecc6a0-ba5d-ba0f-4810-27ddc06d905d (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -14:39.726 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-afecc6a0-ba5d-ba0f-4810-27ddc06d905d.dc1 area=wan -14:39.774 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:39.774 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:39.774 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:39.774 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:39.774 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:39.774 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-afecc6a0-ba5d-ba0f-4810-27ddc06d905d -14:39.774 [INFO] TestAgent.server: member joined, marking health alive: member=Node-afecc6a0-ba5d-ba0f-4810-27ddc06d905d -14:39.775 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-afecc6a0-ba5d-ba0f-4810-27ddc06d905d -14:39.776 [INFO] TestAgent.server: federation state anti-entropy synced -14:39.934 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -14:39.934 [INFO] TestAgent: Synced node info -14:40.039 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:49820 latency=1.334533ms -14:40.056 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/token/eb712135-a701-27f6-606a-01bd91a23504 from=127.0.0.1:49822 latency=2.017945ms -14:40.062 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/token/eb712135-a701-27f6-606a-01bd91a23504 from=127.0.0.1:49820 error="ACL not found" -14:40.082 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/eb712135-a701-27f6-606a-01bd91a23504 from=127.0.0.1:49820 latency=19.54089ms -14:40.086 [INFO] TestAgent: Requesting shutdown -14:40.086 [INFO] TestAgent.server: shutting down server -14:40.086 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:40.086 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:40.086 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:40.086 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:40.086 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:40.086 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:40.086 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:40.086 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:40.086 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:40.086 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:40.086 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:40.086 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:40.086 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:40.086 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:40.086 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:40.086Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:40.086Z [INFO] agent.router.manager: shutting down -14:40.086 [INFO] TestAgent: consul server down -14:40.086 [INFO] TestAgent: shutdown complete -14:40.086 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=tcp -14:40.086 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=udp -14:40.086 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22254 network=tcp -14:40.589 [INFO] TestAgent: Waiting for endpoints to shut down -14:40.589 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenDeleteCommand (1.19s) +2022-12-04T05:49:14.038Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:14.059Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:14.061 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d704cfe7-4baa-0370-ac26-b080e9909392 Address:127.0.0.1:27363}]" +49:14.061 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= +49:14.062 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d704cfe7-4baa-0370-ac26-b080e9909392.dc1 127.0.0.1 +49:14.062 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d704cfe7-4baa-0370-ac26-b080e9909392 127.0.0.1 +2022-12-04T05:49:14.062Z [INFO] agent.router: Initializing LAN area manager +49:14.063 [INFO] TestAgent.server: Adding LAN server: server="Node-d704cfe7-4baa-0370-ac26-b080e9909392 (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +49:14.063 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp +49:14.063 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp +49:14.063 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d704cfe7-4baa-0370-ac26-b080e9909392.dc1 area=wan +49:14.063 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp +49:14.063 [INFO] TestAgent: started state syncer +49:14.063 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp +49:14.114 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:14.118 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:14.118 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 +49:14.118 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:14.118 [DEBUG] TestAgent.server.raft: vote granted: from=d704cfe7-4baa-0370-ac26-b080e9909392 term=2 tally=1 +49:14.118 [INFO] TestAgent.server.raft: election won: tally=1 +49:14.118 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" +49:14.118 [INFO] TestAgent.server: cluster leadership acquired +49:14.118 [INFO] TestAgent.server: initializing acls +49:14.119 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:14.119 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:14.119 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:14.119 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:14.119 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:14.119 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:14.120 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-d704cfe7-4baa-0370-ac26-b080e9909392 +49:14.120 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-d704cfe7-4baa-0370-ac26-b080e9909392.dc1 +49:14.120 [INFO] TestAgent.server: New leader elected: payload=Node-d704cfe7-4baa-0370-ac26-b080e9909392 +49:14.120 [INFO] TestAgent.server: Updating LAN server: server="Node-d704cfe7-4baa-0370-ac26-b080e9909392 (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +49:14.120 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-d704cfe7-4baa-0370-ac26-b080e9909392.dc1 area=wan +49:14.120 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:14.147 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:14.147 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:14.147 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:14.147 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:14.147 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:14.147 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d704cfe7-4baa-0370-ac26-b080e9909392 +49:14.147 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d704cfe7-4baa-0370-ac26-b080e9909392 +49:14.147 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d704cfe7-4baa-0370-ac26-b080e9909392 +49:14.148 [INFO] TestAgent.server: federation state anti-entropy synced +49:14.175 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:14.175 [INFO] TestAgent: Synced node info +49:14.196 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:59672 latency=6.116806ms +49:14.201 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/acl/token/c7bb632e-230f-069c-109f-5bd7f0b6133c from=127.0.0.1:59674 latency=653.558µs +49:14.202 [ERROR] TestAgent.http: Request error: method=GET url=/v1/acl/token/c7bb632e-230f-069c-109f-5bd7f0b6133c from=127.0.0.1:59672 error="ACL not found" +49:14.203 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/c7bb632e-230f-069c-109f-5bd7f0b6133c from=127.0.0.1:59672 latency=985.056µs +49:14.203 [INFO] TestAgent: Requesting shutdown +49:14.203 [INFO] TestAgent.server: shutting down server +49:14.203 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:14.203 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:14.203 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:14.203 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:14.203 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:14.203 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:14.203 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:14.203 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:14.203 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:14.203 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:14.203 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:14.203 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:14.203 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:14.203 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:14.203 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:14.203Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:14.203Z [INFO] agent.router.manager: shutting down +49:14.204 [INFO] TestAgent: consul server down +49:14.204 [INFO] TestAgent: shutdown complete +49:14.204 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp +49:14.204 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp +49:14.204 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp +49:14.704 [INFO] TestAgent: Waiting for endpoints to shut down +49:14.704 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenDeleteCommand (0.71s) PASS -ok github.com/hashicorp/consul/command/acl/token/delete 1.363s +ok github.com/hashicorp/consul/command/acl/token/delete 0.777s === RUN TestTokenListCommand_noTabs === PAUSE TestTokenListCommand_noTabs === RUN TestTokenListCommand_Pretty @@ -77216,170 +77401,173 @@ === RUN TestTokenListCommand_JSON === PAUSE TestTokenListCommand_JSON === CONT TestTokenListCommand_noTabs ---- PASS: TestTokenListCommand_noTabs (0.00s) === CONT TestTokenListCommand_JSON -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestTokenListCommand_Pretty +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestTokenListCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:14:48.741Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:48.800Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:48.823 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7732cc1b-6528-1915-860d-7eb19bcfee43 Address:127.0.0.1:15118}]" -14:48.850 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7732cc1b-6528-1915-860d-7eb19bcfee43.dc1 127.0.0.1 -14:48.851 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7732cc1b-6528-1915-860d-7eb19bcfee43 127.0.0.1 -2021-10-31T23:14:48.851Z [INFO] agent.router: Initializing LAN area manager -14:48.851 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=udp -14:48.851 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15118 [Follower]" leader= -14:48.851 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=tcp -14:48.851 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7732cc1b-6528-1915-860d-7eb19bcfee43.dc1 area=wan -14:48.851 [INFO] TestAgent.server: Adding LAN server: server="Node-7732cc1b-6528-1915-860d-7eb19bcfee43 (Addr: tcp/127.0.0.1:15118) (DC: dc1)" -14:48.852 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15114 network=tcp -14:48.852 [INFO] TestAgent: started state syncer -14:48.852 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15119 network=tcp -14:48.902 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -14:48.918 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:48.918 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15118 [Candidate]" term=2 -14:48.918 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:48.918 [DEBUG] TestAgent.server.raft: vote granted: from=7732cc1b-6528-1915-860d-7eb19bcfee43 term=2 tally=1 -14:48.918 [INFO] TestAgent.server.raft: election won: tally=1 -14:48.918 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15118 [Leader]" -14:48.918 [INFO] TestAgent.server: cluster leadership acquired -14:48.918 [INFO] TestAgent.server: initializing acls -14:48.918 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:48.918 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:48.919 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:48.919 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:48.919 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:48.919 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:48.919 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7732cc1b-6528-1915-860d-7eb19bcfee43 -14:48.919 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7732cc1b-6528-1915-860d-7eb19bcfee43.dc1 -14:48.920 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:48.920 [INFO] TestAgent.server: New leader elected: payload=Node-7732cc1b-6528-1915-860d-7eb19bcfee43 -14:48.920 [INFO] TestAgent.server: Updating LAN server: server="Node-7732cc1b-6528-1915-860d-7eb19bcfee43 (Addr: tcp/127.0.0.1:15118) (DC: dc1)" -14:48.920 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7732cc1b-6528-1915-860d-7eb19bcfee43.dc1 area=wan -14:48.955 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:48.955 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:48.955 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:48.955 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:48.955 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:48.955 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7732cc1b-6528-1915-860d-7eb19bcfee43 -14:48.955 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7732cc1b-6528-1915-860d-7eb19bcfee43 -14:48.956 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7732cc1b-6528-1915-860d-7eb19bcfee43 -14:48.956 [INFO] TestAgent.server: federation state anti-entropy synced -14:49.041 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:41984 latency=1.427141ms -14:49.050 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:41984 latency=9.059457ms -14:49.055 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:41984 latency=910.62µs -14:49.056 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:41984 latency=578.077µs -14:49.057 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:41984 latency=429.688µs -14:49.066 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/tokens from=127.0.0.1:41986 latency=670.069µs -14:49.067 [INFO] TestAgent: Requesting shutdown -14:49.067 [INFO] TestAgent.server: shutting down server -14:49.067 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:49.067 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:49.067 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:49.067 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:49.067 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:49.067 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:49.067 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:49.067 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:49.067 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:49.067 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -14:49.067 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:49.067 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:49.067 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:49.068 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:49.068 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:49.068 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:49.068Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:49.068Z [INFO] agent.router.manager: shutting down -14:49.068 [INFO] TestAgent: consul server down -14:49.068 [INFO] TestAgent: shutdown complete -14:49.068 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=tcp -14:49.068 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=udp -14:49.068 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15114 network=tcp -14:49.569 [INFO] TestAgent: Waiting for endpoints to shut down -14:49.569 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenListCommand_Pretty (0.99s) -2021-10-31T23:14:48.765Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:48.853Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:48.863 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:123a43d8-725f-cfa6-1493-0ba21d0414ae Address:127.0.0.1:15111}]" -14:48.863 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-123a43d8-725f-cfa6-1493-0ba21d0414ae.dc1 127.0.0.1 -14:48.864 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-123a43d8-725f-cfa6-1493-0ba21d0414ae 127.0.0.1 -2021-10-31T23:14:48.864Z [INFO] agent.router: Initializing LAN area manager -14:48.864 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=udp -14:48.864 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15111 [Follower]" leader= -14:48.865 [INFO] TestAgent.server: Adding LAN server: server="Node-123a43d8-725f-cfa6-1493-0ba21d0414ae (Addr: tcp/127.0.0.1:15111) (DC: dc1)" -14:48.865 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-123a43d8-725f-cfa6-1493-0ba21d0414ae.dc1 area=wan -14:48.865 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=tcp -14:48.878 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15107 network=tcp -14:48.878 [INFO] TestAgent: started state syncer -14:48.878 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15112 network=tcp -14:48.913 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:48.914 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15111 [Candidate]" term=2 -14:48.914 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:48.914 [DEBUG] TestAgent.server.raft: vote granted: from=123a43d8-725f-cfa6-1493-0ba21d0414ae term=2 tally=1 -14:48.914 [INFO] TestAgent.server.raft: election won: tally=1 -14:48.914 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15111 [Leader]" -14:48.914 [INFO] TestAgent.server: cluster leadership acquired -14:48.914 [INFO] TestAgent.server: initializing acls -14:48.914 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:48.914 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:48.915 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:48.915 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:48.915 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:48.915 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:48.915 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-123a43d8-725f-cfa6-1493-0ba21d0414ae -14:48.915 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-123a43d8-725f-cfa6-1493-0ba21d0414ae.dc1 -14:48.916 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:48.917 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-123a43d8-725f-cfa6-1493-0ba21d0414ae.dc1 area=wan -14:48.918 [INFO] TestAgent.server: New leader elected: payload=Node-123a43d8-725f-cfa6-1493-0ba21d0414ae -14:48.918 [INFO] TestAgent.server: Updating LAN server: server="Node-123a43d8-725f-cfa6-1493-0ba21d0414ae (Addr: tcp/127.0.0.1:15111) (DC: dc1)" -14:48.963 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:48.963 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:48.963 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:48.963 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:48.963 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:48.963 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-123a43d8-725f-cfa6-1493-0ba21d0414ae -14:48.963 [INFO] TestAgent.server: member joined, marking health alive: member=Node-123a43d8-725f-cfa6-1493-0ba21d0414ae -14:48.979 [INFO] TestAgent.server: federation state anti-entropy synced -14:48.979 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-123a43d8-725f-cfa6-1493-0ba21d0414ae -14:49.030 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -14:49.031 [INFO] TestAgent: Synced node info -14:49.300 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:60820 latency=1.199062ms -14:49.301 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:60820 latency=516.249µs -14:49.303 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:60820 latency=703.61µs -14:49.304 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:60820 latency=504.003µs -14:49.305 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:60820 latency=537.831µs -14:49.321 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/tokens from=127.0.0.1:60822 latency=10.643505ms -14:49.335 [INFO] TestAgent: Requesting shutdown -14:49.335 [INFO] TestAgent.server: shutting down server -14:49.335 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:49.335 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:49.335 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:49.335 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:49.335 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:49.335 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:49.335 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:49.335 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:49.335 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:49.335 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:49.335 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:49.335 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:49.335 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:49.335 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:49.335 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:49.335Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:49.335Z [INFO] agent.router.manager: shutting down -14:49.335 [INFO] TestAgent: consul server down -14:49.335 [INFO] TestAgent: shutdown complete -14:49.335 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=tcp -14:49.335 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=udp -14:49.335 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15107 network=tcp -14:49.837 [INFO] TestAgent: Waiting for endpoints to shut down -14:49.837 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenListCommand_JSON (1.25s) +2022-12-04T05:49:20.000Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.030Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.031 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:25384bdd-3fa7-f229-08b6-4b55ed49aaef Address:127.0.0.1:20216}]" +49:20.031 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20216 [Follower]" leader= +49:20.032 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-25384bdd-3fa7-f229-08b6-4b55ed49aaef.dc1 127.0.0.1 +49:20.032 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-25384bdd-3fa7-f229-08b6-4b55ed49aaef 127.0.0.1 +2022-12-04T05:49:20.032Z [INFO] agent.router: Initializing LAN area manager +49:20.032 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=udp +49:20.032 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-25384bdd-3fa7-f229-08b6-4b55ed49aaef.dc1 area=wan +49:20.033 [INFO] TestAgent.server: Adding LAN server: server="Node-25384bdd-3fa7-f229-08b6-4b55ed49aaef (Addr: tcp/127.0.0.1:20216) (DC: dc1)" +49:20.033 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20211 network=tcp +49:20.033 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20212 network=tcp +49:20.033 [INFO] TestAgent: started state syncer +49:20.033 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20217 network=tcp +49:20.084 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:20.099 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.099 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20216 [Candidate]" term=2 +49:20.099 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.099 [DEBUG] TestAgent.server.raft: vote granted: from=25384bdd-3fa7-f229-08b6-4b55ed49aaef term=2 tally=1 +49:20.099 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.099 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20216 [Leader]" +49:20.100 [INFO] TestAgent.server: cluster leadership acquired +49:20.100 [INFO] TestAgent.server: initializing acls +49:20.100 [INFO] TestAgent.server: New leader elected: payload=Node-25384bdd-3fa7-f229-08b6-4b55ed49aaef +49:20.100 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:20.100 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:20.100 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:20.100 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:20.100 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:20.100 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:20.101 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-25384bdd-3fa7-f229-08b6-4b55ed49aaef +49:20.101 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-25384bdd-3fa7-f229-08b6-4b55ed49aaef.dc1 +49:20.101 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.102 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.102 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.102 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.102 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.102 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.102 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-25384bdd-3fa7-f229-08b6-4b55ed49aaef +49:20.102 [INFO] TestAgent.server: member joined, marking health alive: member=Node-25384bdd-3fa7-f229-08b6-4b55ed49aaef +49:20.103 [INFO] TestAgent.server: Updating LAN server: server="Node-25384bdd-3fa7-f229-08b6-4b55ed49aaef (Addr: tcp/127.0.0.1:20216) (DC: dc1)" +49:20.103 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-25384bdd-3fa7-f229-08b6-4b55ed49aaef +49:20.103 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-25384bdd-3fa7-f229-08b6-4b55ed49aaef.dc1 area=wan +49:20.103 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.135 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.136 [INFO] TestAgent: Synced node info +49:20.136 [DEBUG] TestAgent: Node info in sync +49:20.173 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:42766 latency=6.610794ms +49:20.175 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:42766 latency=596.971µs +49:20.176 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:42766 latency=522.102µs +49:20.177 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:42766 latency=467.789µs +49:20.177 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:42766 latency=501.784µs +49:20.179 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/tokens from=127.0.0.1:42768 latency=616.831µs +49:20.180 [INFO] TestAgent: Requesting shutdown +49:20.180 [INFO] TestAgent.server: shutting down server +49:20.180 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:20.180 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:20.180 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.180 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.180 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.180 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.180 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.180 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:20.180 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:20.180 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.180 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.180 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:20.180 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.180 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.180 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:20.180Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.181Z [INFO] agent.router.manager: shutting down +49:20.181 [INFO] TestAgent: consul server down +49:20.181 [INFO] TestAgent: shutdown complete +49:20.181 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=tcp +49:20.181 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20211 network=udp +49:20.181 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20212 network=tcp +49:20.681 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.681 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenListCommand_JSON (0.72s) +2022-12-04T05:49:20.000Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.010Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.018 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ddee27a6-f2e2-cb3e-b3a6-060ca70739f8 Address:127.0.0.1:20223}]" +49:20.019 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ddee27a6-f2e2-cb3e-b3a6-060ca70739f8.dc1 127.0.0.1 +49:20.020 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:20223 [Follower]" leader= +49:20.023 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ddee27a6-f2e2-cb3e-b3a6-060ca70739f8 127.0.0.1 +2022-12-04T05:49:20.024Z [INFO] agent.router: Initializing LAN area manager +49:20.029 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20218 network=udp +49:20.029 [INFO] TestAgent.server: Adding LAN server: server="Node-ddee27a6-f2e2-cb3e-b3a6-060ca70739f8 (Addr: tcp/127.0.0.1:20223) (DC: dc1)" +49:20.029 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ddee27a6-f2e2-cb3e-b3a6-060ca70739f8.dc1 area=wan +49:20.029 [INFO] TestAgent: Started DNS server: address=127.0.0.1:20218 network=tcp +49:20.029 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:20219 network=tcp +49:20.030 [INFO] TestAgent: started state syncer +49:20.030 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:20224 network=tcp +49:20.076 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.076 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:20223 [Candidate]" term=2 +49:20.076 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.076 [DEBUG] TestAgent.server.raft: vote granted: from=ddee27a6-f2e2-cb3e-b3a6-060ca70739f8 term=2 tally=1 +49:20.076 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.076 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:20223 [Leader]" +49:20.076 [INFO] TestAgent.server: cluster leadership acquired +49:20.077 [INFO] TestAgent.server: initializing acls +49:20.077 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:20.077 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:20.077 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:20.078 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:20.078 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:20.078 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:20.078 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-ddee27a6-f2e2-cb3e-b3a6-060ca70739f8 +49:20.078 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-ddee27a6-f2e2-cb3e-b3a6-060ca70739f8.dc1 +49:20.078 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.080 [INFO] TestAgent.server: New leader elected: payload=Node-ddee27a6-f2e2-cb3e-b3a6-060ca70739f8 +49:20.080 [INFO] TestAgent.server: Updating LAN server: server="Node-ddee27a6-f2e2-cb3e-b3a6-060ca70739f8 (Addr: tcp/127.0.0.1:20223) (DC: dc1)" +49:20.080 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-ddee27a6-f2e2-cb3e-b3a6-060ca70739f8.dc1 area=wan +49:20.094 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.094 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.094 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.094 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.094 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.094 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ddee27a6-f2e2-cb3e-b3a6-060ca70739f8 +49:20.094 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ddee27a6-f2e2-cb3e-b3a6-060ca70739f8 +49:20.095 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ddee27a6-f2e2-cb3e-b3a6-060ca70739f8 +49:20.095 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.274 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:20.275 [INFO] TestAgent: Synced node info +49:20.275 [DEBUG] TestAgent: Node info in sync +49:20.410 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:33842 latency=1.794356ms +49:20.412 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:33842 latency=834.725µs +49:20.413 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:33842 latency=485.902µs +49:20.414 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:33842 latency=650.148µs +49:20.415 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:33842 latency=538.404µs +49:20.416 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/tokens from=127.0.0.1:33844 latency=307.328µs +49:20.417 [INFO] TestAgent: Requesting shutdown +49:20.417 [INFO] TestAgent.server: shutting down server +49:20.417 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:20.417 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:20.417 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.417 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.417 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.417 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.417 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.417 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:20.417 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.417 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:20.417 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.417 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.417 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.417 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.417 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.417Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.417Z [INFO] agent.router.manager: shutting down +49:20.417 [INFO] TestAgent: consul server down +49:20.417 [INFO] TestAgent: shutdown complete +49:20.417 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20218 network=tcp +49:20.417 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:20218 network=udp +49:20.417 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:20219 network=tcp +49:20.918 [INFO] TestAgent: Waiting for endpoints to shut down +49:20.918 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenListCommand_Pretty (0.96s) PASS -ok github.com/hashicorp/consul/command/acl/token/list 1.511s +ok github.com/hashicorp/consul/command/acl/token/list 1.008s === RUN TestTokenReadCommand_noTabs === PAUSE TestTokenReadCommand_noTabs === RUN TestTokenReadCommand_Pretty @@ -77387,164 +77575,160 @@ === RUN TestTokenReadCommand_JSON === PAUSE TestTokenReadCommand_JSON === CONT TestTokenReadCommand_noTabs ---- PASS: TestTokenReadCommand_noTabs (0.01s) === CONT TestTokenReadCommand_JSON -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestTokenReadCommand_Pretty +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestTokenReadCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:14:53.016Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:53.093Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:53.094 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0f49eb0f-13ce-576a-a83f-9a4e1cb96aca Address:127.0.0.1:25321}]" -14:53.095 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0f49eb0f-13ce-576a-a83f-9a4e1cb96aca.dc1 127.0.0.1 -14:53.095 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0f49eb0f-13ce-576a-a83f-9a4e1cb96aca 127.0.0.1 -2021-10-31T23:14:53.095Z [INFO] agent.router: Initializing LAN area manager -14:53.096 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=udp -14:53.096 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25321 [Follower]" leader= -14:53.096 [INFO] TestAgent.server: Adding LAN server: server="Node-0f49eb0f-13ce-576a-a83f-9a4e1cb96aca (Addr: tcp/127.0.0.1:25321) (DC: dc1)" -14:53.096 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0f49eb0f-13ce-576a-a83f-9a4e1cb96aca.dc1 area=wan -14:53.096 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=tcp -14:53.096 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25317 network=tcp -14:53.096 [INFO] TestAgent: started state syncer -14:53.096 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25322 network=tcp -14:53.150 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -14:53.161 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:53.161 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25321 [Candidate]" term=2 -14:53.162 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:53.162 [DEBUG] TestAgent.server.raft: vote granted: from=0f49eb0f-13ce-576a-a83f-9a4e1cb96aca term=2 tally=1 -14:53.162 [INFO] TestAgent.server.raft: election won: tally=1 -14:53.162 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25321 [Leader]" -14:53.162 [INFO] TestAgent.server: cluster leadership acquired -14:53.162 [INFO] TestAgent.server: initializing acls -14:53.162 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:53.162 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:53.163 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:53.163 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:53.163 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:53.163 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:53.163 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-0f49eb0f-13ce-576a-a83f-9a4e1cb96aca -14:53.163 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-0f49eb0f-13ce-576a-a83f-9a4e1cb96aca.dc1 -14:53.164 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:53.164 [INFO] TestAgent.server: New leader elected: payload=Node-0f49eb0f-13ce-576a-a83f-9a4e1cb96aca -14:53.164 [INFO] TestAgent.server: Updating LAN server: server="Node-0f49eb0f-13ce-576a-a83f-9a4e1cb96aca (Addr: tcp/127.0.0.1:25321) (DC: dc1)" -14:53.164 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-0f49eb0f-13ce-576a-a83f-9a4e1cb96aca.dc1 area=wan -14:53.195 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:53.195 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:53.195 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:53.195 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:53.195 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:53.195 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0f49eb0f-13ce-576a-a83f-9a4e1cb96aca -14:53.195 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0f49eb0f-13ce-576a-a83f-9a4e1cb96aca -14:53.196 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0f49eb0f-13ce-576a-a83f-9a4e1cb96aca -14:53.196 [INFO] TestAgent.server: federation state anti-entropy synced -14:53.238 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -14:53.238 [INFO] TestAgent: Synced node info -14:53.238 [DEBUG] TestAgent: Node info in sync -14:53.297 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:37922 latency=1.281579ms -14:53.304 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/f673342e-ba91-d035-00df-9c44b51671c4 from=127.0.0.1:37924 latency=1.427851ms -14:53.310 [INFO] TestAgent: Requesting shutdown -14:53.310 [INFO] TestAgent.server: shutting down server -14:53.310 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:53.310 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:53.310 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:53.310 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:53.310 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:53.310 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:53.310 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:53.310 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:53.310 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:53.310 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:53.310 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:53.310 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:53.310 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:53.310 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:53.310 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:53.310Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:53.310Z [INFO] agent.router.manager: shutting down -14:53.310 [INFO] TestAgent: consul server down -14:53.310 [INFO] TestAgent: shutdown complete -14:53.310 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=tcp -14:53.310 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=udp -14:53.310 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25317 network=tcp -14:53.813 [INFO] TestAgent: Waiting for endpoints to shut down -14:53.813 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenReadCommand_JSON (1.00s) -2021-10-31T23:14:53.010Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:53.035Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:53.037 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:db750bc9-0997-2cae-7f9c-98fb19affbff Address:127.0.0.1:25328}]" -14:53.055 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-db750bc9-0997-2cae-7f9c-98fb19affbff.dc1 127.0.0.1 -14:53.056 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-db750bc9-0997-2cae-7f9c-98fb19affbff 127.0.0.1 -2021-10-31T23:14:53.056Z [INFO] agent.router: Initializing LAN area manager -14:53.066 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=udp -14:53.066 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=tcp -14:53.067 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25328 [Follower]" leader= -14:53.082 [INFO] TestAgent.server: Adding LAN server: server="Node-db750bc9-0997-2cae-7f9c-98fb19affbff (Addr: tcp/127.0.0.1:25328) (DC: dc1)" -14:53.082 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-db750bc9-0997-2cae-7f9c-98fb19affbff.dc1 area=wan -14:53.083 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25324 network=tcp -14:53.083 [INFO] TestAgent: started state syncer -14:53.083 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25329 network=tcp -14:53.113 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:53.114 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25328 [Candidate]" term=2 -14:53.114 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:53.114 [DEBUG] TestAgent.server.raft: vote granted: from=db750bc9-0997-2cae-7f9c-98fb19affbff term=2 tally=1 -14:53.114 [INFO] TestAgent.server.raft: election won: tally=1 -14:53.114 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25328 [Leader]" -14:53.114 [INFO] TestAgent.server: cluster leadership acquired -14:53.114 [INFO] TestAgent.server: initializing acls -14:53.114 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:53.114 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:53.115 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:53.115 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:53.115 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:53.115 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:53.115 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-db750bc9-0997-2cae-7f9c-98fb19affbff -14:53.116 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-db750bc9-0997-2cae-7f9c-98fb19affbff.dc1 -14:53.116 [INFO] TestAgent.server: New leader elected: payload=Node-db750bc9-0997-2cae-7f9c-98fb19affbff -14:53.116 [INFO] TestAgent.server: Updating LAN server: server="Node-db750bc9-0997-2cae-7f9c-98fb19affbff (Addr: tcp/127.0.0.1:25328) (DC: dc1)" -14:53.116 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-db750bc9-0997-2cae-7f9c-98fb19affbff.dc1 area=wan -14:53.116 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:53.226 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:53.226 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:53.226 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:53.226 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:53.226 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:53.226 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-db750bc9-0997-2cae-7f9c-98fb19affbff -14:53.226 [INFO] TestAgent.server: member joined, marking health alive: member=Node-db750bc9-0997-2cae-7f9c-98fb19affbff -14:53.227 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-db750bc9-0997-2cae-7f9c-98fb19affbff -14:53.227 [INFO] TestAgent.server: federation state anti-entropy synced -14:53.286 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -14:53.286 [INFO] TestAgent: Synced node info -14:53.472 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:52136 latency=1.04608ms -14:53.473 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/968c15b2-c194-07e5-b575-885ed1592cac from=127.0.0.1:52138 latency=304.375µs -14:53.478 [INFO] TestAgent: Requesting shutdown -14:53.478 [INFO] TestAgent.server: shutting down server -14:53.478 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:53.478 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:53.478 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:53.478 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:53.478 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:53.478 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:53.478 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:53.478 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:53.478 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:53.478 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:53.478 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:53.478 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:53.478 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:53.478 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:53.478 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:53.478Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:53.478Z [INFO] agent.router.manager: shutting down -14:53.478 [INFO] TestAgent: consul server down -14:53.478 [INFO] TestAgent: shutdown complete -14:53.478 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=tcp -14:53.478 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=udp -14:53.478 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25324 network=tcp -14:53.981 [INFO] TestAgent: Waiting for endpoints to shut down -14:53.981 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenReadCommand_Pretty (1.16s) +2022-12-04T05:49:20.613Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.625Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.628 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8d1527c2-0368-c65d-ea2c-2cd778a09e34 Address:127.0.0.1:11027}]" +49:20.628 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11027 [Follower]" leader= +49:20.628 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8d1527c2-0368-c65d-ea2c-2cd778a09e34.dc1 127.0.0.1 +49:20.629 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8d1527c2-0368-c65d-ea2c-2cd778a09e34 127.0.0.1 +2022-12-04T05:49:20.629Z [INFO] agent.router: Initializing LAN area manager +49:20.629 [INFO] TestAgent.server: Adding LAN server: server="Node-8d1527c2-0368-c65d-ea2c-2cd778a09e34 (Addr: tcp/127.0.0.1:11027) (DC: dc1)" +49:20.629 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=udp +49:20.629 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8d1527c2-0368-c65d-ea2c-2cd778a09e34.dc1 area=wan +49:20.629 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=tcp +49:20.630 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11023 network=tcp +49:20.630 [INFO] TestAgent: started state syncer +49:20.630 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11028 network=tcp +49:20.667 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.667 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11027 [Candidate]" term=2 +49:20.667 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.667 [DEBUG] TestAgent.server.raft: vote granted: from=8d1527c2-0368-c65d-ea2c-2cd778a09e34 term=2 tally=1 +49:20.667 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.667 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11027 [Leader]" +49:20.667 [INFO] TestAgent.server: cluster leadership acquired +49:20.667 [INFO] TestAgent.server: initializing acls +49:20.667 [INFO] TestAgent.server: New leader elected: payload=Node-8d1527c2-0368-c65d-ea2c-2cd778a09e34 +49:20.667 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:20.667 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:20.668 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:20.668 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:20.668 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:20.668 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:20.668 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-8d1527c2-0368-c65d-ea2c-2cd778a09e34 +49:20.668 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-8d1527c2-0368-c65d-ea2c-2cd778a09e34.dc1 +49:20.669 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.672 [INFO] TestAgent.server: Updating LAN server: server="Node-8d1527c2-0368-c65d-ea2c-2cd778a09e34 (Addr: tcp/127.0.0.1:11027) (DC: dc1)" +49:20.672 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-8d1527c2-0368-c65d-ea2c-2cd778a09e34.dc1 area=wan +49:20.684 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.684 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.684 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.684 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.684 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.685 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8d1527c2-0368-c65d-ea2c-2cd778a09e34 +49:20.685 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8d1527c2-0368-c65d-ea2c-2cd778a09e34 +49:20.685 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.686 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8d1527c2-0368-c65d-ea2c-2cd778a09e34 +49:20.707 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:51730 latency=1.917388ms +49:20.708 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/37db16fb-cb20-f787-0f23-316265da04ff from=127.0.0.1:51732 latency=354.127µs +49:20.708 [INFO] TestAgent: Requesting shutdown +49:20.709 [INFO] TestAgent.server: shutting down server +49:20.709 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.709 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.709 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:20.709 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:20.709 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.709 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.709 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.709 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.709 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.709 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:20.709 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.709 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.709 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:20.709 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:20.709 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.709 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.709Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.709Z [INFO] agent.router.manager: shutting down +49:20.709 [INFO] TestAgent: consul server down +49:20.709 [INFO] TestAgent: shutdown complete +49:20.709 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=tcp +49:20.709 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=udp +49:20.709 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11023 network=tcp +49:21.210 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.210 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenReadCommand_Pretty (0.63s) +2022-12-04T05:49:20.615Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:20.629Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:20.630 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:69af234a-928a-2cef-cdfc-fe9927f8ad8d Address:127.0.0.1:11034}]" +49:20.630 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11034 [Follower]" leader= +49:20.638 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-69af234a-928a-2cef-cdfc-fe9927f8ad8d.dc1 127.0.0.1 +49:20.639 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-69af234a-928a-2cef-cdfc-fe9927f8ad8d 127.0.0.1 +2022-12-04T05:49:20.639Z [INFO] agent.router: Initializing LAN area manager +49:20.639 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11029 network=udp +49:20.639 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-69af234a-928a-2cef-cdfc-fe9927f8ad8d.dc1 area=wan +49:20.639 [INFO] TestAgent.server: Adding LAN server: server="Node-69af234a-928a-2cef-cdfc-fe9927f8ad8d (Addr: tcp/127.0.0.1:11034) (DC: dc1)" +49:20.639 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11029 network=tcp +49:20.641 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11030 network=tcp +49:20.641 [INFO] TestAgent: started state syncer +49:20.641 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11035 network=tcp +49:20.679 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:20.679 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11034 [Candidate]" term=2 +49:20.679 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:20.679 [DEBUG] TestAgent.server.raft: vote granted: from=69af234a-928a-2cef-cdfc-fe9927f8ad8d term=2 tally=1 +49:20.679 [INFO] TestAgent.server.raft: election won: tally=1 +49:20.679 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11034 [Leader]" +49:20.679 [INFO] TestAgent.server: cluster leadership acquired +49:20.679 [INFO] TestAgent.server: New leader elected: payload=Node-69af234a-928a-2cef-cdfc-fe9927f8ad8d +49:20.679 [INFO] TestAgent.server: initializing acls +49:20.680 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:20.680 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:20.680 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:20.680 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:20.680 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:20.680 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:20.681 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-69af234a-928a-2cef-cdfc-fe9927f8ad8d +49:20.681 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-69af234a-928a-2cef-cdfc-fe9927f8ad8d.dc1 +49:20.681 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-69af234a-928a-2cef-cdfc-fe9927f8ad8d.dc1 area=wan +49:20.681 [INFO] TestAgent.server: Updating LAN server: server="Node-69af234a-928a-2cef-cdfc-fe9927f8ad8d (Addr: tcp/127.0.0.1:11034) (DC: dc1)" +49:20.681 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:20.686 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:20.686 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:20.686 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:20.686 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:20.686 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:20.686 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-69af234a-928a-2cef-cdfc-fe9927f8ad8d +49:20.686 [INFO] TestAgent.server: member joined, marking health alive: member=Node-69af234a-928a-2cef-cdfc-fe9927f8ad8d +49:20.686 [INFO] TestAgent.server: federation state anti-entropy synced +49:20.686 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-69af234a-928a-2cef-cdfc-fe9927f8ad8d +49:20.895 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:53416 latency=5.316966ms +49:20.897 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/a35b8835-d0a4-4093-b9e6-3ef1f9790b2a from=127.0.0.1:53418 latency=438.39µs +49:20.898 [INFO] TestAgent: Requesting shutdown +49:20.898 [INFO] TestAgent.server: shutting down server +49:20.898 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:20.898 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:20.898 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:20.898 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:20.898 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:20.898 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:20.898 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:20.898 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:20.898 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:20.898 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:20.898 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:20.898 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:20.898 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:20.898 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:20.898 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:20.898 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:20.898Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:20.898Z [INFO] agent.router.manager: shutting down +49:20.898 [INFO] TestAgent: consul server down +49:20.898 [INFO] TestAgent: shutdown complete +49:20.898 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11029 network=tcp +49:20.898 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11029 network=udp +49:20.898 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11030 network=tcp +49:21.399 [INFO] TestAgent: Waiting for endpoints to shut down +49:21.399 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenReadCommand_JSON (0.82s) PASS -ok github.com/hashicorp/consul/command/acl/token/read 2.131s +ok github.com/hashicorp/consul/command/acl/token/read 0.876s === RUN TestTokenUpdateCommand_noTabs === PAUSE TestTokenUpdateCommand_noTabs === RUN TestTokenUpdateCommand @@ -77552,10 +77736,10 @@ === RUN TestTokenUpdateCommand_JSON === PAUSE TestTokenUpdateCommand_JSON === CONT TestTokenUpdateCommand_noTabs ---- PASS: TestTokenUpdateCommand_noTabs (0.00s) === CONT TestTokenUpdateCommand_JSON -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestTokenUpdateCommand +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestTokenUpdateCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestTokenUpdateCommand/node-identity @@ -77564,366 +77748,363 @@ === RUN TestTokenUpdateCommand/policy-id === RUN TestTokenUpdateCommand/merge-description === RUN TestTokenUpdateCommand_JSON/update_with_policy_by_name -2021-10-31T23:14:59.512Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:59.596Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:59.631 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:60e12070-1ac6-5004-7db6-5ee32ce303ed Address:127.0.0.1:22258}]" -14:59.632 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-60e12070-1ac6-5004-7db6-5ee32ce303ed.dc1 127.0.0.1 -14:59.632 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22258 [Follower]" leader= -14:59.632 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-60e12070-1ac6-5004-7db6-5ee32ce303ed 127.0.0.1 -2021-10-31T23:14:59.632Z [INFO] agent.router: Initializing LAN area manager -14:59.632 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-60e12070-1ac6-5004-7db6-5ee32ce303ed.dc1 area=wan -14:59.633 [INFO] TestAgent.server: Adding LAN server: server="Node-60e12070-1ac6-5004-7db6-5ee32ce303ed (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -14:59.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=udp -14:59.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=tcp -14:59.634 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22254 network=tcp -14:59.634 [INFO] TestAgent: started state syncer -14:59.634 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22259 network=tcp -14:59.681 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:59.681 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22258 [Candidate]" term=2 -14:59.682 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:59.682 [DEBUG] TestAgent.server.raft: vote granted: from=60e12070-1ac6-5004-7db6-5ee32ce303ed term=2 tally=1 -14:59.682 [INFO] TestAgent.server.raft: election won: tally=1 -14:59.682 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22258 [Leader]" -14:59.682 [INFO] TestAgent.server: cluster leadership acquired -14:59.682 [INFO] TestAgent.server: New leader elected: payload=Node-60e12070-1ac6-5004-7db6-5ee32ce303ed -14:59.682 [INFO] TestAgent.server: initializing acls -14:59.682 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:59.682 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:59.682 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:59.683 [INFO] TestAgent.server: initializing acls -14:59.683 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:59.683 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:59.683 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:59.683 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:59.683 [DEBUG] TestAgent.server: transitioning out of legacy ACL mode -14:59.683 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-60e12070-1ac6-5004-7db6-5ee32ce303ed -14:59.683 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-60e12070-1ac6-5004-7db6-5ee32ce303ed.dc1 -14:59.683 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-60e12070-1ac6-5004-7db6-5ee32ce303ed.dc1 area=wan -14:59.683 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:59.683 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-60e12070-1ac6-5004-7db6-5ee32ce303ed -14:59.684 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-60e12070-1ac6-5004-7db6-5ee32ce303ed.dc1 -14:59.684 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:59.684 [INFO] TestAgent.server: Updating LAN server: server="Node-60e12070-1ac6-5004-7db6-5ee32ce303ed (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -14:59.684 [INFO] TestAgent.server: Updating LAN server: server="Node-60e12070-1ac6-5004-7db6-5ee32ce303ed (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -14:59.684 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-60e12070-1ac6-5004-7db6-5ee32ce303ed.dc1 area=wan -14:59.747 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:59.747 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:59.747 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:59.747 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:59.747 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:59.754 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-60e12070-1ac6-5004-7db6-5ee32ce303ed -14:59.754 [INFO] TestAgent.server: member joined, marking health alive: member=Node-60e12070-1ac6-5004-7db6-5ee32ce303ed -14:59.764 [INFO] TestAgent.server: federation state anti-entropy synced -14:59.765 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-60e12070-1ac6-5004-7db6-5ee32ce303ed -14:59.765 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-60e12070-1ac6-5004-7db6-5ee32ce303ed -14:59.874 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -14:59.874 [INFO] TestAgent: Synced node info -14:59.963 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:49854 latency=1.106471ms -14:59.971 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:49854 latency=910.626µs -14:59.973 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/85510834-614e-fa7b-614a-9c6997c8890f from=127.0.0.1:49856 latency=303.172µs -14:59.990 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/85510834-614e-fa7b-614a-9c6997c8890f from=127.0.0.1:49856 latency=17.16599ms -14:59.991 [INFO] TestAgent: Requesting shutdown -14:59.991 [INFO] TestAgent.server: shutting down server -14:59.991 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -14:59.991 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -14:59.991 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -14:59.991 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -14:59.991 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -14:59.991 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -14:59.991 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -14:59.991 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -14:59.991 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -14:59.991 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -14:59.991 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -14:59.991 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -14:59.991 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -14:59.991 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -14:59.991 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:14:59.991Z [INFO] agent.router.manager: shutting down -2021-10-31T23:14:59.991Z [INFO] agent.router.manager: shutting down -14:59.998 [INFO] TestAgent: consul server down -14:59.998 [INFO] TestAgent: shutdown complete -14:59.998 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=tcp -14:59.998 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=udp -14:59.998 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22254 network=tcp -15:00.501 [INFO] TestAgent: Waiting for endpoints to shut down -15:00.501 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenUpdateCommand_JSON (1.23s) - --- PASS: TestTokenUpdateCommand_JSON/update_with_policy_by_name (0.02s) +2022-12-04T05:49:21.093Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.105Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.109 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5a16a83a-99ad-dfc0-cef9-943823d28904 Address:127.0.0.1:27370}]" +49:21.109 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27370 [Follower]" leader= +49:21.111 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5a16a83a-99ad-dfc0-cef9-943823d28904.dc1 127.0.0.1 +49:21.112 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5a16a83a-99ad-dfc0-cef9-943823d28904 127.0.0.1 +2022-12-04T05:49:21.112Z [INFO] agent.router: Initializing LAN area manager +49:21.112 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5a16a83a-99ad-dfc0-cef9-943823d28904.dc1 area=wan +49:21.112 [INFO] TestAgent.server: Adding LAN server: server="Node-5a16a83a-99ad-dfc0-cef9-943823d28904 (Addr: tcp/127.0.0.1:27370) (DC: dc1)" +49:21.112 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27365 network=udp +49:21.112 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27365 network=tcp +49:21.113 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27366 network=tcp +49:21.113 [INFO] TestAgent: started state syncer +49:21.113 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27371 network=tcp +49:21.149 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.149 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27370 [Candidate]" term=2 +49:21.150 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.150 [DEBUG] TestAgent.server.raft: vote granted: from=5a16a83a-99ad-dfc0-cef9-943823d28904 term=2 tally=1 +49:21.150 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.150 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27370 [Leader]" +49:21.150 [INFO] TestAgent.server: cluster leadership acquired +49:21.150 [INFO] TestAgent.server: initializing acls +49:21.150 [INFO] TestAgent.server: New leader elected: payload=Node-5a16a83a-99ad-dfc0-cef9-943823d28904 +49:21.150 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:21.150 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:21.150 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:21.151 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:21.151 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:21.151 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:21.151 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-5a16a83a-99ad-dfc0-cef9-943823d28904 +49:21.151 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-5a16a83a-99ad-dfc0-cef9-943823d28904.dc1 +49:21.151 [INFO] TestAgent.server: Updating LAN server: server="Node-5a16a83a-99ad-dfc0-cef9-943823d28904 (Addr: tcp/127.0.0.1:27370) (DC: dc1)" +49:21.151 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-5a16a83a-99ad-dfc0-cef9-943823d28904.dc1 area=wan +49:21.151 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.167 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.167 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.167 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.167 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.167 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.167 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a16a83a-99ad-dfc0-cef9-943823d28904 +49:21.167 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5a16a83a-99ad-dfc0-cef9-943823d28904 +49:21.168 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.168 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5a16a83a-99ad-dfc0-cef9-943823d28904 +49:21.353 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.353 [INFO] TestAgent: Synced node info +49:21.520 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:37522 latency=6.48542ms +49:21.522 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:37522 latency=604.933µs +49:21.524 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/ad1f8419-b326-88ce-209b-6199d5c221de from=127.0.0.1:37524 latency=312.536µs +49:21.526 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/ad1f8419-b326-88ce-209b-6199d5c221de from=127.0.0.1:37524 latency=739.083µs +49:21.526 [INFO] TestAgent: Requesting shutdown +49:21.526 [INFO] TestAgent.server: shutting down server +49:21.526 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:21.526 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:21.526 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:21.526 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:21.526 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:21.526 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:21.526 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:21.526 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:21.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:21.526 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:21.526 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:21.526 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:21.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:21.526 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:21.526 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:21.526Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:21.526Z [INFO] agent.router.manager: shutting down +49:21.526 [INFO] TestAgent: consul server down +49:21.526 [INFO] TestAgent: shutdown complete +49:21.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27365 network=tcp +49:21.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27365 network=udp +49:21.526 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27366 network=tcp +49:22.027 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.027 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenUpdateCommand_JSON (0.97s) + --- PASS: TestTokenUpdateCommand_JSON/update_with_policy_by_name (0.00s) === RUN TestTokenUpdateCommand/legacy-upgrade -2021-10-31T23:14:59.510Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:14:59.597Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -14:59.631 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e85d9e7f-8398-665a-d1e9-db7c449ea412 Address:127.0.0.1:22265}]" -14:59.632 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e85d9e7f-8398-665a-d1e9-db7c449ea412.dc1 127.0.0.1 -14:59.632 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22265 [Follower]" leader= -14:59.632 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e85d9e7f-8398-665a-d1e9-db7c449ea412 127.0.0.1 -2021-10-31T23:14:59.632Z [INFO] agent.router: Initializing LAN area manager -14:59.633 [INFO] TestAgent.server: Adding LAN server: server="Node-e85d9e7f-8398-665a-d1e9-db7c449ea412 (Addr: tcp/127.0.0.1:22265) (DC: dc1)" -14:59.633 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e85d9e7f-8398-665a-d1e9-db7c449ea412.dc1 area=wan -14:59.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=udp -14:59.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=tcp -14:59.642 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22261 network=tcp -14:59.642 [INFO] TestAgent: started state syncer -14:59.642 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22266 network=tcp -14:59.677 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -14:59.678 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22265 [Candidate]" term=2 -14:59.678 [DEBUG] TestAgent.server.raft: votes: needed=1 -14:59.678 [DEBUG] TestAgent.server.raft: vote granted: from=e85d9e7f-8398-665a-d1e9-db7c449ea412 term=2 tally=1 -14:59.678 [INFO] TestAgent.server.raft: election won: tally=1 -14:59.678 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22265 [Leader]" -14:59.678 [INFO] TestAgent.server: cluster leadership acquired -14:59.678 [INFO] TestAgent.server: initializing acls -14:59.679 [INFO] TestAgent.server: Created ACL 'global-management' policy -14:59.679 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -14:59.679 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -14:59.679 [INFO] TestAgent.server: Created ACL anonymous token from configuration -14:59.680 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -14:59.680 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -14:59.680 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e85d9e7f-8398-665a-d1e9-db7c449ea412 -14:59.680 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e85d9e7f-8398-665a-d1e9-db7c449ea412.dc1 -14:59.680 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -14:59.682 [INFO] TestAgent.server: New leader elected: payload=Node-e85d9e7f-8398-665a-d1e9-db7c449ea412 -14:59.682 [INFO] TestAgent.server: Updating LAN server: server="Node-e85d9e7f-8398-665a-d1e9-db7c449ea412 (Addr: tcp/127.0.0.1:22265) (DC: dc1)" -14:59.682 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e85d9e7f-8398-665a-d1e9-db7c449ea412.dc1 area=wan -14:59.744 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -14:59.744 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -14:59.744 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -14:59.744 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -14:59.744 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -14:59.744 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e85d9e7f-8398-665a-d1e9-db7c449ea412 -14:59.744 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e85d9e7f-8398-665a-d1e9-db7c449ea412 -14:59.745 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e85d9e7f-8398-665a-d1e9-db7c449ea412 -14:59.746 [INFO] TestAgent.server: federation state anti-entropy synced -14:59.800 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:60286 latency=1.327488ms -14:59.814 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:60286 latency=13.330949ms -14:59.820 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/create from=127.0.0.1:60286 latency=1.878736ms -14:59.829 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/b317620d-db25-93e8-5a0d-8787c8211d53 from=127.0.0.1:60288 latency=416.854µs -14:59.834 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/b317620d-db25-93e8-5a0d-8787c8211d53 from=127.0.0.1:60288 latency=917.753µs -14:59.838 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/b317620d-db25-93e8-5a0d-8787c8211d53 from=127.0.0.1:60290 latency=527.153µs -14:59.867 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/b317620d-db25-93e8-5a0d-8787c8211d53 from=127.0.0.1:60290 latency=984.496µs -14:59.878 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/b317620d-db25-93e8-5a0d-8787c8211d53 from=127.0.0.1:60292 latency=443.37µs -14:59.887 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/b317620d-db25-93e8-5a0d-8787c8211d53 from=127.0.0.1:60292 latency=1.003889ms -14:59.889 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/b317620d-db25-93e8-5a0d-8787c8211d53 from=127.0.0.1:60294 latency=382.086µs -14:59.900 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/b317620d-db25-93e8-5a0d-8787c8211d53 from=127.0.0.1:60294 latency=1.349025ms -14:59.920 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/b317620d-db25-93e8-5a0d-8787c8211d53 from=127.0.0.1:60296 latency=1.880483ms -14:59.926 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/b317620d-db25-93e8-5a0d-8787c8211d53 from=127.0.0.1:60296 latency=5.425383ms -14:59.931 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=242.412µs -14:59.954 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -14:59.954 [INFO] TestAgent: Synced node info -14:59.962 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=457.498µs -14:59.990 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=348.141µs -15:00.022 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=537.894µs -15:00.050 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=356.465µs -15:00.078 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=302.967µs -15:00.110 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=343.736µs -15:00.138 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=321.325µs -15:00.166 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=280.215µs -15:00.194 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=276.928µs -15:00.222 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=423.608µs -15:00.250 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=437.477µs -15:00.282 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=465.261µs -15:00.314 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=372.398µs -15:00.346 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=353.332µs -15:00.378 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=238.423µs -15:00.406 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=421.082µs -15:00.442 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=326.99µs -15:00.478 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=282.82µs -15:00.510 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=287.755µs -15:00.538 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=289.398µs -15:00.570 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=275.544µs -15:00.602 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=357.142µs -15:00.634 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=274.671µs -15:00.667 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=379.946µs -15:00.694 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:60286 latency=562.221µs -15:00.697 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/52a69bce-937c-006c-fbee-fbc7a2101a0b from=127.0.0.1:60302 latency=362.006µs -15:00.715 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/52a69bce-937c-006c-fbee-fbc7a2101a0b from=127.0.0.1:60302 latency=17.961758ms -15:00.737 [INFO] TestAgent: Requesting shutdown -15:00.738 [INFO] TestAgent.server: shutting down server -15:00.738 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -15:00.738 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -15:00.738 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -15:00.738 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -15:00.738 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -15:00.738 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -15:00.738 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -15:00.738 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -15:00.738 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -15:00.738 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -15:00.738 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -15:00.738 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -15:00.738 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -15:00.738 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -15:00.738 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:15:00.738Z [INFO] agent.router.manager: shutting down -2021-10-31T23:15:00.738Z [INFO] agent.router.manager: shutting down -15:00.739 [INFO] TestAgent: consul server down -15:00.739 [INFO] TestAgent: shutdown complete -15:00.739 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=tcp -15:00.740 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=udp -15:00.740 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22261 network=tcp -15:01.240 [INFO] TestAgent: Waiting for endpoints to shut down -15:01.240 [INFO] TestAgent: Endpoints down ---- PASS: TestTokenUpdateCommand (1.96s) - --- PASS: TestTokenUpdateCommand/node-identity (0.02s) - --- PASS: TestTokenUpdateCommand/node-identity-merge (0.03s) - --- PASS: TestTokenUpdateCommand/policy-name (0.02s) - --- PASS: TestTokenUpdateCommand/policy-id (0.01s) - --- PASS: TestTokenUpdateCommand/merge-description (0.03s) - --- PASS: TestTokenUpdateCommand/legacy-upgrade (0.04s) +2022-12-04T05:49:21.093Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:21.105Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:21.109 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e701c6c0-9ca3-1243-e2bb-51d2e8a2776a Address:127.0.0.1:27363}]" +49:21.110 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= +49:21.110 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e701c6c0-9ca3-1243-e2bb-51d2e8a2776a.dc1 127.0.0.1 +49:21.111 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e701c6c0-9ca3-1243-e2bb-51d2e8a2776a 127.0.0.1 +2022-12-04T05:49:21.111Z [INFO] agent.router: Initializing LAN area manager +49:21.112 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e701c6c0-9ca3-1243-e2bb-51d2e8a2776a.dc1 area=wan +49:21.112 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp +49:21.112 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp +49:21.112 [INFO] TestAgent.server: Adding LAN server: server="Node-e701c6c0-9ca3-1243-e2bb-51d2e8a2776a (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +49:21.112 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp +49:21.112 [INFO] TestAgent: started state syncer +49:21.112 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp +49:21.162 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +49:21.179 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:21.179 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 +49:21.179 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:21.179 [DEBUG] TestAgent.server.raft: vote granted: from=e701c6c0-9ca3-1243-e2bb-51d2e8a2776a term=2 tally=1 +49:21.179 [INFO] TestAgent.server.raft: election won: tally=1 +49:21.179 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" +49:21.179 [INFO] TestAgent.server: cluster leadership acquired +49:21.179 [INFO] TestAgent.server: New leader elected: payload=Node-e701c6c0-9ca3-1243-e2bb-51d2e8a2776a +49:21.179 [INFO] TestAgent.server: initializing acls +49:21.179 [INFO] TestAgent.server: Created ACL 'global-management' policy +49:21.179 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +49:21.180 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +49:21.180 [INFO] TestAgent.server: Created ACL anonymous token from configuration +49:21.180 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +49:21.180 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +49:21.180 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-e701c6c0-9ca3-1243-e2bb-51d2e8a2776a +49:21.180 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-e701c6c0-9ca3-1243-e2bb-51d2e8a2776a.dc1 +49:21.180 [INFO] TestAgent.server: Updating LAN server: server="Node-e701c6c0-9ca3-1243-e2bb-51d2e8a2776a (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +49:21.180 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-e701c6c0-9ca3-1243-e2bb-51d2e8a2776a.dc1 area=wan +49:21.180 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:21.181 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:21.181 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:21.181 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:21.181 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:21.182 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:21.182 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e701c6c0-9ca3-1243-e2bb-51d2e8a2776a +49:21.182 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e701c6c0-9ca3-1243-e2bb-51d2e8a2776a +49:21.182 [INFO] TestAgent.server: federation state anti-entropy synced +49:21.182 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e701c6c0-9ca3-1243-e2bb-51d2e8a2776a +49:21.234 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:21.235 [INFO] TestAgent: Synced node info +49:21.362 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/policy from=127.0.0.1:59676 latency=6.572916ms +49:21.365 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:59676 latency=878.095µs +49:21.366 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/create from=127.0.0.1:59676 latency=1.241211ms +49:21.368 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/ddb12a6e-60e9-c9f3-9a6d-ca95228c667f from=127.0.0.1:59678 latency=308.365µs +49:21.369 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/ddb12a6e-60e9-c9f3-9a6d-ca95228c667f from=127.0.0.1:59678 latency=551.595µs +49:21.371 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/ddb12a6e-60e9-c9f3-9a6d-ca95228c667f from=127.0.0.1:59680 latency=288.562µs +49:21.372 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/ddb12a6e-60e9-c9f3-9a6d-ca95228c667f from=127.0.0.1:59680 latency=554.907µs +49:21.373 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/ddb12a6e-60e9-c9f3-9a6d-ca95228c667f from=127.0.0.1:59682 latency=171.417µs +49:21.374 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/ddb12a6e-60e9-c9f3-9a6d-ca95228c667f from=127.0.0.1:59682 latency=497.858µs +49:21.375 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/ddb12a6e-60e9-c9f3-9a6d-ca95228c667f from=127.0.0.1:59684 latency=161.591µs +49:21.376 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/ddb12a6e-60e9-c9f3-9a6d-ca95228c667f from=127.0.0.1:59684 latency=436.873µs +49:21.378 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/ddb12a6e-60e9-c9f3-9a6d-ca95228c667f from=127.0.0.1:59686 latency=275.979µs +49:21.379 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/ddb12a6e-60e9-c9f3-9a6d-ca95228c667f from=127.0.0.1:59686 latency=557.208µs +49:21.380 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=179.716µs +49:21.406 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=206.492µs +49:21.433 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=235.399µs +49:21.459 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=225.529µs +49:21.485 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=280.479µs +49:21.514 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=530.56µs +49:21.541 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=203.566µs +49:21.568 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=625.225µs +49:21.594 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=260.24µs +49:21.620 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=297.76µs +49:21.646 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=322.197µs +49:21.673 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=389.87µs +49:21.699 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=232.946µs +49:21.726 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=240.326µs +49:21.752 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=276.338µs +49:21.778 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=263.2µs +49:21.805 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=292.8µs +49:21.831 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=251.057µs +49:21.857 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=244.906µs +49:21.883 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=204.931µs +49:21.909 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=256.195µs +49:21.935 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=271.363µs +49:21.961 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=263.033µs +49:21.988 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=276.275µs +49:22.015 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=381.043µs +49:22.041 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=190.59µs +49:22.067 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=289.003µs +49:22.100 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=241.255µs +49:22.126 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=279.649µs +49:22.153 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=313.646µs +49:22.180 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=1.111758ms +49:22.206 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/self from=127.0.0.1:59676 latency=229.259µs +49:22.208 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/acl/token/fa756141-b711-15be-0b96-28e0732cdfe1 from=127.0.0.1:59688 latency=331.628µs +49:22.210 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token/fa756141-b711-15be-0b96-28e0732cdfe1 from=127.0.0.1:59688 latency=664.35µs +49:22.210 [INFO] TestAgent: Requesting shutdown +49:22.210 [INFO] TestAgent.server: shutting down server +49:22.210 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:22.210 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +49:22.210 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +49:22.210 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:22.210 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:22.210 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:22.210 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:22.210 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:22.210 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:22.210 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:22.210 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +49:22.210 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +49:22.210 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:22.210 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:22.210 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:22.210Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:22.210Z [INFO] agent.router.manager: shutting down +49:22.210 [INFO] TestAgent: consul server down +49:22.210 [INFO] TestAgent: shutdown complete +49:22.210 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp +49:22.210 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp +49:22.210 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp +49:22.712 [INFO] TestAgent: Waiting for endpoints to shut down +49:22.712 [INFO] TestAgent: Endpoints down +--- PASS: TestTokenUpdateCommand (1.66s) + --- PASS: TestTokenUpdateCommand/node-identity (0.00s) + --- PASS: TestTokenUpdateCommand/node-identity-merge (0.00s) + --- PASS: TestTokenUpdateCommand/policy-name (0.00s) + --- PASS: TestTokenUpdateCommand/policy-id (0.00s) + --- PASS: TestTokenUpdateCommand/merge-description (0.00s) + --- PASS: TestTokenUpdateCommand/legacy-upgrade (0.00s) PASS -ok github.com/hashicorp/consul/command/acl/token/update 2.857s +ok github.com/hashicorp/consul/command/acl/token/update 1.724s === RUN TestConfigFail === PAUSE TestConfigFail === RUN TestRetryJoin [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:15:09.031Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:15:09.046Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -15:09.048 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e583bce1-58ad-0dc0-9aac-75496c57fb8c Address:127.0.0.1:22265}]" -15:09.048 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e583bce1-58ad-0dc0-9aac-75496c57fb8c.dc1 127.0.0.1 -15:09.049 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22265 [Follower]" leader= -15:09.050 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e583bce1-58ad-0dc0-9aac-75496c57fb8c 127.0.0.1 -2021-10-31T23:15:09.050Z [INFO] agent.router: Initializing LAN area manager -15:09.050 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=udp -15:09.050 [INFO] TestAgent.server: Adding LAN server: server="Node-e583bce1-58ad-0dc0-9aac-75496c57fb8c (Addr: tcp/127.0.0.1:22265) (DC: dc1)" -15:09.050 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e583bce1-58ad-0dc0-9aac-75496c57fb8c.dc1 area=wan -15:09.050 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=tcp -15:09.057 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22261 network=tcp -15:09.057 [INFO] TestAgent: started state syncer -15:09.057 [INFO] TestAgent: Retry join is supported for the following discovery methods: cluster=LAN discovery_methods="aliyun aws gce mdns os packet" -15:09.057 [INFO] TestAgent: Joining cluster...: cluster=LAN -15:09.057 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22266 network=tcp -15:09.057 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:22256] -15:09.057 [INFO] TestAgent: Retry join is supported for the following discovery methods: cluster=WAN discovery_methods="aliyun aws gce mdns os packet" -15:09.057 [INFO] TestAgent: Joining cluster...: cluster=WAN -15:09.057 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:22257] -15:09.058 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:22257 -15:09.058 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:22256 -15:09.061 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251 127.0.0.1 -15:09.061 [INFO] TestAgent.server: Adding LAN server: server="Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251 (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -15:09.062 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -15:09.062 [DEBUG] TestAgent: systemd notify failed: error="No socket" -15:09.062 [INFO] TestAgent: Join cluster completed. Synced with initial agents: cluster=LAN num_agents=1 -15:09.062 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251.dc1 127.0.0.1 -15:09.062 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -15:09.062 [INFO] TestAgent: Join cluster completed. Synced with initial agents: cluster=WAN num_agents=1 -15:09.062 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251.dc1 area=wan -15:09.062 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251.dc1 127.0.0.1:22257 -15:09.063 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251.dc1 address=127.0.0.1:22257 -15:09.118 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -15:09.118 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22265 [Candidate]" term=2 -15:09.119 [DEBUG] TestAgent.server.raft: votes: needed=1 -15:09.119 [DEBUG] TestAgent.server.raft: vote granted: from=e583bce1-58ad-0dc0-9aac-75496c57fb8c term=2 tally=1 -15:09.119 [INFO] TestAgent.server.raft: election won: tally=1 -15:09.119 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22265 [Leader]" -15:09.119 [INFO] TestAgent.server: cluster leadership acquired -15:09.119 [INFO] TestAgent.server: New leader elected: payload=Node-e583bce1-58ad-0dc0-9aac-75496c57fb8c -15:09.119 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22265 -15:09.120 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -15:09.121 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -15:09.121 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -15:09.121 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -15:09.121 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -15:09.121 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -15:09.121 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251 other=Node-e583bce1-58ad-0dc0-9aac-75496c57fb8c -15:09.121 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251 -15:09.122 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-e583bce1-58ad-0dc0-9aac-75496c57fb8c other=Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251 -15:09.122 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e583bce1-58ad-0dc0-9aac-75496c57fb8c -15:09.123 [INFO] TestAgent.server: federation state anti-entropy synced -15:09.154 [INFO] TestAgent: Requesting shutdown -15:09.154 [INFO] TestAgent.server: shutting down server -15:09.154 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -15:09.154 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -15:09.154 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -15:09.154 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -15:09.154 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -15:09.154 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -15:09.154 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -15:09.154 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -15:09.154 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -15:09.154 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -15:09.154 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -15:09.154 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:15:09.154Z [INFO] agent.router.manager: shutting down -2021-10-31T23:15:09.154Z [INFO] agent.router.manager: shutting down -15:09.154 [INFO] TestAgent: consul server down -15:09.154 [INFO] TestAgent: shutdown complete -15:09.154 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=tcp -15:09.154 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=udp -15:09.154 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22261 network=tcp -15:09.654 [INFO] TestAgent: Waiting for endpoints to shut down -15:09.654 [INFO] TestAgent: Endpoints down -2021-10-31T23:15:08.627Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:15:08.641Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -15:08.644 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5bb8e5a4-a4a4-0b26-68f2-b486b035b251 Address:127.0.0.1:22258}]" -15:08.644 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22258 [Follower]" leader= -15:08.644 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251.dc1 127.0.0.1 -15:08.645 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251 127.0.0.1 -2021-10-31T23:15:08.645Z [INFO] agent.router: Initializing LAN area manager -15:08.645 [INFO] TestAgent.server: Adding LAN server: server="Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251 (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -15:08.645 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251.dc1 area=wan -15:08.645 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=tcp -15:08.645 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=udp -15:08.646 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22254 network=tcp -15:08.646 [INFO] TestAgent: started state syncer -15:08.646 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22259 network=tcp -15:08.714 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -15:08.714 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22258 [Candidate]" term=2 -15:08.714 [DEBUG] TestAgent.server.raft: votes: needed=1 -15:08.714 [DEBUG] TestAgent.server.raft: vote granted: from=5bb8e5a4-a4a4-0b26-68f2-b486b035b251 term=2 tally=1 -15:08.714 [INFO] TestAgent.server.raft: election won: tally=1 -15:08.714 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22258 [Leader]" -15:08.714 [INFO] TestAgent.server: cluster leadership acquired -15:08.715 [INFO] TestAgent.server: New leader elected: payload=Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251 -15:08.715 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22258 -15:08.716 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -15:08.733 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -15:08.733 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -15:08.733 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -15:08.733 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -15:08.733 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -15:08.733 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251 -15:08.733 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251 -15:08.735 [INFO] TestAgent.server: federation state anti-entropy synced -15:08.778 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -15:08.779 [INFO] TestAgent: Synced node info -15:08.779 [DEBUG] TestAgent: Node info in sync -15:09.058 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:41654 -15:09.058 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:53432 -15:09.060 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e583bce1-58ad-0dc0-9aac-75496c57fb8c.dc1 127.0.0.1 -15:09.060 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e583bce1-58ad-0dc0-9aac-75496c57fb8c.dc1 area=wan -15:09.061 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e583bce1-58ad-0dc0-9aac-75496c57fb8c 127.0.0.1 -15:09.061 [INFO] TestAgent.server: Adding LAN server: server="Node-e583bce1-58ad-0dc0-9aac-75496c57fb8c (Addr: tcp/127.0.0.1:22265) (DC: dc1)" -15:09.061 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-e583bce1-58ad-0dc0-9aac-75496c57fb8c other=Node-5bb8e5a4-a4a4-0b26-68f2-b486b035b251 -15:09.061 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e583bce1-58ad-0dc0-9aac-75496c57fb8c -15:09.062 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:41656 -15:09.519 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -15:09.519 [DEBUG] TestAgent: Node info in sync -15:09.654 [INFO] TestAgent: Requesting shutdown -15:09.655 [INFO] TestAgent.server: shutting down server -15:09.655 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -15:09.655 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -15:09.655 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -15:09.655 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -15:09.655 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -15:09.655 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -15:09.655 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -15:09.655 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -15:09.655 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -15:09.655 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -15:09.655 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:15:09.655Z [INFO] agent.router.manager: shutting down -2021-10-31T23:15:09.655Z [INFO] agent.router.manager: shutting down -15:09.655 [INFO] TestAgent: consul server down -15:09.655 [INFO] TestAgent: shutdown complete -15:09.655 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=tcp -15:09.655 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=udp -15:09.655 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22254 network=tcp -15:10.145 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-e583bce1-58ad-0dc0-9aac-75496c57fb8c (timeout reached) -15:10.157 [INFO] TestAgent: Waiting for endpoints to shut down -15:10.157 [INFO] TestAgent: Endpoints down ---- PASS: TestRetryJoin (1.56s) +2022-12-04T05:49:26.703Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.722Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.723 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6b8fb8cd-4cc5-3688-ad1d-233ef8f81404 Address:127.0.0.1:10013}]" +49:26.724 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10013 [Follower]" leader= +49:26.724 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6b8fb8cd-4cc5-3688-ad1d-233ef8f81404.dc1 127.0.0.1 +49:26.725 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6b8fb8cd-4cc5-3688-ad1d-233ef8f81404 127.0.0.1 +2022-12-04T05:49:26.725Z [INFO] agent.router: Initializing LAN area manager +49:26.725 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10008 network=udp +49:26.725 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6b8fb8cd-4cc5-3688-ad1d-233ef8f81404.dc1 area=wan +49:26.725 [INFO] TestAgent.server: Adding LAN server: server="Node-6b8fb8cd-4cc5-3688-ad1d-233ef8f81404 (Addr: tcp/127.0.0.1:10013) (DC: dc1)" +49:26.725 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10008 network=tcp +49:26.726 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10009 network=tcp +49:26.726 [INFO] TestAgent: started state syncer +49:26.726 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10014 network=tcp +49:26.726 [INFO] TestAgent: Retry join is supported for the following discovery methods: cluster=LAN discovery_methods="aliyun aws gce mdns os packet" +49:26.726 [INFO] TestAgent: Joining cluster...: cluster=LAN +49:26.726 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:10004] +49:26.726 [INFO] TestAgent: Retry join is supported for the following discovery methods: cluster=WAN discovery_methods="aliyun aws gce mdns os packet" +49:26.726 [INFO] TestAgent: Joining cluster...: cluster=WAN +49:26.726 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:10005] +49:26.726 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:10004 +49:26.726 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:10005 +49:26.729 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-db4f4267-f70e-8740-c60b-e9b9a4abb10b.dc1 127.0.0.1 +49:26.729 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:43708 +49:26.729 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +49:26.729 [INFO] TestAgent: Join cluster completed. Synced with initial agents: cluster=WAN num_agents=1 +49:26.729 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-db4f4267-f70e-8740-c60b-e9b9a4abb10b 127.0.0.1 +49:26.729 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-db4f4267-f70e-8740-c60b-e9b9a4abb10b.dc1 area=wan +49:26.729 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +49:26.729 [DEBUG] TestAgent: systemd notify failed: error="No socket" +49:26.729 [INFO] TestAgent: Join cluster completed. Synced with initial agents: cluster=LAN num_agents=1 +49:26.729 [INFO] TestAgent.server: Adding LAN server: server="Node-db4f4267-f70e-8740-c60b-e9b9a4abb10b (Addr: tcp/127.0.0.1:10006) (DC: dc1)" +49:26.777 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.777 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10013 [Candidate]" term=2 +49:26.777 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.777 [DEBUG] TestAgent.server.raft: vote granted: from=6b8fb8cd-4cc5-3688-ad1d-233ef8f81404 term=2 tally=1 +49:26.777 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.777 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10013 [Leader]" +49:26.778 [INFO] TestAgent.server: cluster leadership acquired +49:26.778 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10013 +49:26.778 [INFO] TestAgent.server: New leader elected: payload=Node-6b8fb8cd-4cc5-3688-ad1d-233ef8f81404 +49:26.779 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.780 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.780 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.780 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.780 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.780 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.780 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-6b8fb8cd-4cc5-3688-ad1d-233ef8f81404 other=Node-db4f4267-f70e-8740-c60b-e9b9a4abb10b +49:26.780 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6b8fb8cd-4cc5-3688-ad1d-233ef8f81404 +49:26.781 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-db4f4267-f70e-8740-c60b-e9b9a4abb10b other=Node-6b8fb8cd-4cc5-3688-ad1d-233ef8f81404 +49:26.781 [INFO] TestAgent.server: member joined, marking health alive: member=Node-db4f4267-f70e-8740-c60b-e9b9a4abb10b +49:26.781 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.801 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:26.802 [INFO] TestAgent: Synced node info +49:26.814 [DEBUG] TestAgent.server.serf.lan: serf: messageUserEventType: consul:new-leader +49:26.858 [INFO] TestAgent: Requesting shutdown +49:26.858 [INFO] TestAgent.server: shutting down server +49:26.858 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:26.858 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:26.858 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:26.858 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:26.858 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:26.858 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:26.858 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:26.858 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:26.858 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:26.858 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:26.858 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:26.858Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:26.858Z [INFO] agent.router.manager: shutting down +49:26.858 [INFO] TestAgent: consul server down +49:26.858 [INFO] TestAgent: shutdown complete +49:26.858 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10008 network=tcp +49:26.858 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10008 network=udp +49:26.858 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10009 network=tcp +49:27.359 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.359 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:26.593Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:26.606Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:26.609 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:db4f4267-f70e-8740-c60b-e9b9a4abb10b Address:127.0.0.1:10006}]" +49:26.610 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10006 [Follower]" leader= +49:26.610 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-db4f4267-f70e-8740-c60b-e9b9a4abb10b.dc1 127.0.0.1 +49:26.612 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-db4f4267-f70e-8740-c60b-e9b9a4abb10b 127.0.0.1 +2022-12-04T05:49:26.612Z [INFO] agent.router: Initializing LAN area manager +49:26.613 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=tcp +49:26.613 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-db4f4267-f70e-8740-c60b-e9b9a4abb10b.dc1 area=wan +49:26.613 [INFO] TestAgent.server: Adding LAN server: server="Node-db4f4267-f70e-8740-c60b-e9b9a4abb10b (Addr: tcp/127.0.0.1:10006) (DC: dc1)" +49:26.613 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=udp +49:26.614 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10002 network=tcp +49:26.615 [INFO] TestAgent: started state syncer +49:26.615 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10007 network=tcp +49:26.647 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:26.647 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10006 [Candidate]" term=2 +49:26.647 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:26.647 [DEBUG] TestAgent.server.raft: vote granted: from=db4f4267-f70e-8740-c60b-e9b9a4abb10b term=2 tally=1 +49:26.647 [INFO] TestAgent.server.raft: election won: tally=1 +49:26.647 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10006 [Leader]" +49:26.647 [INFO] TestAgent.server: cluster leadership acquired +49:26.648 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10006 +49:26.648 [INFO] TestAgent.server: New leader elected: payload=Node-db4f4267-f70e-8740-c60b-e9b9a4abb10b +49:26.649 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:26.675 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:26.675 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:26.675 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:26.675 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:26.675 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:26.676 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-db4f4267-f70e-8740-c60b-e9b9a4abb10b +49:26.676 [INFO] TestAgent.server: member joined, marking health alive: member=Node-db4f4267-f70e-8740-c60b-e9b9a4abb10b +49:26.676 [INFO] TestAgent.server: federation state anti-entropy synced +49:26.726 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:48814 +49:26.726 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:35102 +49:26.727 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6b8fb8cd-4cc5-3688-ad1d-233ef8f81404 127.0.0.1 +49:26.727 [INFO] TestAgent.server: Adding LAN server: server="Node-6b8fb8cd-4cc5-3688-ad1d-233ef8f81404 (Addr: tcp/127.0.0.1:10013) (DC: dc1)" +49:26.728 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-6b8fb8cd-4cc5-3688-ad1d-233ef8f81404 other=Node-db4f4267-f70e-8740-c60b-e9b9a4abb10b +49:26.728 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6b8fb8cd-4cc5-3688-ad1d-233ef8f81404 +49:26.728 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6b8fb8cd-4cc5-3688-ad1d-233ef8f81404.dc1 127.0.0.1 +49:26.728 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6b8fb8cd-4cc5-3688-ad1d-233ef8f81404.dc1 area=wan +49:26.728 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-6b8fb8cd-4cc5-3688-ad1d-233ef8f81404.dc1 127.0.0.1:10012 +49:26.737 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-6b8fb8cd-4cc5-3688-ad1d-233ef8f81404.dc1 address=127.0.0.1:10012 +49:26.797 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:26.798 [INFO] TestAgent: Synced node info +49:26.798 [DEBUG] TestAgent: Node info in sync +49:27.359 [INFO] TestAgent: Requesting shutdown +49:27.359 [INFO] TestAgent.server: shutting down server +49:27.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.359 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.359 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.359 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.359 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.359 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.359 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.359 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.359 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:27.359 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:27.359Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.359Z [INFO] agent.router.manager: shutting down +49:27.359 [INFO] TestAgent: consul server down +49:27.359 [INFO] TestAgent: shutdown complete +49:27.359 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=tcp +49:27.359 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=udp +49:27.359 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10002 network=tcp +49:27.859 [INFO] TestAgent: Waiting for endpoints to shut down +49:27.860 [INFO] TestAgent: Endpoints down +--- PASS: TestRetryJoin (1.30s) === RUN TestRetryJoinFail === PAUSE TestRetryJoinFail === RUN TestRetryJoinWanFail @@ -77933,41 +78114,41 @@ === RUN TestBadDataDirPermissions === PAUSE TestBadDataDirPermissions === CONT TestConfigFail -=== RUN TestConfigFail/agent_-server_-bind=10.0.0.1_-datacenter= === CONT TestProtectDataDir +=== CONT TestRetryJoinFail === CONT TestBadDataDirPermissions +=== RUN TestConfigFail/agent_-server_-bind=10.0.0.1_-datacenter= === CONT TestRetryJoinWanFail -=== CONT TestRetryJoinFail ---- PASS: TestProtectDataDir (0.05s) ---- PASS: TestBadDataDirPermissions (0.06s) ---- PASS: TestRetryJoinFail (0.12s) +--- PASS: TestBadDataDirPermissions (0.02s) +--- PASS: TestProtectDataDir (0.02s) +--- PASS: TestRetryJoinFail (0.05s) === RUN TestConfigFail/agent_-server_-bind=10.0.0.1_-datacenter=foo_some-other-arg === RUN TestConfigFail/agent_-server_-bind=10.0.0.1 ---- PASS: TestRetryJoinWanFail (4.73s) -=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul261510199_-advertise_0.0.0.0_-bind_10.0.0.1 -=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul261510199_-advertise_::_-bind_10.0.0.1 -=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul261510199_-advertise_[::]_-bind_10.0.0.1 -=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul261510199_-advertise-wan_0.0.0.0_-bind_10.0.0.1 -=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul261510199_-advertise-wan_::_-bind_10.0.0.1 -=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul261510199_-advertise-wan_[::]_-bind_10.0.0.1 ---- PASS: TestConfigFail (12.34s) - --- PASS: TestConfigFail/agent_-server_-bind=10.0.0.1_-datacenter= (0.18s) - --- PASS: TestConfigFail/agent_-server_-bind=10.0.0.1_-datacenter=foo_some-other-arg (4.55s) - --- PASS: TestConfigFail/agent_-server_-bind=10.0.0.1 (2.20s) - --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul261510199_-advertise_0.0.0.0_-bind_10.0.0.1 (1.12s) - --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul261510199_-advertise_::_-bind_10.0.0.1 (1.23s) - --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul261510199_-advertise_[::]_-bind_10.0.0.1 (0.27s) - --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul261510199_-advertise-wan_0.0.0.0_-bind_10.0.0.1 (1.15s) - --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul261510199_-advertise-wan_::_-bind_10.0.0.1 (1.26s) - --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul261510199_-advertise-wan_[::]_-bind_10.0.0.1 (0.38s) +=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul408529570_-advertise_0.0.0.0_-bind_10.0.0.1 +=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul408529570_-advertise_::_-bind_10.0.0.1 +=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul408529570_-advertise_[::]_-bind_10.0.0.1 +=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul408529570_-advertise-wan_0.0.0.0_-bind_10.0.0.1 +=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul408529570_-advertise-wan_::_-bind_10.0.0.1 +--- PASS: TestRetryJoinWanFail (0.56s) +=== RUN TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul408529570_-advertise-wan_[::]_-bind_10.0.0.1 +--- PASS: TestConfigFail (0.63s) + --- PASS: TestConfigFail/agent_-server_-bind=10.0.0.1_-datacenter= (0.10s) + --- PASS: TestConfigFail/agent_-server_-bind=10.0.0.1_-datacenter=foo_some-other-arg (0.05s) + --- PASS: TestConfigFail/agent_-server_-bind=10.0.0.1 (0.05s) + --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul408529570_-advertise_0.0.0.0_-bind_10.0.0.1 (0.06s) + --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul408529570_-advertise_::_-bind_10.0.0.1 (0.08s) + --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul408529570_-advertise_[::]_-bind_10.0.0.1 (0.08s) + --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul408529570_-advertise-wan_0.0.0.0_-bind_10.0.0.1 (0.09s) + --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul408529570_-advertise-wan_::_-bind_10.0.0.1 (0.07s) + --- PASS: TestConfigFail/agent_-server_-data-dir_/tmp/consul-test/TestConfigFail-consul408529570_-advertise-wan_[::]_-bind_10.0.0.1 (0.06s) PASS -ok github.com/hashicorp/consul/command/agent 13.975s +ok github.com/hashicorp/consul/command/agent 1.981s === RUN TestCatalogCommand_noTabs === PAUSE TestCatalogCommand_noTabs === CONT TestCatalogCommand_noTabs --- PASS: TestCatalogCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/catalog 0.051s +ok github.com/hashicorp/consul/command/catalog 0.005s === RUN TestCatalogListDatacentersCommand_noTabs === PAUSE TestCatalogListDatacentersCommand_noTabs === RUN TestCatalogListDatacentersCommand_Validation @@ -77976,73 +78157,73 @@ === PAUSE TestCatalogListDatacentersCommand === CONT TestCatalogListDatacentersCommand_noTabs === CONT TestCatalogListDatacentersCommand -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestCatalogListDatacentersCommand_Validation +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... --- PASS: TestCatalogListDatacentersCommand_noTabs (0.00s) --- PASS: TestCatalogListDatacentersCommand_Validation (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:15:31.494Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:15:31.537Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -15:31.554 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:53db43c2-ee1e-3754-1f80-f238e15f7f3b Address:127.0.0.1:14090}]" -15:31.555 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-53db43c2-ee1e-3754-1f80-f238e15f7f3b.dc1 127.0.0.1 -15:31.555 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-53db43c2-ee1e-3754-1f80-f238e15f7f3b 127.0.0.1 -2021-10-31T23:15:31.556Z [INFO] agent.router: Initializing LAN area manager -15:31.556 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp -15:31.556 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= -15:31.556 [INFO] TestAgent.server: Adding LAN server: server="Node-53db43c2-ee1e-3754-1f80-f238e15f7f3b (Addr: tcp/127.0.0.1:14090) (DC: dc1)" -15:31.556 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-53db43c2-ee1e-3754-1f80-f238e15f7f3b.dc1 area=wan -15:31.556 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp -15:31.557 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp -15:31.557 [INFO] TestAgent: started state syncer -15:31.557 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp -15:31.617 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -15:31.617 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 -15:31.617 [DEBUG] TestAgent.server.raft: votes: needed=1 -15:31.617 [DEBUG] TestAgent.server.raft: vote granted: from=53db43c2-ee1e-3754-1f80-f238e15f7f3b term=2 tally=1 -15:31.617 [INFO] TestAgent.server.raft: election won: tally=1 -15:31.617 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" -15:31.617 [INFO] TestAgent.server: cluster leadership acquired -15:31.618 [INFO] TestAgent.server: New leader elected: payload=Node-53db43c2-ee1e-3754-1f80-f238e15f7f3b -15:31.618 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14090 -15:31.618 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -15:31.689 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -15:31.689 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -15:31.689 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -15:31.689 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -15:31.689 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -15:31.689 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-53db43c2-ee1e-3754-1f80-f238e15f7f3b -15:31.689 [INFO] TestAgent.server: member joined, marking health alive: member=Node-53db43c2-ee1e-3754-1f80-f238e15f7f3b -15:31.696 [INFO] TestAgent.server: federation state anti-entropy synced -15:31.696 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -15:31.697 [INFO] TestAgent: Synced node info -15:31.697 [DEBUG] TestAgent: Node info in sync -15:31.838 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/datacenters from=127.0.0.1:49536 latency=427.064µs -15:31.839 [INFO] TestAgent: Requesting shutdown -15:31.839 [INFO] TestAgent.server: shutting down server -15:31.839 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -15:31.839 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -15:31.839 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -15:31.839 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -15:31.839 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -15:31.839 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -15:31.839 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -15:31.839 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -15:31.839 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -15:31.839 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -15:31.839 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:15:31.839Z [INFO] agent.router.manager: shutting down -2021-10-31T23:15:31.839Z [INFO] agent.router.manager: shutting down -15:31.839 [INFO] TestAgent: consul server down -15:31.839 [INFO] TestAgent: shutdown complete -15:31.839 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp -15:31.839 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp -15:31.839 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp -15:32.341 [INFO] TestAgent: Waiting for endpoints to shut down -15:32.341 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogListDatacentersCommand (1.00s) +2022-12-04T05:49:27.433Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:27.447Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:27.449 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5b7edacc-4156-1607-d2c4-1b3229fde183 Address:127.0.0.1:28384}]" +49:27.449 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28384 [Follower]" leader= +49:27.449 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5b7edacc-4156-1607-d2c4-1b3229fde183.dc1 127.0.0.1 +49:27.449 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5b7edacc-4156-1607-d2c4-1b3229fde183 127.0.0.1 +2022-12-04T05:49:27.450Z [INFO] agent.router: Initializing LAN area manager +49:27.450 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=udp +49:27.450 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5b7edacc-4156-1607-d2c4-1b3229fde183.dc1 area=wan +49:27.450 [INFO] TestAgent.server: Adding LAN server: server="Node-5b7edacc-4156-1607-d2c4-1b3229fde183 (Addr: tcp/127.0.0.1:28384) (DC: dc1)" +49:27.450 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=tcp +49:27.450 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28380 network=tcp +49:27.451 [INFO] TestAgent: started state syncer +49:27.451 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28385 network=tcp +49:27.492 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:27.492 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28384 [Candidate]" term=2 +49:27.492 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:27.492 [DEBUG] TestAgent.server.raft: vote granted: from=5b7edacc-4156-1607-d2c4-1b3229fde183 term=2 tally=1 +49:27.492 [INFO] TestAgent.server.raft: election won: tally=1 +49:27.492 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28384 [Leader]" +49:27.492 [INFO] TestAgent.server: cluster leadership acquired +49:27.492 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28384 +49:27.493 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:27.496 [INFO] TestAgent.server: New leader elected: payload=Node-5b7edacc-4156-1607-d2c4-1b3229fde183 +49:27.515 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:27.515 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:27.515 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:27.515 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:27.515 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:27.515 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5b7edacc-4156-1607-d2c4-1b3229fde183 +49:27.515 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5b7edacc-4156-1607-d2c4-1b3229fde183 +49:27.517 [INFO] TestAgent.server: federation state anti-entropy synced +49:27.604 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:27.604 [INFO] TestAgent: Synced node info +49:27.604 [DEBUG] TestAgent: Node info in sync +49:27.736 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/datacenters from=127.0.0.1:54782 latency=4.59313ms +49:27.738 [INFO] TestAgent: Requesting shutdown +49:27.738 [INFO] TestAgent.server: shutting down server +49:27.738 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:27.738 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:27.738 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:27.738 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:27.738 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:27.738 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:27.738 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:27.738 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:27.738 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:27.738 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:27.738 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:27.738Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:27.738Z [INFO] agent.router.manager: shutting down +49:27.738 [INFO] TestAgent: consul server down +49:27.738 [INFO] TestAgent: shutdown complete +49:27.738 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=tcp +49:27.738 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=udp +49:27.738 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28380 network=tcp +49:28.239 [INFO] TestAgent: Waiting for endpoints to shut down +49:28.239 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogListDatacentersCommand (0.85s) PASS -ok github.com/hashicorp/consul/command/catalog/list/dc 1.489s +ok github.com/hashicorp/consul/command/catalog/list/dc 0.897s === RUN TestCatalogListNodesCommand_noTabs === PAUSE TestCatalogListNodesCommand_noTabs === RUN TestCatalogListNodesCommand_Validation @@ -78053,9 +78234,9 @@ catalog_list_nodes_test.go:168: DM-skipped --- SKIP: TestCatalogListNodesCommand_verticalBar (0.00s) === CONT TestCatalogListNodesCommand_noTabs ---- PASS: TestCatalogListNodesCommand_noTabs (0.00s) === CONT TestCatalogListNodesCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestCatalogListNodesCommand_noTabs (0.00s) === CONT TestCatalogListNodesCommand_Validation --- PASS: TestCatalogListNodesCommand_Validation (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] @@ -78067,82 +78248,80 @@ === RUN TestCatalogListNodesCommand/near === RUN TestCatalogListNodesCommand/service_present === RUN TestCatalogListNodesCommand/service_missing -2021-10-31T23:15:29.998Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:15:30.056Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -15:30.063 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:051f3dcf-8c5a-347c-e255-d5e326f9188a Address:127.0.0.1:15111}]" -15:30.071 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-051f3dcf-8c5a-347c-e255-d5e326f9188a.dc1 127.0.0.1 -15:30.073 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-051f3dcf-8c5a-347c-e255-d5e326f9188a 127.0.0.1 -2021-10-31T23:15:30.073Z [INFO] agent.router: Initializing LAN area manager -15:30.079 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=tcp -15:30.079 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=udp -15:30.079 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15111 [Follower]" leader= -15:30.080 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15107 network=tcp -15:30.080 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-051f3dcf-8c5a-347c-e255-d5e326f9188a.dc1 area=wan -15:30.080 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15112 network=tcp -15:30.080 [INFO] TestAgent: started state syncer -15:30.091 [INFO] TestAgent.server: Adding LAN server: server="Node-051f3dcf-8c5a-347c-e255-d5e326f9188a (Addr: tcp/127.0.0.1:15111) (DC: dc1)" -15:30.121 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -15:30.121 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15111 [Candidate]" term=2 -15:30.122 [DEBUG] TestAgent.server.raft: votes: needed=1 -15:30.122 [DEBUG] TestAgent.server.raft: vote granted: from=051f3dcf-8c5a-347c-e255-d5e326f9188a term=2 tally=1 -15:30.122 [INFO] TestAgent.server.raft: election won: tally=1 -15:30.122 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15111 [Leader]" -15:30.122 [INFO] TestAgent.server: cluster leadership acquired -15:30.122 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15111 -15:30.123 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -15:30.125 [INFO] TestAgent.server: New leader elected: payload=Node-051f3dcf-8c5a-347c-e255-d5e326f9188a -15:30.201 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -15:30.201 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -15:30.201 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -15:30.201 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -15:30.201 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -15:30.201 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-051f3dcf-8c5a-347c-e255-d5e326f9188a -15:30.201 [INFO] TestAgent.server: member joined, marking health alive: member=Node-051f3dcf-8c5a-347c-e255-d5e326f9188a -15:30.214 [INFO] TestAgent.server: federation state anti-entropy synced -15:30.358 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -15:30.358 [INFO] TestAgent: Synced node info -15:30.460 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:60866 latency=1.181714ms -15:30.462 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -15:30.462 [DEBUG] TestAgent: Node info in sync -15:30.462 [DEBUG] TestAgent: Node info in sync -15:30.470 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:60868 latency=337.912µs -15:30.479 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes?node-meta=foo%3Abar from=127.0.0.1:60870 latency=6.29534ms -15:30.488 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes?filter=Meta.foo+%3D%3D+bar from=127.0.0.1:60872 latency=687.394µs -15:30.504 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes?near=_agent from=127.0.0.1:60874 latency=1.181219ms -15:30.531 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/consul from=127.0.0.1:60876 latency=1.025494ms -15:30.533 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/this-service-will-literally-never-exist from=127.0.0.1:60878 latency=219.448µs -15:30.533 [INFO] TestAgent: Requesting shutdown -15:30.533 [INFO] TestAgent.server: shutting down server -15:30.533 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -15:30.533 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -15:30.533 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -15:30.533 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -15:30.533 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -15:30.533 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -15:30.533 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -15:30.533 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -15:30.533 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -15:30.533 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -15:30.534 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:15:30.534Z [INFO] agent.router.manager: shutting down -2021-10-31T23:15:30.534Z [INFO] agent.router.manager: shutting down -15:30.550 [INFO] TestAgent: consul server down -15:30.550 [INFO] TestAgent: shutdown complete -15:30.550 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=tcp -15:30.550 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=udp -15:30.550 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15107 network=tcp -15:31.053 [INFO] TestAgent: Waiting for endpoints to shut down -15:31.053 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogListNodesCommand (1.23s) +2022-12-04T05:49:28.380Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:28.391Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:28.394 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2b04c588-a349-b54d-2ec6-c2e0d35bcdc5 Address:127.0.0.1:21237}]" +49:28.395 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2b04c588-a349-b54d-2ec6-c2e0d35bcdc5.dc1 127.0.0.1 +49:28.395 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2b04c588-a349-b54d-2ec6-c2e0d35bcdc5 127.0.0.1 +2022-12-04T05:49:28.395Z [INFO] agent.router: Initializing LAN area manager +49:28.395 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=udp +49:28.395 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21237 [Follower]" leader= +49:28.401 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=tcp +49:28.402 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2b04c588-a349-b54d-2ec6-c2e0d35bcdc5.dc1 area=wan +49:28.403 [INFO] TestAgent.server: Adding LAN server: server="Node-2b04c588-a349-b54d-2ec6-c2e0d35bcdc5 (Addr: tcp/127.0.0.1:21237) (DC: dc1)" +49:28.407 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21233 network=tcp +49:28.407 [INFO] TestAgent: started state syncer +49:28.407 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21238 network=tcp +49:28.435 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:28.435 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21237 [Candidate]" term=2 +49:28.435 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:28.435 [DEBUG] TestAgent.server.raft: vote granted: from=2b04c588-a349-b54d-2ec6-c2e0d35bcdc5 term=2 tally=1 +49:28.435 [INFO] TestAgent.server.raft: election won: tally=1 +49:28.435 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21237 [Leader]" +49:28.435 [INFO] TestAgent.server: cluster leadership acquired +49:28.435 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21237 +49:28.435 [INFO] TestAgent.server: New leader elected: payload=Node-2b04c588-a349-b54d-2ec6-c2e0d35bcdc5 +49:28.436 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:28.452 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:28.452 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:28.452 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:28.452 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:28.452 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:28.452 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2b04c588-a349-b54d-2ec6-c2e0d35bcdc5 +49:28.452 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2b04c588-a349-b54d-2ec6-c2e0d35bcdc5 +49:28.453 [INFO] TestAgent.server: federation state anti-entropy synced +49:28.717 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:51956 latency=563.742µs +49:28.720 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:51958 latency=254.812µs +49:28.724 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes?node-meta=foo%3Abar from=127.0.0.1:51960 latency=2.320785ms +49:28.737 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes?filter=Meta.foo+%3D%3D+bar from=127.0.0.1:51962 latency=781.455µs +49:28.739 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes?near=_agent from=127.0.0.1:51966 latency=752.158µs +49:28.745 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/consul from=127.0.0.1:51968 latency=1.71264ms +49:28.748 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:28.749 [INFO] TestAgent: Synced node info +49:28.749 [DEBUG] TestAgent: Node info in sync +49:28.753 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/this-service-will-literally-never-exist from=127.0.0.1:51972 latency=1.086135ms +49:28.753 [INFO] TestAgent: Requesting shutdown +49:28.753 [INFO] TestAgent.server: shutting down server +49:28.753 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:28.753 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:28.753 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:28.753 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:28.753 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:28.753 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:28.753 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:28.753 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:28.753 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:28.753 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:28.753 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:28.753Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:28.753Z [INFO] agent.router.manager: shutting down +49:28.754 [INFO] TestAgent: consul server down +49:28.754 [INFO] TestAgent: shutdown complete +49:28.754 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=tcp +49:28.754 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=udp +49:28.754 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21233 network=tcp +49:29.254 [INFO] TestAgent: Waiting for endpoints to shut down +49:29.254 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogListNodesCommand (0.91s) --- PASS: TestCatalogListNodesCommand/simple (0.00s) - --- PASS: TestCatalogListNodesCommand/detailed (0.01s) + --- PASS: TestCatalogListNodesCommand/detailed (0.00s) --- PASS: TestCatalogListNodesCommand/node-meta (0.01s) - --- PASS: TestCatalogListNodesCommand/filter (0.00s) - --- PASS: TestCatalogListNodesCommand/near (0.02s) - --- PASS: TestCatalogListNodesCommand/service_present (0.03s) - --- PASS: TestCatalogListNodesCommand/service_missing (0.00s) + --- PASS: TestCatalogListNodesCommand/filter (0.01s) + --- PASS: TestCatalogListNodesCommand/near (0.00s) + --- PASS: TestCatalogListNodesCommand/service_present (0.01s) + --- PASS: TestCatalogListNodesCommand/service_missing (0.01s) PASS -ok github.com/hashicorp/consul/command/catalog/list/nodes 1.440s +ok github.com/hashicorp/consul/command/catalog/list/nodes 0.978s === RUN TestCatalogListServicesCommand_noTabs === PAUSE TestCatalogListServicesCommand_noTabs === RUN TestCatalogListServicesCommand_Validation @@ -78150,11 +78329,11 @@ === RUN TestCatalogListServicesCommand === PAUSE TestCatalogListServicesCommand === CONT TestCatalogListServicesCommand_noTabs ---- PASS: TestCatalogListServicesCommand_noTabs (0.01s) === CONT TestCatalogListServicesCommand -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestCatalogListServicesCommand_Validation +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... --- PASS: TestCatalogListServicesCommand_Validation (0.00s) +--- PASS: TestCatalogListServicesCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestCatalogListServicesCommand/simple @@ -78162,77 +78341,83 @@ === RUN TestCatalogListServicesCommand/node_missing === RUN TestCatalogListServicesCommand/node_present === RUN TestCatalogListServicesCommand/node-meta -2021-10-31T23:15:42.429Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:15:42.508Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -15:42.511 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:72976e32-5a06-8849-7513-b5d9d9309ac2 Address:127.0.0.1:16132}]" -15:42.512 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-72976e32-5a06-8849-7513-b5d9d9309ac2.dc1 127.0.0.1 -15:42.512 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-72976e32-5a06-8849-7513-b5d9d9309ac2 127.0.0.1 -2021-10-31T23:15:42.512Z [INFO] agent.router: Initializing LAN area manager -15:42.512 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=udp -15:42.513 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16132 [Follower]" leader= -15:42.513 [INFO] TestAgent.server: Adding LAN server: server="Node-72976e32-5a06-8849-7513-b5d9d9309ac2 (Addr: tcp/127.0.0.1:16132) (DC: dc1)" -15:42.513 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-72976e32-5a06-8849-7513-b5d9d9309ac2.dc1 area=wan -15:42.513 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=tcp -15:42.538 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16128 network=tcp -15:42.538 [INFO] TestAgent: started state syncer -15:42.538 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16133 network=tcp -15:42.581 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -15:42.582 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16132 [Candidate]" term=2 -15:42.582 [DEBUG] TestAgent.server.raft: votes: needed=1 -15:42.582 [DEBUG] TestAgent.server.raft: vote granted: from=72976e32-5a06-8849-7513-b5d9d9309ac2 term=2 tally=1 -15:42.582 [INFO] TestAgent.server.raft: election won: tally=1 -15:42.582 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16132 [Leader]" -15:42.582 [INFO] TestAgent.server: cluster leadership acquired -15:42.582 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16132 -15:42.583 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -15:42.585 [INFO] TestAgent.server: New leader elected: payload=Node-72976e32-5a06-8849-7513-b5d9d9309ac2 -15:42.641 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -15:42.641 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -15:42.641 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -15:42.641 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -15:42.641 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -15:42.641 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-72976e32-5a06-8849-7513-b5d9d9309ac2 -15:42.641 [INFO] TestAgent.server: member joined, marking health alive: member=Node-72976e32-5a06-8849-7513-b5d9d9309ac2 -15:42.655 [INFO] TestAgent.server: federation state anti-entropy synced -15:42.672 [INFO] TestAgent: Synced node info -15:42.672 [INFO] TestAgent: Synced service: service=testing -15:42.672 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:43366 latency=1.456754ms -15:42.686 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/services from=127.0.0.1:43368 latency=379.748µs -15:42.688 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/services from=127.0.0.1:43370 latency=172.685µs -15:42.724 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/node/not-a-real-node from=127.0.0.1:43372 latency=1.37896ms -15:42.767 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/node/Node-72976e32-5a06-8849-7513-b5d9d9309ac2 from=127.0.0.1:43374 latency=4.871596ms -15:42.772 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/services?node-meta=foo%3Abar from=127.0.0.1:43376 latency=435.255µs -15:42.772 [INFO] TestAgent: Requesting shutdown -15:42.772 [INFO] TestAgent.server: shutting down server -15:42.772 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -15:42.772 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -15:42.772 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -15:42.772 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -15:42.772 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -15:42.772 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -15:42.772 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -15:42.772 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -15:42.772 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -15:42.772 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -15:42.772 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -15:42.772 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:15:42.772Z [INFO] agent.router.manager: shutting down -2021-10-31T23:15:42.772Z [INFO] agent.router.manager: shutting down -15:42.773 [INFO] TestAgent: consul server down -15:42.773 [INFO] TestAgent: shutdown complete -15:42.773 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=tcp -15:42.773 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=udp -15:42.773 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16128 network=tcp -15:43.273 [INFO] TestAgent: Waiting for endpoints to shut down -15:43.273 [INFO] TestAgent: Endpoints down ---- PASS: TestCatalogListServicesCommand (0.99s) - --- PASS: TestCatalogListServicesCommand/simple (0.01s) +2022-12-04T05:49:34.744Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:34.759Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:34.761 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bc48c9d3-0856-4287-c56c-a34e3bd448e4 Address:127.0.0.1:21237}]" +49:34.761 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21237 [Follower]" leader= +49:34.764 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bc48c9d3-0856-4287-c56c-a34e3bd448e4.dc1 127.0.0.1 +49:34.765 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bc48c9d3-0856-4287-c56c-a34e3bd448e4 127.0.0.1 +2022-12-04T05:49:34.765Z [INFO] agent.router: Initializing LAN area manager +49:34.766 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bc48c9d3-0856-4287-c56c-a34e3bd448e4.dc1 area=wan +49:34.766 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=udp +49:34.766 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=tcp +49:34.766 [INFO] TestAgent.server: Adding LAN server: server="Node-bc48c9d3-0856-4287-c56c-a34e3bd448e4 (Addr: tcp/127.0.0.1:21237) (DC: dc1)" +49:34.767 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21233 network=tcp +49:34.767 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21238 network=tcp +49:34.767 [INFO] TestAgent: started state syncer +49:34.817 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:34.817 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21237 [Candidate]" term=2 +49:34.818 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:34.818 [DEBUG] TestAgent.server.raft: vote granted: from=bc48c9d3-0856-4287-c56c-a34e3bd448e4 term=2 tally=1 +49:34.818 [INFO] TestAgent.server.raft: election won: tally=1 +49:34.818 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21237 [Leader]" +49:34.818 [INFO] TestAgent.server: cluster leadership acquired +49:34.818 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21237 +49:34.819 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:34.820 [INFO] TestAgent.server: New leader elected: payload=Node-bc48c9d3-0856-4287-c56c-a34e3bd448e4 +49:34.847 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:34.847 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:34.847 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:34.847 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:34.847 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:34.847 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bc48c9d3-0856-4287-c56c-a34e3bd448e4 +49:34.847 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bc48c9d3-0856-4287-c56c-a34e3bd448e4 +49:34.848 [INFO] TestAgent.server: federation state anti-entropy synced +49:34.850 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.851 [INFO] TestAgent: Synced node info +49:34.998 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:34.998 [DEBUG] TestAgent: Node info in sync +49:34.998 [DEBUG] TestAgent: Node info in sync +49:35.094 [DEBUG] TestAgent: Node info in sync +49:35.095 [INFO] TestAgent: Synced service: service=testing +49:35.095 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:51974 latency=1.235828ms +49:35.095 [DEBUG] TestAgent: Node info in sync +49:35.095 [DEBUG] TestAgent: Service in sync: service=testing +49:35.097 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/services from=127.0.0.1:51976 latency=531.91µs +49:35.098 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/services from=127.0.0.1:51978 latency=320.332µs +49:35.105 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/node/not-a-real-node from=127.0.0.1:51980 latency=4.859638ms +49:35.109 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/node/Node-bc48c9d3-0856-4287-c56c-a34e3bd448e4 from=127.0.0.1:51982 latency=342.217µs +49:35.110 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/services?node-meta=foo%3Abar from=127.0.0.1:51984 latency=274.729µs +49:35.110 [INFO] TestAgent: Requesting shutdown +49:35.111 [INFO] TestAgent.server: shutting down server +49:35.111 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.111 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.111 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.111 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.111 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.111 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.111 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:35.111 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.111 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.111 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.111 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:35.111Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.111Z [INFO] agent.router.manager: shutting down +49:35.111 [INFO] TestAgent: consul server down +49:35.111 [INFO] TestAgent: shutdown complete +49:35.111 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=tcp +49:35.111 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=udp +49:35.111 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21233 network=tcp +49:35.611 [INFO] TestAgent: Waiting for endpoints to shut down +49:35.611 [INFO] TestAgent: Endpoints down +--- PASS: TestCatalogListServicesCommand (0.92s) + --- PASS: TestCatalogListServicesCommand/simple (0.00s) --- PASS: TestCatalogListServicesCommand/tags (0.00s) - --- PASS: TestCatalogListServicesCommand/node_missing (0.04s) - --- PASS: TestCatalogListServicesCommand/node_present (0.05s) + --- PASS: TestCatalogListServicesCommand/node_missing (0.01s) + --- PASS: TestCatalogListServicesCommand/node_present (0.00s) --- PASS: TestCatalogListServicesCommand/node-meta (0.00s) PASS -ok github.com/hashicorp/consul/command/catalog/list/services 1.269s +ok github.com/hashicorp/consul/command/catalog/list/services 1.002s ? github.com/hashicorp/consul/command/config [no test files] === RUN TestConfigDelete_noTabs === PAUSE TestConfigDelete_noTabs @@ -78241,148 +78426,147 @@ === RUN TestConfigDelete_InvalidArgs === PAUSE TestConfigDelete_InvalidArgs === CONT TestConfigDelete_noTabs ---- PASS: TestConfigDelete_noTabs (0.00s) === CONT TestConfigDelete_InvalidArgs -=== RUN TestConfigDelete_InvalidArgs/no_name +=== CONT TestConfigDelete === RUN TestConfigDelete_InvalidArgs/no_kind +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestConfigDelete_noTabs (0.00s) +=== RUN TestConfigDelete_InvalidArgs/no_name --- PASS: TestConfigDelete_InvalidArgs (0.00s) - --- PASS: TestConfigDelete_InvalidArgs/no_name (0.00s) --- PASS: TestConfigDelete_InvalidArgs/no_kind (0.00s) -=== CONT TestConfigDelete -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... + --- PASS: TestConfigDelete_InvalidArgs/no_name (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:15:56.693Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:15:56.753Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -15:56.780 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:280baa04-18a1-d92d-a730-13eec92f8e30 Address:127.0.0.1:13069}]" -15:56.781 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-280baa04-18a1-d92d-a730-13eec92f8e30.dc1 127.0.0.1 -15:56.781 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-280baa04-18a1-d92d-a730-13eec92f8e30 127.0.0.1 -2021-10-31T23:15:56.781Z [INFO] agent.router: Initializing LAN area manager -15:56.782 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= -15:56.790 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp -15:56.790 [INFO] TestAgent.server: Adding LAN server: server="Node-280baa04-18a1-d92d-a730-13eec92f8e30 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" -15:56.790 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-280baa04-18a1-d92d-a730-13eec92f8e30.dc1 area=wan -15:56.790 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp -15:56.791 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp -15:56.791 [INFO] TestAgent: started state syncer -15:56.791 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp -15:56.825 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -15:56.826 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 -15:56.826 [DEBUG] TestAgent.server.raft: votes: needed=1 -15:56.826 [DEBUG] TestAgent.server.raft: vote granted: from=280baa04-18a1-d92d-a730-13eec92f8e30 term=2 tally=1 -15:56.826 [INFO] TestAgent.server.raft: election won: tally=1 -15:56.826 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" -15:56.826 [INFO] TestAgent.server: cluster leadership acquired -15:56.826 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13069 -15:56.827 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -15:56.829 [INFO] TestAgent.server: New leader elected: payload=Node-280baa04-18a1-d92d-a730-13eec92f8e30 -15:56.914 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -15:56.914 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -15:56.914 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -15:56.914 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -15:56.914 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -15:56.914 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-280baa04-18a1-d92d-a730-13eec92f8e30 -15:56.914 [INFO] TestAgent.server: member joined, marking health alive: member=Node-280baa04-18a1-d92d-a730-13eec92f8e30 -15:56.914 [INFO] TestAgent: Synced node info -15:56.914 [DEBUG] TestAgent: Node info in sync -15:56.916 [INFO] TestAgent.server: federation state anti-entropy synced -15:57.072 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:52008 latency=1.079176ms -15:57.074 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/config/service-defaults/web from=127.0.0.1:52010 latency=1.539902ms -15:57.078 [ERROR] TestAgent.http: Request error: method=GET url=/v1/config/service-defaults/web from=127.0.0.1:52008 error="Config entry not found for "service-defaults" / "web"" -15:57.078 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/service-defaults/web from=127.0.0.1:52008 latency=283.614µs -15:57.078 [INFO] TestAgent: Requesting shutdown -15:57.079 [INFO] TestAgent.server: shutting down server -15:57.079 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -15:57.079 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -15:57.079 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -15:57.079 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -15:57.079 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -15:57.079 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -15:57.079 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -15:57.079 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -15:57.079 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -15:57.079 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -15:57.079 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:15:57.079Z [INFO] agent.router.manager: shutting down -2021-10-31T23:15:57.079Z [INFO] agent.router.manager: shutting down -15:57.079 [INFO] TestAgent: consul server down -15:57.079 [INFO] TestAgent: shutdown complete -15:57.079 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp -15:57.079 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp -15:57.079 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp -15:57.581 [INFO] TestAgent: Waiting for endpoints to shut down -15:57.581 [INFO] TestAgent: Endpoints down ---- PASS: TestConfigDelete (1.06s) +2022-12-04T05:49:35.051Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:35.061Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:35.063 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:da2531b5-05fb-bb1a-4e80-ffc9aa941772 Address:127.0.0.1:27363}]" +49:35.064 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= +49:35.064 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-da2531b5-05fb-bb1a-4e80-ffc9aa941772.dc1 127.0.0.1 +49:35.065 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-da2531b5-05fb-bb1a-4e80-ffc9aa941772 127.0.0.1 +2022-12-04T05:49:35.065Z [INFO] agent.router: Initializing LAN area manager +49:35.065 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp +49:35.065 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp +49:35.065 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-da2531b5-05fb-bb1a-4e80-ffc9aa941772.dc1 area=wan +49:35.066 [INFO] TestAgent.server: Adding LAN server: server="Node-da2531b5-05fb-bb1a-4e80-ffc9aa941772 (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +49:35.066 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp +49:35.066 [INFO] TestAgent: started state syncer +49:35.066 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp +49:35.103 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.103 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 +49:35.103 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:35.103 [DEBUG] TestAgent.server.raft: vote granted: from=da2531b5-05fb-bb1a-4e80-ffc9aa941772 term=2 tally=1 +49:35.103 [INFO] TestAgent.server.raft: election won: tally=1 +49:35.103 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" +49:35.103 [INFO] TestAgent.server: cluster leadership acquired +49:35.104 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27363 +49:35.104 [INFO] TestAgent.server: New leader elected: payload=Node-da2531b5-05fb-bb1a-4e80-ffc9aa941772 +49:35.104 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:35.120 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:35.120 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:35.120 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:35.120 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:35.120 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:35.120 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-da2531b5-05fb-bb1a-4e80-ffc9aa941772 +49:35.120 [INFO] TestAgent.server: member joined, marking health alive: member=Node-da2531b5-05fb-bb1a-4e80-ffc9aa941772 +49:35.121 [INFO] TestAgent.server: federation state anti-entropy synced +49:35.189 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:35.195 [INFO] TestAgent: Synced node info +49:35.195 [DEBUG] TestAgent: Node info in sync +49:35.335 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:59690 latency=1.686269ms +49:35.337 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/config/service-defaults/web from=127.0.0.1:59692 latency=619.015µs +49:35.337 [ERROR] TestAgent.http: Request error: method=GET url=/v1/config/service-defaults/web from=127.0.0.1:59690 error="Config entry not found for "service-defaults" / "web"" +49:35.338 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/service-defaults/web from=127.0.0.1:59690 latency=910.973µs +49:35.338 [INFO] TestAgent: Requesting shutdown +49:35.338 [INFO] TestAgent.server: shutting down server +49:35.338 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:35.338 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:35.338 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:35.338 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:35.338 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:35.338 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:35.338 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:35.339 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:35.339 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:35.339 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:35.339 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:35.339Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:35.339Z [INFO] agent.router.manager: shutting down +49:35.339 [INFO] TestAgent: consul server down +49:35.339 [INFO] TestAgent: shutdown complete +49:35.339 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp +49:35.339 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp +49:35.339 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp +49:35.840 [INFO] TestAgent: Waiting for endpoints to shut down +49:35.840 [INFO] TestAgent: Endpoints down +--- PASS: TestConfigDelete (0.83s) PASS -ok github.com/hashicorp/consul/command/config/delete 2.019s +ok github.com/hashicorp/consul/command/config/delete 0.914s === RUN TestConfigList_noTabs === PAUSE TestConfigList_noTabs === RUN TestConfigList [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:16:01.553Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:01.633Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:01.644 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:748e3318-e6cc-5fc2-2f6d-ed585ca8f019 Address:127.0.0.1:24300}]" -16:01.644 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-748e3318-e6cc-5fc2-2f6d-ed585ca8f019.dc1 127.0.0.1 -16:01.645 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-748e3318-e6cc-5fc2-2f6d-ed585ca8f019 127.0.0.1 -2021-10-31T23:16:01.645Z [INFO] agent.router: Initializing LAN area manager -16:01.645 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp -16:01.645 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= -16:01.647 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-748e3318-e6cc-5fc2-2f6d-ed585ca8f019.dc1 area=wan -16:01.647 [INFO] TestAgent.server: Adding LAN server: server="Node-748e3318-e6cc-5fc2-2f6d-ed585ca8f019 (Addr: tcp/127.0.0.1:24300) (DC: dc1)" -16:01.647 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp -16:01.648 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp -16:01.648 [INFO] TestAgent: started state syncer -16:01.648 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp -16:01.697 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:01.697 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 -16:01.698 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:01.698 [DEBUG] TestAgent.server.raft: vote granted: from=748e3318-e6cc-5fc2-2f6d-ed585ca8f019 term=2 tally=1 -16:01.698 [INFO] TestAgent.server.raft: election won: tally=1 -16:01.698 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" -16:01.698 [INFO] TestAgent.server: cluster leadership acquired -16:01.698 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24300 -16:01.699 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:01.701 [INFO] TestAgent.server: New leader elected: payload=Node-748e3318-e6cc-5fc2-2f6d-ed585ca8f019 -16:01.764 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:01.764 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:01.764 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:01.764 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:01.764 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:01.764 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-748e3318-e6cc-5fc2-2f6d-ed585ca8f019 -16:01.764 [INFO] TestAgent.server: member joined, marking health alive: member=Node-748e3318-e6cc-5fc2-2f6d-ed585ca8f019 -16:01.774 [INFO] TestAgent.server: federation state anti-entropy synced -16:01.918 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:01.918 [INFO] TestAgent: Synced node info -16:01.918 [DEBUG] TestAgent: Node info in sync -16:02.067 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:36500 latency=6.843282ms -16:02.083 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:36500 latency=10.028479ms -16:02.084 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:36500 latency=899.483µs -16:02.085 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/service-defaults from=127.0.0.1:36502 latency=443.888µs -16:02.086 [INFO] TestAgent: Requesting shutdown -16:02.086 [INFO] TestAgent.server: shutting down server -16:02.086 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:02.086 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:02.086 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:02.086 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:02.086 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:02.086 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:02.086 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:02.086 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:02.086 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:02.086 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:02.086 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:02.086Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:02.086Z [INFO] agent.router.manager: shutting down -16:02.086 [INFO] TestAgent: consul server down -16:02.086 [INFO] TestAgent: shutdown complete -16:02.086 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp -16:02.086 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp -16:02.086 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp -16:02.589 [INFO] TestAgent: Waiting for endpoints to shut down -16:02.589 [INFO] TestAgent: Endpoints down ---- PASS: TestConfigList (1.22s) +2022-12-04T05:49:35.611Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:35.626Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:35.628 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4883031e-74dd-c6d4-2ef6-6f2ccbe8ed19 Address:127.0.0.1:18174}]" +49:35.628 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18174 [Follower]" leader= +49:35.628 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4883031e-74dd-c6d4-2ef6-6f2ccbe8ed19.dc1 127.0.0.1 +49:35.629 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4883031e-74dd-c6d4-2ef6-6f2ccbe8ed19 127.0.0.1 +2022-12-04T05:49:35.629Z [INFO] agent.router: Initializing LAN area manager +49:35.629 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4883031e-74dd-c6d4-2ef6-6f2ccbe8ed19.dc1 area=wan +49:35.629 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=tcp +49:35.629 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=udp +49:35.632 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18170 network=tcp +49:35.632 [INFO] TestAgent.server: Adding LAN server: server="Node-4883031e-74dd-c6d4-2ef6-6f2ccbe8ed19 (Addr: tcp/127.0.0.1:18174) (DC: dc1)" +49:35.632 [INFO] TestAgent: started state syncer +49:35.632 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18175 network=tcp +49:35.695 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:35.695 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18174 [Candidate]" term=2 +49:35.696 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:35.696 [DEBUG] TestAgent.server.raft: vote granted: from=4883031e-74dd-c6d4-2ef6-6f2ccbe8ed19 term=2 tally=1 +49:35.696 [INFO] TestAgent.server.raft: election won: tally=1 +49:35.696 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18174 [Leader]" +49:35.696 [INFO] TestAgent.server: cluster leadership acquired +49:35.696 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18174 +49:35.696 [INFO] TestAgent.server: New leader elected: payload=Node-4883031e-74dd-c6d4-2ef6-6f2ccbe8ed19 +49:35.697 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:35.712 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:35.712 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:35.712 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:35.712 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:35.712 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:35.712 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4883031e-74dd-c6d4-2ef6-6f2ccbe8ed19 +49:35.712 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4883031e-74dd-c6d4-2ef6-6f2ccbe8ed19 +49:35.713 [INFO] TestAgent.server: federation state anti-entropy synced +49:36.031 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:50292 latency=1.072733ms +49:36.033 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:50292 latency=1.396148ms +49:36.041 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:50292 latency=4.592745ms +49:36.052 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/service-defaults from=127.0.0.1:50294 latency=613.818µs +49:36.053 [INFO] TestAgent: Requesting shutdown +49:36.053 [INFO] TestAgent.server: shutting down server +49:36.053 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:36.053 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:36.053 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:36.053 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:36.053 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:36.053 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:36.053 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:36.053 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:36.053 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:36.053 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:36.053 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:36.053 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:36.053Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:36.053Z [INFO] agent.router.manager: shutting down +49:36.053 [INFO] TestAgent: consul server down +49:36.053 [INFO] TestAgent: shutdown complete +49:36.053 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=tcp +49:36.053 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=udp +49:36.053 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18170 network=tcp +49:36.553 [INFO] TestAgent: Waiting for endpoints to shut down +49:36.553 [INFO] TestAgent: Endpoints down +--- PASS: TestConfigList (0.97s) === RUN TestConfigList_InvalidArgs === PAUSE TestConfigList_InvalidArgs === CONT TestConfigList_noTabs @@ -78392,7 +78576,7 @@ --- PASS: TestConfigList_InvalidArgs (0.00s) --- PASS: TestConfigList_InvalidArgs/no_kind (0.00s) PASS -ok github.com/hashicorp/consul/command/config/list 1.717s +ok github.com/hashicorp/consul/command/config/list 1.040s === RUN TestConfigRead_noTabs === PAUSE TestConfigRead_noTabs === RUN TestConfigRead @@ -78400,79 +78584,77 @@ === RUN TestConfigRead_InvalidArgs === PAUSE TestConfigRead_InvalidArgs === CONT TestConfigRead_noTabs ---- PASS: TestConfigRead_noTabs (0.00s) === CONT TestConfigRead_InvalidArgs +=== CONT TestConfigRead === RUN TestConfigRead_InvalidArgs/no_kind +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestConfigRead_noTabs (0.00s) === RUN TestConfigRead_InvalidArgs/no_name --- PASS: TestConfigRead_InvalidArgs (0.00s) --- PASS: TestConfigRead_InvalidArgs/no_kind (0.00s) --- PASS: TestConfigRead_InvalidArgs/no_name (0.00s) -=== CONT TestConfigRead -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:16:04.472Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:04.507Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:04.514 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:81d2e438-c0a2-8c57-c15e-729b676b374e Address:127.0.0.1:18174}]" -16:04.515 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-81d2e438-c0a2-8c57-c15e-729b676b374e.dc1 127.0.0.1 -16:04.516 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-81d2e438-c0a2-8c57-c15e-729b676b374e 127.0.0.1 -2021-10-31T23:16:04.516Z [INFO] agent.router: Initializing LAN area manager -16:04.516 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=udp -16:04.516 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18174 [Follower]" leader= -16:04.516 [INFO] TestAgent.server: Adding LAN server: server="Node-81d2e438-c0a2-8c57-c15e-729b676b374e (Addr: tcp/127.0.0.1:18174) (DC: dc1)" -16:04.516 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-81d2e438-c0a2-8c57-c15e-729b676b374e.dc1 area=wan -16:04.516 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=tcp -16:04.517 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18170 network=tcp -16:04.517 [INFO] TestAgent: started state syncer -16:04.518 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18175 network=tcp -16:04.577 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:04.578 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18174 [Candidate]" term=2 -16:04.578 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:04.578 [DEBUG] TestAgent.server.raft: vote granted: from=81d2e438-c0a2-8c57-c15e-729b676b374e term=2 tally=1 -16:04.578 [INFO] TestAgent.server.raft: election won: tally=1 -16:04.578 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18174 [Leader]" -16:04.578 [INFO] TestAgent.server: cluster leadership acquired -16:04.578 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18174 -16:04.579 [INFO] TestAgent.server: New leader elected: payload=Node-81d2e438-c0a2-8c57-c15e-729b676b374e -16:04.580 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:04.616 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:04.616 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:04.616 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:04.616 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:04.616 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:04.616 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-81d2e438-c0a2-8c57-c15e-729b676b374e -16:04.616 [INFO] TestAgent.server: member joined, marking health alive: member=Node-81d2e438-c0a2-8c57-c15e-729b676b374e -16:04.617 [INFO] TestAgent.server: federation state anti-entropy synced -16:04.710 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:04.710 [INFO] TestAgent: Synced node info -16:04.710 [DEBUG] TestAgent: Node info in sync -16:04.839 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:46290 latency=9.135632ms -16:04.866 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/service-defaults/web from=127.0.0.1:46292 latency=22.438565ms -16:04.874 [INFO] TestAgent: Requesting shutdown -16:04.874 [INFO] TestAgent.server: shutting down server -16:04.874 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:04.874 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:04.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:04.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:04.874 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:04.874 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:04.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:04.874 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:04.874 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:04.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:04.874 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:04.875Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:04.875Z [INFO] agent.router.manager: shutting down -16:04.875 [INFO] TestAgent: consul server down -16:04.875 [INFO] TestAgent: shutdown complete -16:04.875 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=tcp -16:04.875 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=udp -16:04.875 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18170 network=tcp -16:05.378 [INFO] TestAgent: Waiting for endpoints to shut down -16:05.378 [INFO] TestAgent: Endpoints down ---- PASS: TestConfigRead (1.06s) +2022-12-04T05:49:41.307Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:41.320Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:41.322 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c02f3fa7-8d44-f244-8d21-53e8c59926a4 Address:127.0.0.1:29405}]" +49:41.322 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29405 [Follower]" leader= +49:41.322 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c02f3fa7-8d44-f244-8d21-53e8c59926a4.dc1 127.0.0.1 +49:41.323 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c02f3fa7-8d44-f244-8d21-53e8c59926a4 127.0.0.1 +2022-12-04T05:49:41.323Z [INFO] agent.router: Initializing LAN area manager +49:41.323 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c02f3fa7-8d44-f244-8d21-53e8c59926a4.dc1 area=wan +49:41.323 [INFO] TestAgent.server: Adding LAN server: server="Node-c02f3fa7-8d44-f244-8d21-53e8c59926a4 (Addr: tcp/127.0.0.1:29405) (DC: dc1)" +49:41.323 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=udp +49:41.323 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=tcp +49:41.324 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29401 network=tcp +49:41.324 [INFO] TestAgent: started state syncer +49:41.324 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29406 network=tcp +49:41.370 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:41.370 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29405 [Candidate]" term=2 +49:41.370 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:41.370 [DEBUG] TestAgent.server.raft: vote granted: from=c02f3fa7-8d44-f244-8d21-53e8c59926a4 term=2 tally=1 +49:41.370 [INFO] TestAgent.server.raft: election won: tally=1 +49:41.370 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29405 [Leader]" +49:41.370 [INFO] TestAgent.server: cluster leadership acquired +49:41.370 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29405 +49:41.370 [INFO] TestAgent.server: New leader elected: payload=Node-c02f3fa7-8d44-f244-8d21-53e8c59926a4 +49:41.371 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:41.386 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:41.386 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:41.386 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:41.386 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:41.386 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:41.386 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c02f3fa7-8d44-f244-8d21-53e8c59926a4 +49:41.386 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c02f3fa7-8d44-f244-8d21-53e8c59926a4 +49:41.387 [INFO] TestAgent.server: federation state anti-entropy synced +49:41.583 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:56156 latency=2.394214ms +49:41.593 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/service-defaults/web from=127.0.0.1:56158 latency=8.442315ms +49:41.593 [INFO] TestAgent: Requesting shutdown +49:41.593 [INFO] TestAgent.server: shutting down server +49:41.593 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:41.593 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:41.593 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:41.593 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:41.593 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:41.593 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:41.593 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:41.593 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:41.593 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:41.593 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:41.593 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:41.593 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:41.593Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:41.594Z [INFO] agent.router.manager: shutting down +49:41.594 [INFO] TestAgent: consul server down +49:41.594 [INFO] TestAgent: shutdown complete +49:41.594 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=tcp +49:41.594 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=udp +49:41.594 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29401 network=tcp +49:42.094 [INFO] TestAgent: Waiting for endpoints to shut down +49:42.094 [INFO] TestAgent: Endpoints down +--- PASS: TestConfigRead (0.82s) PASS -ok github.com/hashicorp/consul/command/config/read 1.361s +ok github.com/hashicorp/consul/command/config/read 0.862s === RUN TestConfigWrite_noTabs === PAUSE TestConfigWrite_noTabs === RUN TestConfigWrite @@ -78480,16 +78662,19 @@ === RUN TestParseConfigEntry === PAUSE TestParseConfigEntry === CONT TestConfigWrite_noTabs ---- PASS: TestConfigWrite_noTabs (0.01s) === CONT TestParseConfigEntry -=== RUN TestParseConfigEntry/proxy-defaults:_extra_fields_or_typo_(hcl_snake_case) === CONT TestConfigWrite +=== RUN TestParseConfigEntry/proxy-defaults:_extra_fields_or_typo_(hcl_snake_case) +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === RUN TestParseConfigEntry/proxy-defaults:_extra_fields_or_typo_(hcl_camel_case) === RUN TestParseConfigEntry/proxy-defaults:_extra_fields_or_typo_(json_snake_case) === RUN TestParseConfigEntry/proxy-defaults:_extra_fields_or_typo_(json_camel_case) === RUN TestParseConfigEntry/proxy-defaults_(hcl_snake_case) === RUN TestParseConfigEntry/proxy-defaults_(hcl_camel_case) === RUN TestParseConfigEntry/proxy-defaults_(json_snake_case) +[INFO] freeport: detected ephemeral port range of [32768, 60999] +[INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range +--- PASS: TestConfigWrite_noTabs (0.00s) === RUN TestParseConfigEntry/proxy-defaults_(json_camel_case) === RUN TestParseConfigEntry/terminating-gateway_(hcl_snake_case) === RUN TestParseConfigEntry/terminating-gateway_(hcl_camel_case) @@ -78500,15 +78685,12 @@ === RUN TestParseConfigEntry/service-defaults_(json_snake_case) === RUN TestParseConfigEntry/service-defaults_(json_camel_case) === RUN TestParseConfigEntry/service-router:_kitchen_sink_(hcl_snake_case) -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === RUN TestParseConfigEntry/service-router:_kitchen_sink_(hcl_camel_case) === RUN TestParseConfigEntry/service-router:_kitchen_sink_(json_snake_case) === RUN TestParseConfigEntry/service-router:_kitchen_sink_(json_camel_case) === RUN TestParseConfigEntry/service-splitter:_kitchen_sink_(hcl_snake_case) === RUN TestParseConfigEntry/service-splitter:_kitchen_sink_(hcl_camel_case) === RUN TestParseConfigEntry/service-splitter:_kitchen_sink_(json_snake_case) -[INFO] freeport: detected ephemeral port range of [32768, 60999] -[INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestParseConfigEntry/service-splitter:_kitchen_sink_(json_camel_case) === RUN TestParseConfigEntry/service-resolver:_subsets_with_failover_(hcl_snake_case) === RUN TestParseConfigEntry/service-resolver:_subsets_with_failover_(hcl_camel_case) @@ -78534,12 +78716,12 @@ === RUN TestParseConfigEntry/ingress-gateway:_kitchen_sink_(hcl_camel_case) === RUN TestParseConfigEntry/ingress-gateway:_kitchen_sink_(json_snake_case) === RUN TestParseConfigEntry/ingress-gateway:_kitchen_sink_(json_camel_case) ---- PASS: TestParseConfigEntry (0.11s) +--- PASS: TestParseConfigEntry (0.02s) --- PASS: TestParseConfigEntry/proxy-defaults:_extra_fields_or_typo_(hcl_snake_case) (0.00s) --- PASS: TestParseConfigEntry/proxy-defaults:_extra_fields_or_typo_(hcl_camel_case) (0.00s) --- PASS: TestParseConfigEntry/proxy-defaults:_extra_fields_or_typo_(json_snake_case) (0.00s) --- PASS: TestParseConfigEntry/proxy-defaults:_extra_fields_or_typo_(json_camel_case) (0.00s) - --- PASS: TestParseConfigEntry/proxy-defaults_(hcl_snake_case) (0.01s) + --- PASS: TestParseConfigEntry/proxy-defaults_(hcl_snake_case) (0.00s) --- PASS: TestParseConfigEntry/proxy-defaults_(hcl_camel_case) (0.00s) --- PASS: TestParseConfigEntry/proxy-defaults_(json_snake_case) (0.00s) --- PASS: TestParseConfigEntry/proxy-defaults_(json_camel_case) (0.00s) @@ -78551,8 +78733,8 @@ --- PASS: TestParseConfigEntry/service-defaults_(hcl_camel_case) (0.00s) --- PASS: TestParseConfigEntry/service-defaults_(json_snake_case) (0.00s) --- PASS: TestParseConfigEntry/service-defaults_(json_camel_case) (0.00s) - --- PASS: TestParseConfigEntry/service-router:_kitchen_sink_(hcl_snake_case) (0.02s) - --- PASS: TestParseConfigEntry/service-router:_kitchen_sink_(hcl_camel_case) (0.02s) + --- PASS: TestParseConfigEntry/service-router:_kitchen_sink_(hcl_snake_case) (0.00s) + --- PASS: TestParseConfigEntry/service-router:_kitchen_sink_(hcl_camel_case) (0.00s) --- PASS: TestParseConfigEntry/service-router:_kitchen_sink_(json_snake_case) (0.00s) --- PASS: TestParseConfigEntry/service-router:_kitchen_sink_(json_camel_case) (0.00s) --- PASS: TestParseConfigEntry/service-splitter:_kitchen_sink_(hcl_snake_case) (0.00s) @@ -78579,91 +78761,90 @@ --- PASS: TestParseConfigEntry/expose_paths:_kitchen_sink_service_defaults_(hcl_camel_case) (0.00s) --- PASS: TestParseConfigEntry/expose_paths:_kitchen_sink_service_defaults_(json_snake_case) (0.00s) --- PASS: TestParseConfigEntry/expose_paths:_kitchen_sink_service_defaults_(json_camel_case) (0.00s) - --- PASS: TestParseConfigEntry/ingress-gateway:_kitchen_sink_(hcl_snake_case) (0.02s) + --- PASS: TestParseConfigEntry/ingress-gateway:_kitchen_sink_(hcl_snake_case) (0.00s) --- PASS: TestParseConfigEntry/ingress-gateway:_kitchen_sink_(hcl_camel_case) (0.00s) --- PASS: TestParseConfigEntry/ingress-gateway:_kitchen_sink_(json_snake_case) (0.00s) --- PASS: TestParseConfigEntry/ingress-gateway:_kitchen_sink_(json_camel_case) (0.00s) === RUN TestConfigWrite/File === RUN TestConfigWrite/Stdin === RUN TestConfigWrite/No_config -2021-10-31T23:16:11.176Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:11.236Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:11.255 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6000da90-4ff7-5da4-5a2b-0d23a40ea6b3 Address:127.0.0.1:10006}]" -16:11.256 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6000da90-4ff7-5da4-5a2b-0d23a40ea6b3.dc1 127.0.0.1 -16:11.256 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6000da90-4ff7-5da4-5a2b-0d23a40ea6b3 127.0.0.1 -2021-10-31T23:16:11.256Z [INFO] agent.router: Initializing LAN area manager -16:11.257 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=udp -16:11.257 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10006 [Follower]" leader= -16:11.257 [INFO] TestAgent.server: Adding LAN server: server="Node-6000da90-4ff7-5da4-5a2b-0d23a40ea6b3 (Addr: tcp/127.0.0.1:10006) (DC: dc1)" -16:11.257 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6000da90-4ff7-5da4-5a2b-0d23a40ea6b3.dc1 area=wan -16:11.257 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=tcp -16:11.278 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10002 network=tcp -16:11.278 [INFO] TestAgent: started state syncer -16:11.278 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10007 network=tcp -16:11.309 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:11.310 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10006 [Candidate]" term=2 -16:11.310 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:11.310 [DEBUG] TestAgent.server.raft: vote granted: from=6000da90-4ff7-5da4-5a2b-0d23a40ea6b3 term=2 tally=1 -16:11.310 [INFO] TestAgent.server.raft: election won: tally=1 -16:11.310 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10006 [Leader]" -16:11.310 [INFO] TestAgent.server: cluster leadership acquired -16:11.310 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10006 -16:11.311 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:11.313 [INFO] TestAgent.server: New leader elected: payload=Node-6000da90-4ff7-5da4-5a2b-0d23a40ea6b3 -16:11.371 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:11.371 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:11.371 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:11.371 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:11.371 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:11.371 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6000da90-4ff7-5da4-5a2b-0d23a40ea6b3 -16:11.371 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6000da90-4ff7-5da4-5a2b-0d23a40ea6b3 -16:11.372 [INFO] TestAgent.server: federation state anti-entropy synced -16:11.467 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:55922 latency=8.713926ms -16:11.472 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:11.472 [INFO] TestAgent: Synced node info -16:11.478 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/service-defaults/web from=127.0.0.1:55924 latency=5.834502ms -16:11.519 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:55926 latency=1.800379ms -16:11.524 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/proxy-defaults/global from=127.0.0.1:55924 latency=354.644µs -16:11.524 [INFO] TestAgent: Requesting shutdown -16:11.525 [INFO] TestAgent.server: shutting down server -16:11.525 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:11.525 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:11.525 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:11.525 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:11.525 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:11.525 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:11.525 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:11.525 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:11.525 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:11.525 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:11.525 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:11.525Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:11.525Z [INFO] agent.router.manager: shutting down -16:11.525 [INFO] TestAgent: consul server down -16:11.525 [INFO] TestAgent: shutdown complete -16:11.525 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=tcp -16:11.525 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=udp -16:11.525 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10002 network=tcp -16:12.025 [INFO] TestAgent: Waiting for endpoints to shut down -16:12.025 [INFO] TestAgent: Endpoints down ---- PASS: TestConfigWrite (1.04s) - --- PASS: TestConfigWrite/File (0.03s) - --- PASS: TestConfigWrite/Stdin (0.04s) +2022-12-04T05:49:41.570Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:41.580Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:41.582 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e93b471f-628d-6102-af29-0328c6567d25 Address:127.0.0.1:16132}]" +49:41.582 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16132 [Follower]" leader= +49:41.583 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e93b471f-628d-6102-af29-0328c6567d25.dc1 127.0.0.1 +49:41.586 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e93b471f-628d-6102-af29-0328c6567d25 127.0.0.1 +2022-12-04T05:49:41.586Z [INFO] agent.router: Initializing LAN area manager +49:41.587 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e93b471f-628d-6102-af29-0328c6567d25.dc1 area=wan +49:41.587 [INFO] TestAgent.server: Adding LAN server: server="Node-e93b471f-628d-6102-af29-0328c6567d25 (Addr: tcp/127.0.0.1:16132) (DC: dc1)" +49:41.587 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=udp +49:41.587 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=tcp +49:41.588 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16128 network=tcp +49:41.589 [INFO] TestAgent: started state syncer +49:41.589 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16133 network=tcp +49:41.631 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:41.631 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16132 [Candidate]" term=2 +49:41.631 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:41.631 [DEBUG] TestAgent.server.raft: vote granted: from=e93b471f-628d-6102-af29-0328c6567d25 term=2 tally=1 +49:41.631 [INFO] TestAgent.server.raft: election won: tally=1 +49:41.631 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16132 [Leader]" +49:41.631 [INFO] TestAgent.server: cluster leadership acquired +49:41.631 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16132 +49:41.632 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:41.636 [INFO] TestAgent.server: New leader elected: payload=Node-e93b471f-628d-6102-af29-0328c6567d25 +49:41.649 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:41.649 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:41.649 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:41.649 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:41.649 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:41.649 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e93b471f-628d-6102-af29-0328c6567d25 +49:41.649 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e93b471f-628d-6102-af29-0328c6567d25 +49:41.649 [INFO] TestAgent.server: federation state anti-entropy synced +49:41.951 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:40520 latency=1.097378ms +49:41.952 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/service-defaults/web from=127.0.0.1:40522 latency=428.147µs +49:41.954 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:40524 latency=827.579µs +49:41.960 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/proxy-defaults/global from=127.0.0.1:40522 latency=261.51µs +49:41.961 [INFO] TestAgent: Requesting shutdown +49:41.961 [INFO] TestAgent.server: shutting down server +49:41.961 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:41.961 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:41.961 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:41.961 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:41.961 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:41.962 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:41.962 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:41.962 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:41.962 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:41.962 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:41.962 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:41.962 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:41.962Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:41.962Z [INFO] agent.router.manager: shutting down +49:41.962 [INFO] TestAgent: consul server down +49:41.962 [INFO] TestAgent: shutdown complete +49:41.962 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=tcp +49:41.962 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=udp +49:41.962 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16128 network=tcp +49:42.462 [INFO] TestAgent: Waiting for endpoints to shut down +49:42.462 [INFO] TestAgent: Endpoints down +--- PASS: TestConfigWrite (0.93s) + --- PASS: TestConfigWrite/File (0.00s) + --- PASS: TestConfigWrite/Stdin (0.01s) --- PASS: TestConfigWrite/No_config (0.00s) PASS -ok github.com/hashicorp/consul/command/config/write 1.897s +ok github.com/hashicorp/consul/command/config/write 0.983s === RUN TestConnectCommand_noTabs === PAUSE TestConnectCommand_noTabs === CONT TestConnectCommand_noTabs --- PASS: TestConnectCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/connect 0.070s +ok github.com/hashicorp/consul/command/connect 0.020s === RUN TestCatalogCommand_noTabs === PAUSE TestCatalogCommand_noTabs === CONT TestCatalogCommand_noTabs --- PASS: TestCatalogCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/connect/ca 0.053s +ok github.com/hashicorp/consul/command/connect/ca 0.005s === RUN TestConnectCAGetConfigCommand_noTabs === PAUSE TestConnectCAGetConfigCommand_noTabs === RUN TestConnectCAGetConfigCommand @@ -78674,69 +78855,67 @@ [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:16:29.697Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:29.739Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:29.749 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8f4d9d75-032b-7d97-bc6f-6afd966bb392 Address:127.0.0.1:13069}]" -16:29.749 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= -16:29.750 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8f4d9d75-032b-7d97-bc6f-6afd966bb392.dc1 127.0.0.1 -16:29.751 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8f4d9d75-032b-7d97-bc6f-6afd966bb392 127.0.0.1 -2021-10-31T23:16:29.751Z [INFO] agent.router: Initializing LAN area manager -16:29.751 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp -16:29.752 [INFO] TestAgent.server: Adding LAN server: server="Node-8f4d9d75-032b-7d97-bc6f-6afd966bb392 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" -16:29.752 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8f4d9d75-032b-7d97-bc6f-6afd966bb392.dc1 area=wan -16:29.752 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp -16:29.752 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp -16:29.752 [INFO] TestAgent: started state syncer -16:29.753 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp -16:29.789 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:29.789 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 -16:29.790 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:29.790 [DEBUG] TestAgent.server.raft: vote granted: from=8f4d9d75-032b-7d97-bc6f-6afd966bb392 term=2 tally=1 -16:29.790 [INFO] TestAgent.server.raft: election won: tally=1 -16:29.790 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" -16:29.790 [INFO] TestAgent.server: cluster leadership acquired -16:29.790 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13069 -16:29.790 [INFO] TestAgent.server: New leader elected: payload=Node-8f4d9d75-032b-7d97-bc6f-6afd966bb392 -16:29.791 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:29.836 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:29.836 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:29.836 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:29.836 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:29.836 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:29.836 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8f4d9d75-032b-7d97-bc6f-6afd966bb392 -16:29.836 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8f4d9d75-032b-7d97-bc6f-6afd966bb392 -16:29.837 [INFO] TestAgent.server: federation state anti-entropy synced -16:29.882 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:29.882 [INFO] TestAgent: Synced node info -16:29.882 [DEBUG] TestAgent: Node info in sync -16:30.102 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:30.102 [DEBUG] TestAgent: Node info in sync -16:30.203 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/ca/configuration from=127.0.0.1:52028 latency=8.645277ms -16:30.209 [INFO] TestAgent: Requesting shutdown -16:30.209 [INFO] TestAgent.server: shutting down server -16:30.209 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:30.209 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:30.209 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:30.209 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:30.209 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:30.209 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:30.209 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:30.210 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:30.210 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:30.210Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:30.210Z [INFO] agent.router.manager: shutting down -16:30.210 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:30.210 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:30.229 [INFO] TestAgent: consul server down -16:30.229 [INFO] TestAgent: shutdown complete -16:30.229 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp -16:30.229 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp -16:30.229 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp -16:30.729 [INFO] TestAgent: Waiting for endpoints to shut down -16:30.729 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectCAGetConfigCommand (1.17s) +2022-12-04T05:49:43.163Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:43.177Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:43.179 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fe9c9860-70fd-db3b-23bc-5d9c888e8fa2 Address:127.0.0.1:19195}]" +49:43.180 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= +49:43.180 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fe9c9860-70fd-db3b-23bc-5d9c888e8fa2.dc1 127.0.0.1 +49:43.195 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fe9c9860-70fd-db3b-23bc-5d9c888e8fa2 127.0.0.1 +2022-12-04T05:49:43.195Z [INFO] agent.router: Initializing LAN area manager +49:43.195 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fe9c9860-70fd-db3b-23bc-5d9c888e8fa2.dc1 area=wan +49:43.195 [INFO] TestAgent.server: Adding LAN server: server="Node-fe9c9860-70fd-db3b-23bc-5d9c888e8fa2 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +49:43.195 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp +49:43.196 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp +49:43.196 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp +49:43.196 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp +49:43.196 [INFO] TestAgent: started state syncer +49:43.242 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:43.242 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 +49:43.242 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:43.242 [DEBUG] TestAgent.server.raft: vote granted: from=fe9c9860-70fd-db3b-23bc-5d9c888e8fa2 term=2 tally=1 +49:43.242 [INFO] TestAgent.server.raft: election won: tally=1 +49:43.242 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" +49:43.242 [INFO] TestAgent.server: cluster leadership acquired +49:43.242 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 +49:43.243 [INFO] TestAgent.server: New leader elected: payload=Node-fe9c9860-70fd-db3b-23bc-5d9c888e8fa2 +49:43.243 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:43.259 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:43.259 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:43.259 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:43.259 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:43.259 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:43.259 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe9c9860-70fd-db3b-23bc-5d9c888e8fa2 +49:43.259 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fe9c9860-70fd-db3b-23bc-5d9c888e8fa2 +49:43.259 [INFO] TestAgent.server: federation state anti-entropy synced +49:43.306 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:43.307 [INFO] TestAgent: Synced node info +49:43.307 [DEBUG] TestAgent: Node info in sync +49:43.374 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/ca/configuration from=127.0.0.1:40366 latency=388.223µs +49:43.380 [INFO] TestAgent: Requesting shutdown +49:43.381 [INFO] TestAgent.server: shutting down server +49:43.381 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:43.381 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:43.381 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:43.381 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:43.381 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:43.381 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:43.381 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:43.381 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:43.381 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:43.381 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:43.381 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:43.381Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:43.381Z [INFO] agent.router.manager: shutting down +49:43.381 [INFO] TestAgent: consul server down +49:43.381 [INFO] TestAgent: shutdown complete +49:43.381 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp +49:43.381 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp +49:43.381 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp +49:43.884 [INFO] TestAgent: Waiting for endpoints to shut down +49:43.884 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectCAGetConfigCommand (0.75s) PASS -ok github.com/hashicorp/consul/command/connect/ca/get 2.164s +ok github.com/hashicorp/consul/command/connect/ca/get 0.831s === RUN TestConnectCASetConfigCommand_noTabs === PAUSE TestConnectCASetConfigCommand_noTabs === RUN TestConnectCASetConfigCommand @@ -78747,66 +78926,66 @@ --- PASS: TestConnectCASetConfigCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:16:24.884Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:24.935Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:24.937 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ebad96d9-a854-1ab8-aee7-33c4e27ecfc7 Address:127.0.0.1:22258}]" -16:24.937 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22258 [Follower]" leader= -16:24.946 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ebad96d9-a854-1ab8-aee7-33c4e27ecfc7.dc1 127.0.0.1 -16:24.947 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ebad96d9-a854-1ab8-aee7-33c4e27ecfc7 127.0.0.1 -2021-10-31T23:16:24.947Z [INFO] agent.router: Initializing LAN area manager -16:24.947 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=udp -16:24.947 [INFO] TestAgent.server: Adding LAN server: server="Node-ebad96d9-a854-1ab8-aee7-33c4e27ecfc7 (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -16:24.947 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ebad96d9-a854-1ab8-aee7-33c4e27ecfc7.dc1 area=wan -16:24.947 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=tcp -16:24.948 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22254 network=tcp -16:24.948 [INFO] TestAgent: started state syncer -16:24.948 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22259 network=tcp -16:24.982 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:24.982 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22258 [Candidate]" term=2 -16:24.982 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:24.982 [DEBUG] TestAgent.server.raft: vote granted: from=ebad96d9-a854-1ab8-aee7-33c4e27ecfc7 term=2 tally=1 -16:24.982 [INFO] TestAgent.server.raft: election won: tally=1 -16:24.982 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22258 [Leader]" -16:24.982 [INFO] TestAgent.server: cluster leadership acquired -16:24.982 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22258 -16:24.983 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:24.985 [INFO] TestAgent.server: New leader elected: payload=Node-ebad96d9-a854-1ab8-aee7-33c4e27ecfc7 -16:25.034 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:25.034 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:25.034 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:25.034 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:25.034 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:25.034 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ebad96d9-a854-1ab8-aee7-33c4e27ecfc7 -16:25.034 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ebad96d9-a854-1ab8-aee7-33c4e27ecfc7 -16:25.035 [INFO] TestAgent.server: federation state anti-entropy synced -16:25.059 [INFO] TestAgent.server.connect: CA provider config updated -16:25.059 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/ca/configuration from=127.0.0.1:49920 latency=837.668µs -16:25.060 [INFO] TestAgent: Requesting shutdown -16:25.060 [INFO] TestAgent.server: shutting down server -16:25.060 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:25.060 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:25.060 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:25.060 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:25.060 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:25.060 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:25.060 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:25.060 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -16:25.060 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:25.060 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:25.060 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:25.060 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:25.060Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:25.060Z [INFO] agent.router.manager: shutting down -16:25.060 [INFO] TestAgent: consul server down -16:25.060 [INFO] TestAgent: shutdown complete -16:25.060 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=tcp -16:25.060 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=udp -16:25.060 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22254 network=tcp -16:25.560 [INFO] TestAgent: Waiting for endpoints to shut down -16:25.560 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectCASetConfigCommand (0.81s) +2022-12-04T05:49:47.935Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:47.952Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:47.954 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e14010f8-1bfa-e9d0-e0b7-1326cf6f6c2a Address:127.0.0.1:30426}]" +49:47.954 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= +49:47.955 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e14010f8-1bfa-e9d0-e0b7-1326cf6f6c2a.dc1 127.0.0.1 +49:47.955 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e14010f8-1bfa-e9d0-e0b7-1326cf6f6c2a 127.0.0.1 +2022-12-04T05:49:47.955Z [INFO] agent.router: Initializing LAN area manager +49:47.955 [INFO] TestAgent.server: Adding LAN server: server="Node-e14010f8-1bfa-e9d0-e0b7-1326cf6f6c2a (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +49:47.955 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e14010f8-1bfa-e9d0-e0b7-1326cf6f6c2a.dc1 area=wan +49:47.955 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp +49:47.955 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp +49:47.956 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp +49:47.956 [INFO] TestAgent: started state syncer +49:47.956 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp +49:48.015 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:48.015 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 +49:48.015 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:48.015 [DEBUG] TestAgent.server.raft: vote granted: from=e14010f8-1bfa-e9d0-e0b7-1326cf6f6c2a term=2 tally=1 +49:48.015 [INFO] TestAgent.server.raft: election won: tally=1 +49:48.015 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" +49:48.015 [INFO] TestAgent.server: cluster leadership acquired +49:48.015 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30426 +49:48.015 [INFO] TestAgent.server: New leader elected: payload=Node-e14010f8-1bfa-e9d0-e0b7-1326cf6f6c2a +49:48.016 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:48.032 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:48.032 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:48.032 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:48.032 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:48.032 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:48.032 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e14010f8-1bfa-e9d0-e0b7-1326cf6f6c2a +49:48.033 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e14010f8-1bfa-e9d0-e0b7-1326cf6f6c2a +49:48.033 [INFO] TestAgent.server: federation state anti-entropy synced +49:48.289 [INFO] TestAgent.server.connect: CA provider config updated +49:48.289 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/ca/configuration from=127.0.0.1:50376 latency=780.045µs +49:48.291 [INFO] TestAgent: Requesting shutdown +49:48.291 [INFO] TestAgent.server: shutting down server +49:48.291 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:48.291 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:48.291 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:48.291 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:48.291 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:48.291 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:48.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:48.291 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:48.291 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:48.291 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:48.291 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:48.291 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:48.291Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:48.291Z [INFO] agent.router.manager: shutting down +49:48.292 [INFO] TestAgent: consul server down +49:48.292 [INFO] TestAgent: shutdown complete +49:48.292 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp +49:48.292 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp +49:48.292 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp +49:48.792 [INFO] TestAgent: Waiting for endpoints to shut down +49:48.792 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectCASetConfigCommand (0.89s) PASS -ok github.com/hashicorp/consul/command/connect/ca/set 1.016s +ok github.com/hashicorp/consul/command/connect/ca/set 0.944s === RUN TestBootstrapConfig_ConfigureArgs === RUN TestBootstrapConfig_ConfigureArgs/defaults === RUN TestBootstrapConfig_ConfigureArgs/extra-stats-sinks @@ -78831,7 +79010,7 @@ === RUN TestBootstrapConfig_ConfigureArgs/ready-bind-addr === RUN TestBootstrapConfig_ConfigureArgs/ready-bind-addr-with-overrides === RUN TestBootstrapConfig_ConfigureArgs/ready-bind-addr-and-prometheus-and-stats ---- PASS: TestBootstrapConfig_ConfigureArgs (0.02s) +--- PASS: TestBootstrapConfig_ConfigureArgs (0.01s) --- PASS: TestBootstrapConfig_ConfigureArgs/defaults (0.00s) --- PASS: TestBootstrapConfig_ConfigureArgs/extra-stats-sinks (0.00s) --- PASS: TestBootstrapConfig_ConfigureArgs/simple-statsd-sink (0.00s) @@ -81870,33 +82049,33 @@ ] } } ---- PASS: TestGenerateConfig (0.49s) +--- PASS: TestGenerateConfig (0.13s) --- PASS: TestGenerateConfig/no-args (0.00s) - --- PASS: TestGenerateConfig/defaults (0.02s) - --- PASS: TestGenerateConfig/token-arg (0.05s) - --- PASS: TestGenerateConfig/token-env (0.01s) - --- PASS: TestGenerateConfig/token-file-arg (0.02s) - --- PASS: TestGenerateConfig/token-file-env (0.01s) - --- PASS: TestGenerateConfig/grpc-addr-flag (0.02s) - --- PASS: TestGenerateConfig/grpc-addr-env (0.01s) - --- PASS: TestGenerateConfig/grpc-addr-unix (0.02s) - --- PASS: TestGenerateConfig/grpc-addr-config (0.04s) - --- PASS: TestGenerateConfig/access-log-path (0.03s) + --- PASS: TestGenerateConfig/defaults (0.01s) + --- PASS: TestGenerateConfig/token-arg (0.00s) + --- PASS: TestGenerateConfig/token-env (0.00s) + --- PASS: TestGenerateConfig/token-file-arg (0.00s) + --- PASS: TestGenerateConfig/token-file-env (0.00s) + --- PASS: TestGenerateConfig/grpc-addr-flag (0.01s) + --- PASS: TestGenerateConfig/grpc-addr-env (0.00s) + --- PASS: TestGenerateConfig/grpc-addr-unix (0.00s) + --- PASS: TestGenerateConfig/grpc-addr-config (0.00s) + --- PASS: TestGenerateConfig/access-log-path (0.01s) --- PASS: TestGenerateConfig/missing-ca-file (0.00s) - --- PASS: TestGenerateConfig/existing-ca-file (0.02s) + --- PASS: TestGenerateConfig/existing-ca-file (0.00s) --- PASS: TestGenerateConfig/missing-ca-path (0.00s) --- PASS: TestGenerateConfig/existing-ca-path (0.01s) - --- PASS: TestGenerateConfig/custom-bootstrap (0.03s) - --- PASS: TestGenerateConfig/extra_-single (0.01s) - --- PASS: TestGenerateConfig/extra_-multiple (0.02s) - --- PASS: TestGenerateConfig/stats-config-override (0.01s) - --- PASS: TestGenerateConfig/zipkin-tracing-config (0.03s) - --- PASS: TestGenerateConfig/CONSUL_HTTP_ADDR-with-https-scheme-enables-tls (0.04s) - --- PASS: TestGenerateConfig/ingress-gateway (0.01s) - --- PASS: TestGenerateConfig/ingress-gateway-address-specified (0.02s) - --- PASS: TestGenerateConfig/ingress-gateway-register-with-service-without-proxy-id (0.02s) - --- PASS: TestGenerateConfig/ingress-gateway-register-with-service-and-proxy-id (0.02s) - --- PASS: TestGenerateConfig/ingress-gateway-no-auto-register (0.04s) + --- PASS: TestGenerateConfig/custom-bootstrap (0.00s) + --- PASS: TestGenerateConfig/extra_-single (0.00s) + --- PASS: TestGenerateConfig/extra_-multiple (0.00s) + --- PASS: TestGenerateConfig/stats-config-override (0.00s) + --- PASS: TestGenerateConfig/zipkin-tracing-config (0.01s) + --- PASS: TestGenerateConfig/CONSUL_HTTP_ADDR-with-https-scheme-enables-tls (0.00s) + --- PASS: TestGenerateConfig/ingress-gateway (0.00s) + --- PASS: TestGenerateConfig/ingress-gateway-address-specified (0.01s) + --- PASS: TestGenerateConfig/ingress-gateway-register-with-service-without-proxy-id (0.01s) + --- PASS: TestGenerateConfig/ingress-gateway-register-with-service-and-proxy-id (0.00s) + --- PASS: TestGenerateConfig/ingress-gateway-no-auto-register (0.01s) === RUN TestEnvoy_GatewayRegistration === PAUSE TestEnvoy_GatewayRegistration === RUN TestEnvoyCommand_canBindInternal @@ -81907,12 +82086,12 @@ === RUN TestExecEnvoy/hot-restart-version === RUN TestExecEnvoy/hot-restart-version#01 === RUN TestExecEnvoy/hot-restart-version#02 ---- PASS: TestExecEnvoy (6.83s) - --- PASS: TestExecEnvoy/default (1.23s) - --- PASS: TestExecEnvoy/hot-restart-epoch (1.08s) - --- PASS: TestExecEnvoy/hot-restart-version (1.77s) - --- PASS: TestExecEnvoy/hot-restart-version#01 (1.36s) - --- PASS: TestExecEnvoy/hot-restart-version#02 (1.39s) +--- PASS: TestExecEnvoy (0.51s) + --- PASS: TestExecEnvoy/default (0.09s) + --- PASS: TestExecEnvoy/hot-restart-epoch (0.11s) + --- PASS: TestExecEnvoy/hot-restart-version (0.10s) + --- PASS: TestExecEnvoy/hot-restart-version#01 (0.11s) + --- PASS: TestExecEnvoy/hot-restart-version#02 (0.10s) === RUN TestHelperProcess --- PASS: TestHelperProcess (0.00s) === RUN TestServiceAddressValue_Value @@ -81949,42 +82128,42 @@ --- PASS: TestEnvoyCommand_noTabs (0.00s) === CONT TestEnvoyCommand_canBindInternal === RUN TestEnvoyCommand_canBindInternal/IPNet -=== RUN TestEnvoyCommand_canBindInternal/IPNet/ipv4_not_found -=== RUN TestEnvoyCommand_canBindInternal/IPNet/ipv6_not_found === RUN TestEnvoyCommand_canBindInternal/IPNet/ipv4 === RUN TestEnvoyCommand_canBindInternal/IPNet/secondary_ipv4 === RUN TestEnvoyCommand_canBindInternal/IPNet/ipv6 +=== RUN TestEnvoyCommand_canBindInternal/IPNet/ipv4_not_found +=== RUN TestEnvoyCommand_canBindInternal/IPNet/ipv6_not_found === RUN TestEnvoyCommand_canBindInternal/IPAddr -=== RUN TestEnvoyCommand_canBindInternal/IPAddr/ipv4_not_found -=== RUN TestEnvoyCommand_canBindInternal/IPAddr/ipv6_not_found === RUN TestEnvoyCommand_canBindInternal/IPAddr/ipv4 === RUN TestEnvoyCommand_canBindInternal/IPAddr/secondary_ipv4 === RUN TestEnvoyCommand_canBindInternal/IPAddr/ipv6 +=== RUN TestEnvoyCommand_canBindInternal/IPAddr/ipv4_not_found +=== RUN TestEnvoyCommand_canBindInternal/IPAddr/ipv6_not_found --- PASS: TestEnvoyCommand_canBindInternal (0.00s) --- PASS: TestEnvoyCommand_canBindInternal/IPNet (0.00s) - --- PASS: TestEnvoyCommand_canBindInternal/IPNet/ipv4_not_found (0.00s) - --- PASS: TestEnvoyCommand_canBindInternal/IPNet/ipv6_not_found (0.00s) --- PASS: TestEnvoyCommand_canBindInternal/IPNet/ipv4 (0.00s) --- PASS: TestEnvoyCommand_canBindInternal/IPNet/secondary_ipv4 (0.00s) --- PASS: TestEnvoyCommand_canBindInternal/IPNet/ipv6 (0.00s) + --- PASS: TestEnvoyCommand_canBindInternal/IPNet/ipv4_not_found (0.00s) + --- PASS: TestEnvoyCommand_canBindInternal/IPNet/ipv6_not_found (0.00s) --- PASS: TestEnvoyCommand_canBindInternal/IPAddr (0.00s) - --- PASS: TestEnvoyCommand_canBindInternal/IPAddr/ipv4_not_found (0.00s) - --- PASS: TestEnvoyCommand_canBindInternal/IPAddr/ipv6_not_found (0.00s) --- PASS: TestEnvoyCommand_canBindInternal/IPAddr/ipv4 (0.00s) --- PASS: TestEnvoyCommand_canBindInternal/IPAddr/secondary_ipv4 (0.00s) --- PASS: TestEnvoyCommand_canBindInternal/IPAddr/ipv6 (0.00s) + --- PASS: TestEnvoyCommand_canBindInternal/IPAddr/ipv4_not_found (0.00s) + --- PASS: TestEnvoyCommand_canBindInternal/IPAddr/ipv6_not_found (0.00s) === CONT TestEnvoy_GatewayRegistration [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestEnvoyGateway_Validation === RUN TestEnvoyGateway_Validation/-register_for_non-gateway === RUN TestEnvoyGateway_Validation/-mesh-gateway_and_-gateway_cannot_be_combined +[INFO] freeport: detected ephemeral port range of [32768, 60999] +[INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestEnvoyGateway_Validation/no_proxy_registration_specified_nor_discovered --- PASS: TestEnvoyGateway_Validation (0.01s) --- PASS: TestEnvoyGateway_Validation/-register_for_non-gateway (0.00s) --- PASS: TestEnvoyGateway_Validation/-mesh-gateway_and_-gateway_cannot_be_combined (0.00s) --- PASS: TestEnvoyGateway_Validation/no_proxy_registration_specified_nor_discovered (0.01s) -[INFO] freeport: detected ephemeral port range of [32768, 60999] -[INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestEnvoy_GatewayRegistration/register_gateway_with_proxy-id_and_name { "admin": { @@ -82015,7 +82194,7 @@ { "socket_address": { "address": "127.0.0.1", - "port_value": 24301 + "port_value": 10007 } } ] @@ -82205,7 +82384,7 @@ { "socket_address": { "address": "127.0.0.1", - "port_value": 24301 + "port_value": 10007 } } ] @@ -82395,7 +82574,7 @@ { "socket_address": { "address": "127.0.0.1", - "port_value": 24301 + "port_value": 10007 } } ] @@ -82585,7 +82764,7 @@ { "socket_address": { "address": "127.0.0.1", - "port_value": 24301 + "port_value": 10007 } } ] @@ -82745,124 +82924,120 @@ ] } } -2021-10-31T23:16:40.831Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:40.881Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:40.884 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3e3ac0b2-abd4-77fb-bba8-ffdb57aa39f8 Address:127.0.0.1:24300}]" -16:40.884 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3e3ac0b2-abd4-77fb-bba8-ffdb57aa39f8.dc1 127.0.0.1 -16:40.884 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3e3ac0b2-abd4-77fb-bba8-ffdb57aa39f8 127.0.0.1 -2021-10-31T23:16:40.885Z [INFO] agent.router: Initializing LAN area manager -16:40.902 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= -16:40.902 [INFO] TestAgent.server: Adding LAN server: server="Node-3e3ac0b2-abd4-77fb-bba8-ffdb57aa39f8 (Addr: tcp/127.0.0.1:24300) (DC: dc1)" -16:40.902 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3e3ac0b2-abd4-77fb-bba8-ffdb57aa39f8.dc1 area=wan -16:40.902 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp -16:40.902 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp -16:40.903 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp -16:40.903 [INFO] TestAgent: started state syncer -16:40.904 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp -16:40.941 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:40.941 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 -16:40.941 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:40.941 [DEBUG] TestAgent.server.raft: vote granted: from=3e3ac0b2-abd4-77fb-bba8-ffdb57aa39f8 term=2 tally=1 -16:40.941 [INFO] TestAgent.server.raft: election won: tally=1 -16:40.941 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" -16:40.941 [INFO] TestAgent.server: cluster leadership acquired -16:40.942 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24300 -16:40.943 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:40.945 [INFO] TestAgent.server: New leader elected: payload=Node-3e3ac0b2-abd4-77fb-bba8-ffdb57aa39f8 -16:40.997 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:40.997 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:40.997 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:40.997 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:40.997 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:40.997 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3e3ac0b2-abd4-77fb-bba8-ffdb57aa39f8 -16:40.997 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3e3ac0b2-abd4-77fb-bba8-ffdb57aa39f8 -16:41.006 [INFO] TestAgent.server: federation state anti-entropy synced -16:41.120 [INFO] TestAgent: Synced node info -16:41.120 [INFO] TestAgent: Synced service: service=us-ingress-1 -16:41.120 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 -16:41.120 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:36572 latency=2.569259ms -16:41.121 [WARN] TestAgent.server.catalog: no terminating-gateway or ingress-gateway associated with this gateway: gateway=us-ingress -16:41.123 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:36572 latency=1.379462ms -16:41.126 [ERROR] TestAgent.proxycfg: watch error: service_id=us-ingress-1 id=gateway-config error="invalid type for config entry: " -16:41.134 [WARN] TestAgent.server.catalog: no terminating-gateway or ingress-gateway associated with this gateway: gateway=us-ingress -16:41.143 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress-1 from=127.0.0.1:36572 latency=17.770033ms -16:41.154 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress-1 from=127.0.0.1:36574 latency=310.083µs -16:41.158 [DEBUG] TestAgent: Node info in sync -16:41.158 [DEBUG] TestAgent: Service in sync: service=us-ingress-1 -16:41.173 [INFO] TestAgent: Synced service: service=us-ingress -16:41.173 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 -16:41.173 [DEBUG] TestAgent: Check in sync: check=service:us-ingress -16:41.173 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:36576 latency=15.677586ms -16:41.186 [ERROR] TestAgent.proxycfg: watch error: service_id=us-ingress id=gateway-config error="invalid type for config entry: " -16:41.192 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:36576 latency=2.703971ms -16:41.211 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress from=127.0.0.1:36576 latency=320.754µs -16:41.213 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress from=127.0.0.1:36574 latency=286.345µs -16:41.222 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:41.222 [DEBUG] TestAgent: Node info in sync -16:41.222 [DEBUG] TestAgent: Service in sync: service=us-ingress -16:41.222 [DEBUG] TestAgent: Service in sync: service=us-ingress-1 -16:41.222 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 -16:41.222 [DEBUG] TestAgent: Check in sync: check=service:us-ingress -16:41.223 [DEBUG] TestAgent: Node info in sync -16:41.223 [INFO] TestAgent: Synced service: service=ingress-gateway -16:41.223 [DEBUG] TestAgent: Service in sync: service=us-ingress-1 -16:41.223 [DEBUG] TestAgent: Service in sync: service=us-ingress -16:41.223 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 -16:41.223 [DEBUG] TestAgent: Check in sync: check=service:us-ingress -16:41.223 [DEBUG] TestAgent: Check in sync: check=service:ingress-gateway -16:41.223 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:36578 latency=1.046174ms -16:41.224 [WARN] TestAgent.server.catalog: no terminating-gateway or ingress-gateway associated with this gateway: gateway=ingress-gateway -16:41.224 [WARN] TestAgent.server.catalog: no terminating-gateway or ingress-gateway associated with this gateway: gateway=ingress-gateway -16:41.224 [ERROR] TestAgent.proxycfg: watch error: service_id=ingress-gateway id=gateway-config error="invalid type for config entry: " -16:41.227 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:36578 latency=3.086225ms -16:41.236 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/ingress-gateway from=127.0.0.1:36578 latency=2.040941ms -16:41.246 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/ingress-gateway from=127.0.0.1:36574 latency=8.462084ms -16:41.252 [DEBUG] TestAgent: Node info in sync -16:41.253 [INFO] TestAgent: Synced service: service=us-ingress-1 -16:41.253 [DEBUG] TestAgent: Service in sync: service=us-ingress -16:41.253 [DEBUG] TestAgent: Service in sync: service=ingress-gateway -16:41.253 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 -16:41.253 [DEBUG] TestAgent: Check in sync: check=service:us-ingress -16:41.253 [DEBUG] TestAgent: Check in sync: check=service:ingress-gateway -16:41.253 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:36580 latency=1.406118ms -16:41.255 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:36580 latency=1.209955ms -16:41.257 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress-1 from=127.0.0.1:36580 latency=277.397µs -16:41.262 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress-1 from=127.0.0.1:36574 latency=248.939µs -16:41.262 [INFO] TestAgent: Requesting shutdown -16:41.262 [INFO] TestAgent.server: shutting down server -16:41.262 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:41.262 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:41.262 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:41.262 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:41.262 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:41.263 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:41.263 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:41.263 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:41.263 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:41.263 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:41.263 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:41.263Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:41.263Z [INFO] agent.router.manager: shutting down -16:41.263 [INFO] TestAgent: consul server down -16:41.263 [INFO] TestAgent: shutdown complete -16:41.263 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp -16:41.263 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp -16:41.263 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp -16:41.765 [INFO] TestAgent: Waiting for endpoints to shut down -16:41.765 [INFO] TestAgent: Endpoints down ---- PASS: TestEnvoy_GatewayRegistration (1.13s) - --- PASS: TestEnvoy_GatewayRegistration/register_gateway_with_proxy-id_and_name (0.05s) - --- PASS: TestEnvoy_GatewayRegistration/register_gateway_without_proxy-id_with_name (0.06s) - --- PASS: TestEnvoy_GatewayRegistration/register_gateway_without_proxy-id_and_without_name (0.04s) +2022-12-04T05:49:49.196Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:49.207Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:49.209 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3c655550-549a-89f7-ba75-da571b65b33f Address:127.0.0.1:10006}]" +49:49.209 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10006 [Follower]" leader= +49:49.211 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3c655550-549a-89f7-ba75-da571b65b33f.dc1 127.0.0.1 +49:49.213 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3c655550-549a-89f7-ba75-da571b65b33f 127.0.0.1 +2022-12-04T05:49:49.213Z [INFO] agent.router: Initializing LAN area manager +49:49.213 [INFO] TestAgent.server: Adding LAN server: server="Node-3c655550-549a-89f7-ba75-da571b65b33f (Addr: tcp/127.0.0.1:10006) (DC: dc1)" +49:49.213 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3c655550-549a-89f7-ba75-da571b65b33f.dc1 area=wan +49:49.214 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=tcp +49:49.214 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=udp +49:49.214 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10002 network=tcp +49:49.214 [INFO] TestAgent: started state syncer +49:49.214 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10007 network=tcp +49:49.261 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:49.262 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10006 [Candidate]" term=2 +49:49.262 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:49.262 [DEBUG] TestAgent.server.raft: vote granted: from=3c655550-549a-89f7-ba75-da571b65b33f term=2 tally=1 +49:49.262 [INFO] TestAgent.server.raft: election won: tally=1 +49:49.262 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10006 [Leader]" +49:49.262 [INFO] TestAgent.server: cluster leadership acquired +49:49.262 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10006 +49:49.262 [INFO] TestAgent.server: New leader elected: payload=Node-3c655550-549a-89f7-ba75-da571b65b33f +49:49.263 [INFO] TestAgent: Synced node info +49:49.263 [DEBUG] TestAgent: Node info in sync +49:49.263 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:49.279 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:49.279 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:49.279 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:49.279 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:49.279 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:49.279 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3c655550-549a-89f7-ba75-da571b65b33f +49:49.279 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3c655550-549a-89f7-ba75-da571b65b33f +49:49.280 [INFO] TestAgent.server: federation state anti-entropy synced +49:49.592 [DEBUG] TestAgent: Node info in sync +49:49.592 [WARN] TestAgent.server.catalog: no terminating-gateway or ingress-gateway associated with this gateway: gateway=us-ingress +49:49.592 [WARN] TestAgent.server.catalog: no terminating-gateway or ingress-gateway associated with this gateway: gateway=us-ingress +49:49.592 [ERROR] TestAgent.proxycfg: watch error: service_id=us-ingress-1 id=gateway-config error="invalid type for config entry: " +49:49.592 [INFO] TestAgent: Synced service: service=us-ingress-1 +49:49.592 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 +49:49.592 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:56252 latency=1.851236ms +49:49.599 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:56252 latency=6.207277ms +49:49.603 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress-1 from=127.0.0.1:56252 latency=1.0301ms +49:49.605 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress-1 from=127.0.0.1:56254 latency=298.484µs +49:49.609 [DEBUG] TestAgent: Node info in sync +49:49.609 [DEBUG] TestAgent: Service in sync: service=us-ingress-1 +49:49.609 [ERROR] TestAgent.proxycfg: watch error: service_id=us-ingress id=gateway-config error="invalid type for config entry: " +49:49.610 [INFO] TestAgent: Synced service: service=us-ingress +49:49.610 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 +49:49.610 [DEBUG] TestAgent: Check in sync: check=service:us-ingress +49:49.610 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:56256 latency=1.585965ms +49:49.611 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:56256 latency=1.002067ms +49:49.614 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress from=127.0.0.1:56256 latency=259.333µs +49:49.615 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress from=127.0.0.1:56254 latency=303.681µs +49:49.618 [DEBUG] TestAgent: Node info in sync +49:49.618 [DEBUG] TestAgent: Service in sync: service=us-ingress-1 +49:49.618 [DEBUG] TestAgent: Service in sync: service=us-ingress +49:49.618 [INFO] TestAgent: Synced service: service=ingress-gateway +49:49.618 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 +49:49.618 [DEBUG] TestAgent: Check in sync: check=service:us-ingress +49:49.618 [DEBUG] TestAgent: Check in sync: check=service:ingress-gateway +49:49.618 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:56258 latency=999.381µs +49:49.619 [WARN] TestAgent.server.catalog: no terminating-gateway or ingress-gateway associated with this gateway: gateway=ingress-gateway +49:49.619 [WARN] TestAgent.server.catalog: no terminating-gateway or ingress-gateway associated with this gateway: gateway=ingress-gateway +49:49.619 [ERROR] TestAgent.proxycfg: watch error: service_id=ingress-gateway id=gateway-config error="invalid type for config entry: " +49:49.620 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:56258 latency=1.117406ms +49:49.622 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/ingress-gateway from=127.0.0.1:56258 latency=195.744µs +49:49.623 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/ingress-gateway from=127.0.0.1:56254 latency=130.089µs +49:49.633 [DEBUG] TestAgent: Node info in sync +49:49.633 [DEBUG] TestAgent: Service in sync: service=us-ingress +49:49.633 [DEBUG] TestAgent: Service in sync: service=ingress-gateway +49:49.633 [INFO] TestAgent: Synced service: service=us-ingress-1 +49:49.633 [DEBUG] TestAgent: Check in sync: check=service:us-ingress-1 +49:49.633 [DEBUG] TestAgent: Check in sync: check=service:us-ingress +49:49.633 [DEBUG] TestAgent: Check in sync: check=service:ingress-gateway +49:49.633 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:56260 latency=1.191283ms +49:49.635 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:56260 latency=1.137988ms +49:49.638 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress-1 from=127.0.0.1:56260 latency=1.090727ms +49:49.640 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/us-ingress-1 from=127.0.0.1:56254 latency=379.653µs +49:49.641 [INFO] TestAgent: Requesting shutdown +49:49.641 [INFO] TestAgent.server: shutting down server +49:49.641 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:49.641 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:49.641 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:49.641 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:49.641 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:49.641 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:49.641 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:49.641 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:49.641 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:49.641 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:49.641 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:49.641Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:49.641Z [INFO] agent.router.manager: shutting down +49:49.641 [INFO] TestAgent: consul server down +49:49.641 [INFO] TestAgent: shutdown complete +49:49.641 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=tcp +49:49.641 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=udp +49:49.641 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10002 network=tcp +49:50.141 [INFO] TestAgent: Waiting for endpoints to shut down +49:50.142 [INFO] TestAgent: Endpoints down +--- PASS: TestEnvoy_GatewayRegistration (0.99s) + --- PASS: TestEnvoy_GatewayRegistration/register_gateway_with_proxy-id_and_name (0.02s) + --- PASS: TestEnvoy_GatewayRegistration/register_gateway_without_proxy-id_with_name (0.01s) + --- PASS: TestEnvoy_GatewayRegistration/register_gateway_without_proxy-id_and_without_name (0.01s) --- PASS: TestEnvoy_GatewayRegistration/register_gateway_with_proxy-id_without_name (0.01s) PASS -ok github.com/hashicorp/consul/command/connect/envoy 8.670s +ok github.com/hashicorp/consul/command/connect/envoy 1.694s === RUN TestConnectEnvoyPipeBootstrapCommand_noTabs === PAUSE TestConnectEnvoyPipeBootstrapCommand_noTabs === CONT TestConnectEnvoyPipeBootstrapCommand_noTabs --- PASS: TestConnectEnvoyPipeBootstrapCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/connect/envoy/pipe-bootstrap 0.051s +ok github.com/hashicorp/consul/command/connect/envoy/pipe-bootstrap 0.004s === RUN TestConnectExpose === PAUSE TestConnectExpose === RUN TestConnectExpose_invalidFlags @@ -82870,225 +83045,223 @@ === RUN TestConnectExpose_existingConfig === PAUSE TestConnectExpose_existingConfig === CONT TestConnectExpose -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestConnectExpose_existingConfig +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestConnectExpose_invalidFlags [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestConnectExpose_invalidFlags/missing_service === RUN TestConnectExpose_invalidFlags/missing_gateway === RUN TestConnectExpose_invalidFlags/missing_port -2021-10-31T23:16:44.145Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:44.277Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:44.283 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7eeabef4-d4f7-16b7-0489-de68b11eef93 Address:127.0.0.1:18188}]" -16:44.284 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7eeabef4-d4f7-16b7-0489-de68b11eef93.dc1 127.0.0.1 -16:44.285 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7eeabef4-d4f7-16b7-0489-de68b11eef93 127.0.0.1 -2021-10-31T23:16:44.285Z [INFO] agent.router: Initializing LAN area manager -16:44.285 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18183 network=udp -16:44.285 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18188 [Follower]" leader= -16:44.285 [INFO] TestAgent.server: Adding LAN server: server="Node-7eeabef4-d4f7-16b7-0489-de68b11eef93 (Addr: tcp/127.0.0.1:18188) (DC: dc1)" -16:44.286 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7eeabef4-d4f7-16b7-0489-de68b11eef93.dc1 area=wan -16:44.286 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18183 network=tcp -16:44.286 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18184 network=tcp -16:44.286 [INFO] TestAgent: started state syncer -16:44.286 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18189 network=tcp -16:44.335 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:44.335 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18188 [Candidate]" term=2 -16:44.335 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:44.335 [DEBUG] TestAgent.server.raft: vote granted: from=7eeabef4-d4f7-16b7-0489-de68b11eef93 term=2 tally=1 -16:44.335 [INFO] TestAgent.server.raft: election won: tally=1 -16:44.335 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18188 [Leader]" -16:44.335 [INFO] TestAgent.server: cluster leadership acquired -16:44.335 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18188 -16:44.336 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:44.336 [INFO] TestAgent.server: New leader elected: payload=Node-7eeabef4-d4f7-16b7-0489-de68b11eef93 -16:44.354 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:44.354 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:44.354 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:44.354 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:44.354 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:44.354 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7eeabef4-d4f7-16b7-0489-de68b11eef93 -16:44.354 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7eeabef4-d4f7-16b7-0489-de68b11eef93 -16:44.355 [INFO] TestAgent.server: federation state anti-entropy synced -16:44.398 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:44.398 [INFO] TestAgent: Synced node info -16:44.628 [INFO] TestAgent: Requesting shutdown -16:44.628 [INFO] TestAgent.server: shutting down server -16:44.628 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:44.628 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:44.628 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:44.628 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:44.628 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:44.628 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:44.628 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:44.628 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:44.628 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:44.628 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:44.628 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:44.629Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:44.629Z [INFO] agent.router.manager: shutting down -16:44.629 [INFO] TestAgent: consul server down -16:44.629 [INFO] TestAgent: shutdown complete -16:44.629 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18183 network=tcp -16:44.629 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18183 network=udp -16:44.629 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18184 network=tcp -16:45.129 [INFO] TestAgent: Waiting for endpoints to shut down -16:45.129 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectExpose_invalidFlags (1.18s) +2022-12-04T05:49:49.977Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:49.993Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:49.996 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:04c0da5b-4a76-7d62-dd29-2e69a79a55a6 Address:127.0.0.1:25328}]" +49:49.997 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-04c0da5b-4a76-7d62-dd29-2e69a79a55a6.dc1 127.0.0.1 +49:49.998 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-04c0da5b-4a76-7d62-dd29-2e69a79a55a6 127.0.0.1 +2022-12-04T05:49:49.998Z [INFO] agent.router: Initializing LAN area manager +49:49.999 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25328 [Follower]" leader= +49:50.000 [INFO] TestAgent.server: Adding LAN server: server="Node-04c0da5b-4a76-7d62-dd29-2e69a79a55a6 (Addr: tcp/127.0.0.1:25328) (DC: dc1)" +49:50.000 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=udp +49:50.000 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-04c0da5b-4a76-7d62-dd29-2e69a79a55a6.dc1 area=wan +49:50.000 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=tcp +49:50.003 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25324 network=tcp +49:50.003 [INFO] TestAgent: started state syncer +49:50.003 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25329 network=tcp +49:50.061 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:50.061 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25328 [Candidate]" term=2 +49:50.061 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:50.061 [DEBUG] TestAgent.server.raft: vote granted: from=04c0da5b-4a76-7d62-dd29-2e69a79a55a6 term=2 tally=1 +49:50.061 [INFO] TestAgent.server.raft: election won: tally=1 +49:50.061 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25328 [Leader]" +49:50.061 [INFO] TestAgent.server: cluster leadership acquired +49:50.062 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25328 +49:50.062 [INFO] TestAgent.server: New leader elected: payload=Node-04c0da5b-4a76-7d62-dd29-2e69a79a55a6 +49:50.062 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:50.078 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:50.078 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:50.078 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:50.078 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:50.078 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:50.078 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-04c0da5b-4a76-7d62-dd29-2e69a79a55a6 +49:50.078 [INFO] TestAgent.server: member joined, marking health alive: member=Node-04c0da5b-4a76-7d62-dd29-2e69a79a55a6 +49:50.078 [INFO] TestAgent.server: federation state anti-entropy synced +49:50.094 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:49024 latency=900.775µs +49:50.095 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:49024 latency=836.393µs +49:50.097 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:49024 latency=940.902µs +49:50.098 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:49026 latency=692.124µs +49:50.100 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=14 from=127.0.0.1:49026 latency=1.336889ms +49:50.101 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:49026 latency=221.176µs +49:50.101 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:49026 latency=442.588µs +49:50.102 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:49024 latency=102.972µs +49:50.103 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:49028 latency=179.586µs +49:50.109 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=15 from=127.0.0.1:49028 latency=5.93317ms +49:50.111 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:49028 latency=594.616µs +49:50.112 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:49028 latency=556.264µs +49:50.114 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:49024 latency=681.388µs +49:50.115 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:49030 latency=195.812µs +49:50.117 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=17 from=127.0.0.1:49030 latency=1.361081ms +49:50.117 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:49030 latency=136.198µs +49:50.118 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:49030 latency=396.347µs +49:50.119 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:49024 latency=123.112µs +49:50.119 [INFO] TestAgent: Requesting shutdown +49:50.119 [INFO] TestAgent.server: shutting down server +49:50.119 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:50.119 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:50.119 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:50.119 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:50.119 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:50.119 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:50.119 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:50.119 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:50.119 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:50.119 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:50.119 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:50.119 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:50.119Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:50.119Z [INFO] agent.router.manager: shutting down +49:50.119 [INFO] TestAgent: consul server down +49:50.119 [INFO] TestAgent: shutdown complete +49:50.119 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=tcp +49:50.119 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=udp +49:50.119 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25324 network=tcp +49:50.620 [INFO] TestAgent: Waiting for endpoints to shut down +49:50.620 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectExpose_existingConfig (0.68s) +2022-12-04T05:49:49.978Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:49.994Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:49.998 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:82a817dd-f94c-5781-3b1e-7519aa0dcf16 Address:127.0.0.1:25321}]" +49:50.000 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-82a817dd-f94c-5781-3b1e-7519aa0dcf16.dc1 127.0.0.1 +49:50.000 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25321 [Follower]" leader= +49:50.001 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-82a817dd-f94c-5781-3b1e-7519aa0dcf16 127.0.0.1 +2022-12-04T05:49:50.001Z [INFO] agent.router: Initializing LAN area manager +49:50.001 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=udp +49:50.001 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-82a817dd-f94c-5781-3b1e-7519aa0dcf16.dc1 area=wan +49:50.001 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=tcp +49:50.002 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25317 network=tcp +49:50.002 [INFO] TestAgent: started state syncer +49:50.002 [INFO] TestAgent.server: Adding LAN server: server="Node-82a817dd-f94c-5781-3b1e-7519aa0dcf16 (Addr: tcp/127.0.0.1:25321) (DC: dc1)" +49:50.002 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25322 network=tcp +49:50.069 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:50.069 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25321 [Candidate]" term=2 +49:50.069 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:50.069 [DEBUG] TestAgent.server.raft: vote granted: from=82a817dd-f94c-5781-3b1e-7519aa0dcf16 term=2 tally=1 +49:50.069 [INFO] TestAgent.server.raft: election won: tally=1 +49:50.069 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25321 [Leader]" +49:50.069 [INFO] TestAgent.server: cluster leadership acquired +49:50.069 [INFO] TestAgent.server: New leader elected: payload=Node-82a817dd-f94c-5781-3b1e-7519aa0dcf16 +49:50.069 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25321 +49:50.070 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:50.078 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:50.078 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:50.078 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:50.078 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:50.078 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:50.078 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-82a817dd-f94c-5781-3b1e-7519aa0dcf16 +49:50.078 [INFO] TestAgent.server: member joined, marking health alive: member=Node-82a817dd-f94c-5781-3b1e-7519aa0dcf16 +49:50.082 [INFO] TestAgent.server: federation state anti-entropy synced +49:50.175 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:50.176 [INFO] TestAgent: Synced node info +49:50.241 [ERROR] TestAgent.http: Request error: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:33952 error="Config entry not found for "ingress-gateway" / "ingress"" +49:50.241 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:33952 latency=241.566µs +49:50.245 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=0 from=127.0.0.1:33952 latency=1.110058ms +49:50.246 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:33952 latency=248.584µs +49:50.247 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:33952 latency=605.777µs +49:50.248 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:33954 latency=333.819µs +49:50.249 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:33954 latency=249.36µs +49:50.251 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:33956 latency=183.385µs +49:50.252 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=13 from=127.0.0.1:33956 latency=1.16718ms +49:50.253 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:33956 latency=223.508µs +49:50.253 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:33954 latency=129.572µs +49:50.254 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:33954 latency=117.483µs +49:50.255 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:33958 latency=129.032µs +49:50.256 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:33954 latency=186.804µs +49:50.256 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:33954 latency=165.798µs +49:50.256 [INFO] TestAgent: Requesting shutdown +49:50.257 [INFO] TestAgent.server: shutting down server +49:50.257 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:50.257 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:50.257 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:50.257 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:50.257 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:50.257 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:50.257 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:50.257 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:50.257 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:50.257 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +2022-12-04T05:49:50.257Z [INFO] agent.router.manager: shutting down +49:50.257 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +2022-12-04T05:49:50.257Z [INFO] agent.router.manager: shutting down +49:50.257 [INFO] TestAgent: consul server down +49:50.257 [INFO] TestAgent: shutdown complete +49:50.257 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=tcp +49:50.257 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=udp +49:50.257 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25317 network=tcp +49:50.757 [INFO] TestAgent: Waiting for endpoints to shut down +49:50.757 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectExpose (0.82s) +2022-12-04T05:49:49.977Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:49.991Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:49.997 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7a648ee6-6817-2ce3-737a-96134d730ddf Address:127.0.0.1:25335}]" +49:49.999 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25335 [Follower]" leader= +49:49.999 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7a648ee6-6817-2ce3-737a-96134d730ddf.dc1 127.0.0.1 +49:50.000 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7a648ee6-6817-2ce3-737a-96134d730ddf 127.0.0.1 +2022-12-04T05:49:50.000Z [INFO] agent.router: Initializing LAN area manager +49:50.001 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7a648ee6-6817-2ce3-737a-96134d730ddf.dc1 area=wan +49:50.001 [INFO] TestAgent.server: Adding LAN server: server="Node-7a648ee6-6817-2ce3-737a-96134d730ddf (Addr: tcp/127.0.0.1:25335) (DC: dc1)" +49:50.002 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25330 network=tcp +49:50.003 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25330 network=udp +49:50.003 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25331 network=tcp +49:50.004 [INFO] TestAgent: started state syncer +49:50.004 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25336 network=tcp +49:50.061 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:50.061 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25335 [Candidate]" term=2 +49:50.061 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:50.061 [DEBUG] TestAgent.server.raft: vote granted: from=7a648ee6-6817-2ce3-737a-96134d730ddf term=2 tally=1 +49:50.061 [INFO] TestAgent.server.raft: election won: tally=1 +49:50.061 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25335 [Leader]" +49:50.061 [INFO] TestAgent.server: cluster leadership acquired +49:50.061 [INFO] TestAgent.server: New leader elected: payload=Node-7a648ee6-6817-2ce3-737a-96134d730ddf +49:50.061 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25335 +49:50.062 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:50.078 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:50.078 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:50.078 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:50.078 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:50.078 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:50.078 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7a648ee6-6817-2ce3-737a-96134d730ddf +49:50.078 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7a648ee6-6817-2ce3-737a-96134d730ddf +49:50.079 [INFO] TestAgent.server: federation state anti-entropy synced +49:50.111 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:50.111 [INFO] TestAgent: Synced node info +49:50.111 [DEBUG] TestAgent: Node info in sync +49:50.310 [INFO] TestAgent: Requesting shutdown +49:50.310 [INFO] TestAgent.server: shutting down server +49:50.310 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:50.310 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:50.310 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:50.310 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:50.310 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:50.310 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:50.310 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:50.310 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:50.310 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:50.310 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:50.310 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:50.310Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:50.310Z [INFO] agent.router.manager: shutting down +49:50.310 [INFO] TestAgent: consul server down +49:50.310 [INFO] TestAgent: shutdown complete +49:50.310 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25330 network=tcp +49:50.310 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25330 network=udp +49:50.310 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25331 network=tcp +49:50.811 [INFO] TestAgent: Waiting for endpoints to shut down +49:50.811 [INFO] TestAgent: Endpoints down +--- PASS: TestConnectExpose_invalidFlags (0.87s) --- PASS: TestConnectExpose_invalidFlags/missing_service (0.00s) --- PASS: TestConnectExpose_invalidFlags/missing_gateway (0.00s) --- PASS: TestConnectExpose_invalidFlags/missing_port (0.00s) -2021-10-31T23:16:44.152Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:44.226Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:44.229 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b9eac50a-ea32-2038-b5c1-4e5b5fd2c2be Address:127.0.0.1:18181}]" -16:44.229 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18181 [Follower]" leader= -16:44.251 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b9eac50a-ea32-2038-b5c1-4e5b5fd2c2be.dc1 127.0.0.1 -16:44.252 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b9eac50a-ea32-2038-b5c1-4e5b5fd2c2be 127.0.0.1 -2021-10-31T23:16:44.252Z [INFO] agent.router: Initializing LAN area manager -16:44.274 [INFO] TestAgent.server: Adding LAN server: server="Node-b9eac50a-ea32-2038-b5c1-4e5b5fd2c2be (Addr: tcp/127.0.0.1:18181) (DC: dc1)" -16:44.274 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b9eac50a-ea32-2038-b5c1-4e5b5fd2c2be.dc1 area=wan -16:44.274 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18176 network=udp -16:44.275 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18176 network=tcp -16:44.275 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18177 network=tcp -16:44.276 [INFO] TestAgent: started state syncer -16:44.278 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18182 network=tcp -16:44.289 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:44.289 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18181 [Candidate]" term=2 -16:44.290 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:44.290 [DEBUG] TestAgent.server.raft: vote granted: from=b9eac50a-ea32-2038-b5c1-4e5b5fd2c2be term=2 tally=1 -16:44.290 [INFO] TestAgent.server.raft: election won: tally=1 -16:44.290 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18181 [Leader]" -16:44.290 [INFO] TestAgent.server: cluster leadership acquired -16:44.290 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18181 -16:44.291 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:44.293 [INFO] TestAgent.server: New leader elected: payload=Node-b9eac50a-ea32-2038-b5c1-4e5b5fd2c2be -16:44.351 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:44.351 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:44.351 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:44.351 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:44.351 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:44.351 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b9eac50a-ea32-2038-b5c1-4e5b5fd2c2be -16:44.351 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b9eac50a-ea32-2038-b5c1-4e5b5fd2c2be -16:44.352 [INFO] TestAgent.server: federation state anti-entropy synced -16:44.578 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:52332 latency=10.580498ms -16:44.584 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:52332 latency=1.327672ms -16:44.595 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config from=127.0.0.1:52332 latency=10.126579ms -16:44.611 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:52334 latency=1.18142ms -16:44.616 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=14 from=127.0.0.1:52334 latency=1.506343ms -16:44.616 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:52334 latency=371.276µs -16:44.618 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:52334 latency=907.886µs -16:44.626 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:52332 latency=349.701µs -16:44.638 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:52338 latency=513.845µs -16:44.641 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=15 from=127.0.0.1:52338 latency=1.741456ms -16:44.646 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:52338 latency=510.993µs -16:44.648 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:52338 latency=750.095µs -16:44.658 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:52332 latency=288.841µs -16:44.660 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:52344 latency=228.46µs -16:44.684 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:44.690 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=17 from=127.0.0.1:52344 latency=7.178829ms -16:44.690 [INFO] TestAgent: Synced node info -16:44.690 [DEBUG] TestAgent: Node info in sync -16:44.691 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:52344 latency=301.714µs -16:44.693 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:52344 latency=909.956µs -16:44.694 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:52332 latency=637.934µs -16:44.698 [INFO] TestAgent: Requesting shutdown -16:44.698 [INFO] TestAgent.server: shutting down server -16:44.698 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:44.698 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:44.698 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:44.698 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:44.698 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:44.698 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:44.698 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:44.698 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:44.698 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:44.698 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:44.698 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:44.698Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:44.699Z [INFO] agent.router.manager: shutting down -16:44.699 [INFO] TestAgent: consul server down -16:44.699 [INFO] TestAgent: shutdown complete -16:44.699 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18176 network=tcp -16:44.699 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18176 network=udp -16:44.699 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18177 network=tcp -16:45.201 [INFO] TestAgent: Waiting for endpoints to shut down -16:45.201 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectExpose_existingConfig (1.25s) -2021-10-31T23:16:44.153Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:44.221Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:44.258 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:88c913f8-8ff5-0ff0-d608-c05ce92249c6 Address:127.0.0.1:18174}]" -16:44.259 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-88c913f8-8ff5-0ff0-d608-c05ce92249c6.dc1 127.0.0.1 -16:44.260 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-88c913f8-8ff5-0ff0-d608-c05ce92249c6 127.0.0.1 -2021-10-31T23:16:44.260Z [INFO] agent.router: Initializing LAN area manager -16:44.263 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18174 [Follower]" leader= -16:44.274 [INFO] TestAgent.server: Adding LAN server: server="Node-88c913f8-8ff5-0ff0-d608-c05ce92249c6 (Addr: tcp/127.0.0.1:18174) (DC: dc1)" -16:44.274 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-88c913f8-8ff5-0ff0-d608-c05ce92249c6.dc1 area=wan -16:44.275 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=udp -16:44.276 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=tcp -16:44.276 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18170 network=tcp -16:44.276 [INFO] TestAgent: started state syncer -16:44.277 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18175 network=tcp -16:44.333 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:44.334 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18174 [Candidate]" term=2 -16:44.334 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:44.334 [DEBUG] TestAgent.server.raft: vote granted: from=88c913f8-8ff5-0ff0-d608-c05ce92249c6 term=2 tally=1 -16:44.334 [INFO] TestAgent.server.raft: election won: tally=1 -16:44.334 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18174 [Leader]" -16:44.334 [INFO] TestAgent.server: cluster leadership acquired -16:44.334 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18174 -16:44.335 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:44.335 [INFO] TestAgent.server: New leader elected: payload=Node-88c913f8-8ff5-0ff0-d608-c05ce92249c6 -16:44.347 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:44.347 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:44.347 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:44.347 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:44.347 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:44.347 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-88c913f8-8ff5-0ff0-d608-c05ce92249c6 -16:44.347 [INFO] TestAgent.server: member joined, marking health alive: member=Node-88c913f8-8ff5-0ff0-d608-c05ce92249c6 -16:44.350 [INFO] TestAgent.server: federation state anti-entropy synced -16:44.558 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:44.558 [INFO] TestAgent: Synced node info -16:44.558 [DEBUG] TestAgent: Node info in sync -16:44.627 [ERROR] TestAgent.http: Request error: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:46374 error="Config entry not found for "ingress-gateway" / "ingress"" -16:44.627 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:46374 latency=403.543µs -16:44.629 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=0 from=127.0.0.1:46374 latency=1.165905ms -16:44.638 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:46374 latency=445.694µs -16:44.639 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:46374 latency=631.321µs -16:44.641 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:46378 latency=473.7µs -16:44.648 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:46378 latency=1.876706ms -16:44.658 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:46380 latency=344.101µs -16:44.660 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/config?cas=13 from=127.0.0.1:46380 latency=1.112815ms -16:44.682 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:46380 latency=22.25763ms -16:44.690 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:46378 latency=562.445µs -16:44.694 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:46378 latency=431.32µs -16:44.721 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:46384 latency=269.436µs -16:44.721 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/config/ingress-gateway/ingress from=127.0.0.1:46378 latency=123.262µs -16:44.722 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:46378 latency=177.364µs -16:44.722 [INFO] TestAgent: Requesting shutdown -16:44.722 [INFO] TestAgent.server: shutting down server -16:44.722 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:44.722 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:44.722 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:44.722 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:44.722 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:44.722 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:44.722 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:44.722 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:44.722 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:44.722 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:44.722 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:44.723Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:44.723Z [INFO] agent.router.manager: shutting down -16:44.723 [INFO] TestAgent: consul server down -16:44.723 [INFO] TestAgent: shutdown complete -16:44.723 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=tcp -16:44.723 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=udp -16:44.723 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18170 network=tcp -16:45.225 [INFO] TestAgent: Waiting for endpoints to shut down -16:45.225 [INFO] TestAgent: Endpoints down ---- PASS: TestConnectExpose (1.28s) PASS -ok github.com/hashicorp/consul/command/connect/expose 2.177s +ok github.com/hashicorp/consul/command/connect/expose 0.932s === RUN TestFlagUpstreams_impl --- PASS: TestFlagUpstreams_impl (0.00s) === RUN TestFlagUpstreams @@ -83119,808 +83292,814 @@ === PAUSE TestRegisterMonitor_heartbeat === CONT TestCommandConfigWatcher === RUN TestCommandConfigWatcher/-service_flag_only -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestRegisterMonitor_heartbeat === CONT TestCatalogCommand_noTabs +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestRegisterMonitor_good ---- PASS: TestCatalogCommand_noTabs (0.01s) +--- PASS: TestCatalogCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:16:49.724Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:49.834Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:49.837 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:496f59e3-ff70-027e-3e22-8f7d2c498fa2 Address:127.0.0.1:11027}]" -16:49.842 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-496f59e3-ff70-027e-3e22-8f7d2c498fa2.dc1 127.0.0.1 -16:49.856 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11027 [Follower]" leader= -16:49.856 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-496f59e3-ff70-027e-3e22-8f7d2c498fa2 127.0.0.1 -2021-10-31T23:16:49.856Z [INFO] agent.router: Initializing LAN area manager -16:49.857 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=udp -16:49.857 [INFO] TestAgent.server: Adding LAN server: server="Node-496f59e3-ff70-027e-3e22-8f7d2c498fa2 (Addr: tcp/127.0.0.1:11027) (DC: dc1)" -16:49.857 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-496f59e3-ff70-027e-3e22-8f7d2c498fa2.dc1 area=wan -16:49.858 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=tcp -16:49.858 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11023 network=tcp -16:49.858 [INFO] TestAgent: started state syncer -16:49.858 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11028 network=tcp -16:49.921 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:49.921 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11027 [Candidate]" term=2 -16:49.922 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:49.922 [DEBUG] TestAgent.server.raft: vote granted: from=496f59e3-ff70-027e-3e22-8f7d2c498fa2 term=2 tally=1 -16:49.922 [INFO] TestAgent.server.raft: election won: tally=1 -16:49.922 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11027 [Leader]" -16:49.922 [INFO] TestAgent.server: cluster leadership acquired -16:49.922 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11027 -16:49.923 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:49.924 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:49.924 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:49.924 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:49.924 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:49.924 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:49.924 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-496f59e3-ff70-027e-3e22-8f7d2c498fa2 -16:49.924 [INFO] TestAgent.server: member joined, marking health alive: member=Node-496f59e3-ff70-027e-3e22-8f7d2c498fa2 -16:49.925 [INFO] TestAgent.server: New leader elected: payload=Node-496f59e3-ff70-027e-3e22-8f7d2c498fa2 -16:49.925 [INFO] TestAgent.server: federation state anti-entropy synced -16:49.977 [WARN] TestAgent: Check socket connection failed: check=service:two-sidecars-sidecar-proxy:1 error="dial tcp 127.0.0.1:21000: connect: connection refused" -16:49.978 [WARN] TestAgent: Check is now critical: check=service:two-sidecars-sidecar-proxy:1 -16:50.038 [INFO] TestAgent: Requesting shutdown -16:50.038 [INFO] TestAgent.server: shutting down server -16:50.038 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:50.038 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:50.038 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:50.038 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:50.038 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:50.038 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:50.038 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:50.038 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -16:50.038 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:50.038 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:50.038 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:50.038 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:50.039Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:50.039Z [INFO] agent.router.manager: shutting down -16:50.039 [INFO] TestAgent: consul server down -16:50.039 [INFO] TestAgent: shutdown complete -16:50.039 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=tcp -16:50.039 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=udp -16:50.039 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11023 network=tcp -16:50.541 [INFO] TestAgent: Waiting for endpoints to shut down -16:50.541 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:54.702Z [INFO] TestRegisterMonitor_heartbeat: registered Consul service: service=foo-proxy +2022-12-04T05:49:54.788Z [INFO] TestRegisterMonitor_heartbeat: stop request received, deregistering +2022-12-04T05:49:54.582Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:54.622Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:54.637 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8a460d2a-5a08-abbf-eb04-613d81d0c47e Address:127.0.0.1:27370}]" +49:54.637 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27370 [Follower]" leader= +49:54.639 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8a460d2a-5a08-abbf-eb04-613d81d0c47e.dc1 127.0.0.1 +49:54.639 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8a460d2a-5a08-abbf-eb04-613d81d0c47e 127.0.0.1 +2022-12-04T05:49:54.639Z [INFO] agent.router: Initializing LAN area manager +49:54.640 [INFO] TestAgent.server: Adding LAN server: server="Node-8a460d2a-5a08-abbf-eb04-613d81d0c47e (Addr: tcp/127.0.0.1:27370) (DC: dc1)" +49:54.640 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8a460d2a-5a08-abbf-eb04-613d81d0c47e.dc1 area=wan +49:54.640 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27365 network=udp +49:54.640 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27365 network=tcp +49:54.640 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27366 network=tcp +49:54.640 [INFO] TestAgent: started state syncer +49:54.640 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27371 network=tcp +49:54.679 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:54.679 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27370 [Candidate]" term=2 +49:54.680 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:54.680 [DEBUG] TestAgent.server.raft: vote granted: from=8a460d2a-5a08-abbf-eb04-613d81d0c47e term=2 tally=1 +49:54.680 [INFO] TestAgent.server.raft: election won: tally=1 +49:54.680 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27370 [Leader]" +49:54.680 [INFO] TestAgent.server: cluster leadership acquired +49:54.680 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27370 +49:54.680 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:54.680 [INFO] TestAgent.server: New leader elected: payload=Node-8a460d2a-5a08-abbf-eb04-613d81d0c47e +49:54.690 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:54.690 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:54.690 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:54.690 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:54.690 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:54.690 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8a460d2a-5a08-abbf-eb04-613d81d0c47e +49:54.690 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8a460d2a-5a08-abbf-eb04-613d81d0c47e +49:54.691 [INFO] TestAgent.server: federation state anti-entropy synced +49:54.699 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:37526 latency=934.407µs +49:54.700 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/foo-proxy?stale= from=127.0.0.1:37528 latency=1.538937ms +49:54.701 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +49:54.702 [INFO] TestAgent: Synced node info +49:54.702 [INFO] TestAgent: Synced service: service=foo-proxy +49:54.702 [DEBUG] TestAgent: Check in sync: check=foo-proxy-ttl +49:54.702 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:37528 latency=1.814344ms +49:54.702 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +49:54.702 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +49:54.702 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +49:54.731 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:37528 latency=6.011266ms +49:54.734 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:37528 latency=1.092289ms +49:54.735 [DEBUG] TestAgent: Check status updated: check=foo-proxy-ttl status=critical +49:54.735 [DEBUG] TestAgent: Node info in sync +49:54.735 [DEBUG] TestAgent: Service in sync: service=foo-proxy +49:54.736 [INFO] TestAgent: Synced check: check=foo-proxy-ttl +49:54.736 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/fail/foo-proxy-ttl?note= from=127.0.0.1:37528 latency=726.668µs +49:54.736 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:37528 latency=137.809µs +49:54.762 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:37528 latency=200.972µs +49:54.769 [DEBUG] TestAgent: Check status updated: check=foo-proxy-ttl status=passing +49:54.770 [DEBUG] TestAgent: Node info in sync +49:54.770 [DEBUG] TestAgent: Service in sync: service=foo-proxy +49:54.770 [INFO] TestAgent: Synced check: check=foo-proxy-ttl +49:54.770 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/pass/foo-proxy-ttl?note= from=127.0.0.1:37528 latency=647.901µs +49:54.788 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:37528 latency=268.936µs +49:54.789 [DEBUG] TestAgent: removed check: check=foo-proxy-ttl +49:54.789 [DEBUG] TestAgent: removed service: service=foo-proxy +49:54.789 [DEBUG] TestAgent: Node info in sync +49:54.789 [INFO] TestAgent: Deregistered service: service=foo-proxy +49:54.789 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/foo-proxy from=127.0.0.1:37528 latency=660.457µs +49:54.789 [INFO] TestAgent: Requesting shutdown +49:54.789 [INFO] TestAgent.server: shutting down server +49:54.789 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:54.789 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:54.789 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:54.789 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:54.789 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:54.789 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:54.789 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:54.789 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:54.789 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:54.789 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:54.789 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:54.790 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:54.790Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:54.790Z [INFO] agent.router.manager: shutting down +49:54.790 [INFO] TestAgent: consul server down +49:54.790 [INFO] TestAgent: shutdown complete +49:54.790 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27365 network=tcp +49:54.790 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27365 network=udp +49:54.790 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27366 network=tcp +49:55.290 [INFO] TestAgent: Waiting for endpoints to shut down +49:55.290 [INFO] TestAgent: Endpoints down +--- PASS: TestRegisterMonitor_heartbeat (0.75s) +2022-12-04T05:49:54.588Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:54.625Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:54.627 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:72b332c6-ac54-26d8-4689-1a91d48ad6b3 Address:127.0.0.1:27363}]" +49:54.627 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-72b332c6-ac54-26d8-4689-1a91d48ad6b3.dc1 127.0.0.1 +49:54.627 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= +49:54.628 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-72b332c6-ac54-26d8-4689-1a91d48ad6b3 127.0.0.1 +2022-12-04T05:49:54.628Z [INFO] agent.router: Initializing LAN area manager +49:54.628 [INFO] TestAgent.server: Adding LAN server: server="Node-72b332c6-ac54-26d8-4689-1a91d48ad6b3 (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +49:54.628 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-72b332c6-ac54-26d8-4689-1a91d48ad6b3.dc1 area=wan +49:54.628 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp +49:54.628 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp +49:54.638 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp +49:54.638 [INFO] TestAgent: started state syncer +49:54.638 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp +49:54.681 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:54.682 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 +49:54.682 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:54.682 [DEBUG] TestAgent.server.raft: vote granted: from=72b332c6-ac54-26d8-4689-1a91d48ad6b3 term=2 tally=1 +49:54.682 [INFO] TestAgent.server.raft: election won: tally=1 +49:54.682 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" +49:54.682 [INFO] TestAgent.server: cluster leadership acquired +49:54.682 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27363 +49:54.682 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:54.682 [INFO] TestAgent.server: New leader elected: payload=Node-72b332c6-ac54-26d8-4689-1a91d48ad6b3 +49:54.688 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:54.688 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:54.688 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:54.688 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:54.688 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:54.689 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-72b332c6-ac54-26d8-4689-1a91d48ad6b3 +49:54.689 [INFO] TestAgent.server: member joined, marking health alive: member=Node-72b332c6-ac54-26d8-4689-1a91d48ad6b3 +49:54.690 [INFO] TestAgent.server: federation state anti-entropy synced +49:54.837 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:54.838 [INFO] TestAgent: Synced node info +49:54.838 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy +49:54.838 [INFO] TestAgent: Synced service: service=two-sidecars +49:54.839 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy +49:54.839 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars +49:54.839 [INFO] TestAgent: Synced service: service=no-sidecar +49:54.839 [INFO] TestAgent: Synced service: service=one-sidecar +49:54.839 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 +49:54.839 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 +49:54.839 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 +49:54.839 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 +49:54.932 [INFO] TestAgent: Requesting shutdown +49:54.932 [INFO] TestAgent.server: shutting down server +49:54.932 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:54.932 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:54.932 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:54.932 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:54.932 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:54.932 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:54.932 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:54.932 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:54.932 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:54.932 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:54.932 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:54.932Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:54.932Z [INFO] agent.router.manager: shutting down +49:54.932 [INFO] TestAgent: consul server down +49:54.932 [INFO] TestAgent: shutdown complete +49:54.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp +49:54.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp +49:54.932 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp +49:55.433 [INFO] TestAgent: Waiting for endpoints to shut down +49:55.433 [INFO] TestAgent: Endpoints down === RUN TestCommandConfigWatcher/-service_flag_with_upstreams -2021-10-31T23:16:49.972Z [INFO] TestRegisterMonitor_heartbeat: registered Consul service: service=foo-proxy -2021-10-31T23:16:50.051Z [INFO] TestRegisterMonitor_heartbeat: stop request received, deregistering -2021-10-31T23:16:49.727Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:49.829Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:49.835 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aeb77b61-5b71-e724-2ea2-75bbada77b76 Address:127.0.0.1:11041}]" -16:49.836 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aeb77b61-5b71-e724-2ea2-75bbada77b76.dc1 127.0.0.1 -16:49.855 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aeb77b61-5b71-e724-2ea2-75bbada77b76 127.0.0.1 -2021-10-31T23:16:49.855Z [INFO] agent.router: Initializing LAN area manager -16:49.855 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11036 network=udp -16:49.855 [INFO] TestAgent.server: Adding LAN server: server="Node-aeb77b61-5b71-e724-2ea2-75bbada77b76 (Addr: tcp/127.0.0.1:11041) (DC: dc1)" -16:49.855 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aeb77b61-5b71-e724-2ea2-75bbada77b76.dc1 area=wan -16:49.855 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11036 network=tcp -16:49.856 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11037 network=tcp -16:49.856 [INFO] TestAgent: started state syncer -16:49.856 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11042 network=tcp -16:49.856 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11041 [Follower]" leader= -16:49.895 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:49.895 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11041 [Candidate]" term=2 -16:49.896 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:49.896 [DEBUG] TestAgent.server.raft: vote granted: from=aeb77b61-5b71-e724-2ea2-75bbada77b76 term=2 tally=1 -16:49.896 [INFO] TestAgent.server.raft: election won: tally=1 -16:49.896 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11041 [Leader]" -16:49.896 [INFO] TestAgent.server: cluster leadership acquired -16:49.896 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11041 -16:49.897 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:49.897 [INFO] TestAgent.server: New leader elected: payload=Node-aeb77b61-5b71-e724-2ea2-75bbada77b76 -16:49.916 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:49.916 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:49.916 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:49.916 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:49.916 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:49.916 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aeb77b61-5b71-e724-2ea2-75bbada77b76 -16:49.916 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aeb77b61-5b71-e724-2ea2-75bbada77b76 -16:49.917 [INFO] TestAgent.server: federation state anti-entropy synced -16:49.947 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/foo-proxy?stale= from=127.0.0.1:47666 latency=824.015µs -16:49.947 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:47664 latency=1.285663ms -16:49.965 [INFO] TestAgent: Synced node info -16:49.966 [INFO] TestAgent: Synced service: service=foo-proxy -16:49.966 [DEBUG] TestAgent: Check in sync: check=foo-proxy-ttl -16:49.966 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:47666 latency=11.708762ms -16:49.966 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:49.967 [INFO] TestAgent: Synced node info -16:49.968 [INFO] TestAgent: Synced service: service=foo-proxy -16:49.968 [DEBUG] TestAgent: Check in sync: check=foo-proxy-ttl -16:49.968 [DEBUG] TestAgent: Node info in sync -16:49.968 [DEBUG] TestAgent: Service in sync: service=foo-proxy -16:49.968 [DEBUG] TestAgent: Check in sync: check=foo-proxy-ttl -16:49.968 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -16:49.968 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -16:49.968 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -16:49.968 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -16:49.985 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:47666 latency=1.323518ms -16:49.986 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:47666 latency=327.323µs -16:49.989 [DEBUG] TestAgent: Check status updated: check=foo-proxy-ttl status=critical -16:49.990 [DEBUG] TestAgent: Node info in sync -16:49.990 [DEBUG] TestAgent: Service in sync: service=foo-proxy -16:49.990 [INFO] TestAgent: Synced check: check=foo-proxy-ttl -16:49.990 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/fail/foo-proxy-ttl?note= from=127.0.0.1:47666 latency=801.583µs -16:49.994 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:47666 latency=351.523µs -16:50.022 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:47666 latency=259.002µs -16:50.040 [DEBUG] TestAgent: Check status updated: check=foo-proxy-ttl status=passing -16:50.041 [DEBUG] TestAgent: Node info in sync -16:50.041 [DEBUG] TestAgent: Service in sync: service=foo-proxy -16:50.041 [INFO] TestAgent: Synced check: check=foo-proxy-ttl -16:50.041 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/check/pass/foo-proxy-ttl?note= from=127.0.0.1:47666 latency=1.003388ms -16:50.050 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:47666 latency=257.166µs -16:50.051 [DEBUG] TestAgent: removed check: check=foo-proxy-ttl -16:50.051 [DEBUG] TestAgent: removed service: service=foo-proxy -16:50.051 [DEBUG] TestAgent: Node info in sync -16:50.052 [INFO] TestAgent: Deregistered service: service=foo-proxy -16:50.052 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/foo-proxy from=127.0.0.1:47666 latency=847.766µs -16:50.052 [INFO] TestAgent: Requesting shutdown -16:50.052 [INFO] TestAgent.server: shutting down server -16:50.052 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:50.052 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:50.052 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:50.052 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:50.052 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:50.052 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:50.052 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:50.052 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:50.052 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:50.052 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:50.052 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:50.052Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:50.052Z [INFO] agent.router.manager: shutting down -16:50.052 [INFO] TestAgent: consul server down -16:50.052 [INFO] TestAgent: shutdown complete -16:50.052 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11036 network=tcp -16:50.052 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11036 network=udp -16:50.053 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11037 network=tcp -16:50.553 [INFO] TestAgent: Waiting for endpoints to shut down -16:50.553 [INFO] TestAgent: Endpoints down ---- PASS: TestRegisterMonitor_heartbeat (0.99s) -2021-10-31T23:16:50.094Z [INFO] TestRegisterMonitor_good: registered Consul service: service=foo-proxy -2021-10-31T23:16:50.121Z [INFO] TestRegisterMonitor_good: stop request received, deregistering -2021-10-31T23:16:49.722Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:49.796Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:49.803 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:70d5447a-c1bb-6605-1087-6c44869e63ce Address:127.0.0.1:11034}]" -16:49.804 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-70d5447a-c1bb-6605-1087-6c44869e63ce.dc1 127.0.0.1 -16:49.804 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-70d5447a-c1bb-6605-1087-6c44869e63ce 127.0.0.1 -2021-10-31T23:16:49.804Z [INFO] agent.router: Initializing LAN area manager -16:49.804 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11029 network=udp -16:49.804 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11034 [Follower]" leader= -16:49.805 [INFO] TestAgent.server: Adding LAN server: server="Node-70d5447a-c1bb-6605-1087-6c44869e63ce (Addr: tcp/127.0.0.1:11034) (DC: dc1)" -16:49.805 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-70d5447a-c1bb-6605-1087-6c44869e63ce.dc1 area=wan -16:49.805 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11029 network=tcp -16:49.827 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11030 network=tcp -16:49.827 [INFO] TestAgent: started state syncer -16:49.827 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11035 network=tcp -16:49.873 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:49.874 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11034 [Candidate]" term=2 -16:49.874 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:49.874 [DEBUG] TestAgent.server.raft: vote granted: from=70d5447a-c1bb-6605-1087-6c44869e63ce term=2 tally=1 -16:49.874 [INFO] TestAgent.server.raft: election won: tally=1 -16:49.874 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11034 [Leader]" -16:49.874 [INFO] TestAgent.server: cluster leadership acquired -16:49.874 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11034 -16:49.875 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:49.877 [INFO] TestAgent.server: New leader elected: payload=Node-70d5447a-c1bb-6605-1087-6c44869e63ce -16:49.909 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:49.909 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:49.909 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:49.909 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:49.909 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:49.909 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-70d5447a-c1bb-6605-1087-6c44869e63ce -16:49.909 [INFO] TestAgent.server: member joined, marking health alive: member=Node-70d5447a-c1bb-6605-1087-6c44869e63ce -16:49.917 [INFO] TestAgent.server: federation state anti-entropy synced -16:50.091 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/foo-proxy?stale= from=127.0.0.1:53994 latency=352.875µs -16:50.092 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:53992 latency=79.454µs -16:50.092 [INFO] TestAgent: Synced node info -16:50.093 [INFO] TestAgent: Synced service: service=foo-proxy -16:50.093 [DEBUG] TestAgent: Check in sync: check=foo-proxy-ttl -16:50.093 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:53994 latency=1.211957ms -16:50.094 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -16:50.094 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -16:50.094 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -16:50.094 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -16:50.120 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:53994 latency=2.181771ms -16:50.121 [DEBUG] TestAgent: removed check: check=foo-proxy-ttl -16:50.121 [DEBUG] TestAgent: removed service: service=foo-proxy -16:50.121 [DEBUG] TestAgent: Node info in sync -16:50.122 [INFO] TestAgent: Deregistered service: service=foo-proxy -16:50.122 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/foo-proxy from=127.0.0.1:53994 latency=928.852µs -16:50.137 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:53994 latency=7.827258ms -16:50.138 [INFO] TestAgent: Requesting shutdown -16:50.138 [INFO] TestAgent.server: shutting down server -16:50.138 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:50.138 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:50.138 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:50.138 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:50.138 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:50.138 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:50.138 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:50.138 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -16:50.139 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:50.139 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:50.139 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:50.139 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:50.139Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:50.139Z [INFO] agent.router.manager: shutting down -16:50.139 [INFO] TestAgent: consul server down -16:50.139 [INFO] TestAgent: shutdown complete -16:50.139 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11029 network=tcp -16:50.139 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11029 network=udp -16:50.139 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11030 network=tcp -16:50.641 [INFO] TestAgent: Waiting for endpoints to shut down -16:50.641 [INFO] TestAgent: Endpoints down ---- PASS: TestRegisterMonitor_good (1.07s) -2021-10-31T23:16:50.577Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:50.623Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:50.625 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f68813d9-e15d-e90a-1801-f580cb7ed3ba Address:127.0.0.1:11048}]" -16:50.625 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11048 [Follower]" leader= -16:50.635 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f68813d9-e15d-e90a-1801-f580cb7ed3ba.dc1 127.0.0.1 -16:50.635 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f68813d9-e15d-e90a-1801-f580cb7ed3ba 127.0.0.1 -2021-10-31T23:16:50.635Z [INFO] agent.router: Initializing LAN area manager -16:50.636 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11043 network=udp -16:50.636 [INFO] TestAgent.server: Adding LAN server: server="Node-f68813d9-e15d-e90a-1801-f580cb7ed3ba (Addr: tcp/127.0.0.1:11048) (DC: dc1)" -16:50.636 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f68813d9-e15d-e90a-1801-f580cb7ed3ba.dc1 area=wan -16:50.636 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11043 network=tcp -16:50.637 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11044 network=tcp -16:50.637 [INFO] TestAgent: started state syncer -16:50.637 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11049 network=tcp -16:50.668 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:50.668 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11048 [Candidate]" term=2 -16:50.669 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:50.669 [DEBUG] TestAgent.server.raft: vote granted: from=f68813d9-e15d-e90a-1801-f580cb7ed3ba term=2 tally=1 -16:50.669 [INFO] TestAgent.server.raft: election won: tally=1 -16:50.669 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11048 [Leader]" -16:50.669 [INFO] TestAgent.server: cluster leadership acquired -16:50.669 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11048 -16:50.669 [INFO] TestAgent.server: New leader elected: payload=Node-f68813d9-e15d-e90a-1801-f580cb7ed3ba -16:50.670 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:50.671 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:50.671 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:50.671 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:50.671 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:50.671 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:50.671 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f68813d9-e15d-e90a-1801-f580cb7ed3ba -16:50.671 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f68813d9-e15d-e90a-1801-f580cb7ed3ba -16:50.672 [INFO] TestAgent.server: federation state anti-entropy synced -16:50.737 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:50.738 [INFO] TestAgent: Synced node info -16:50.738 [INFO] TestAgent: Synced service: service=two-sidecars -16:50.739 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy -16:50.739 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars -16:50.739 [INFO] TestAgent: Synced service: service=no-sidecar -16:50.740 [INFO] TestAgent: Synced service: service=one-sidecar -16:50.740 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy -16:50.740 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 -16:50.740 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 -16:50.740 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 -16:50.740 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 -16:50.994 [INFO] TestAgent: Requesting shutdown -16:50.994 [INFO] TestAgent.server: shutting down server -16:50.994 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:50.994 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:50.994 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:50.994 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:50.994 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:50.995 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:50.995 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:50.995 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:50.995 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:50.995 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:50.995 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:50.995Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:50.995Z [INFO] agent.router.manager: shutting down -16:50.995 [INFO] TestAgent: consul server down -16:50.995 [INFO] TestAgent: shutdown complete -16:50.995 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11043 network=tcp -16:50.995 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11043 network=udp -16:50.995 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11044 network=tcp -16:51.497 [INFO] TestAgent: Waiting for endpoints to shut down -16:51.498 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:55.038Z [INFO] TestRegisterMonitor_good: registered Consul service: service=foo-proxy +2022-12-04T05:49:55.061Z [INFO] TestRegisterMonitor_good: stop request received, deregistering +2022-12-04T05:49:54.601Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:54.623Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:54.625 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bc355d57-be66-d044-c4c7-7dde2cb7ea91 Address:127.0.0.1:27377}]" +49:54.625 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bc355d57-be66-d044-c4c7-7dde2cb7ea91.dc1 127.0.0.1 +49:54.626 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bc355d57-be66-d044-c4c7-7dde2cb7ea91 127.0.0.1 +2022-12-04T05:49:54.626Z [INFO] agent.router: Initializing LAN area manager +49:54.626 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27372 network=udp +49:54.626 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27377 [Follower]" leader= +49:54.627 [INFO] TestAgent.server: Adding LAN server: server="Node-bc355d57-be66-d044-c4c7-7dde2cb7ea91 (Addr: tcp/127.0.0.1:27377) (DC: dc1)" +49:54.627 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bc355d57-be66-d044-c4c7-7dde2cb7ea91.dc1 area=wan +49:54.627 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27372 network=tcp +49:54.627 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27373 network=tcp +49:54.627 [INFO] TestAgent: started state syncer +49:54.627 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27378 network=tcp +49:54.670 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:54.670 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27377 [Candidate]" term=2 +49:54.670 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:54.670 [DEBUG] TestAgent.server.raft: vote granted: from=bc355d57-be66-d044-c4c7-7dde2cb7ea91 term=2 tally=1 +49:54.670 [INFO] TestAgent.server.raft: election won: tally=1 +49:54.670 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27377 [Leader]" +49:54.670 [INFO] TestAgent.server: cluster leadership acquired +49:54.671 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27377 +49:54.671 [INFO] TestAgent.server: New leader elected: payload=Node-bc355d57-be66-d044-c4c7-7dde2cb7ea91 +49:54.671 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:54.688 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:54.688 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:54.688 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:54.688 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:54.688 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:54.688 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bc355d57-be66-d044-c4c7-7dde2cb7ea91 +49:54.688 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bc355d57-be66-d044-c4c7-7dde2cb7ea91 +49:54.688 [INFO] TestAgent.server: federation state anti-entropy synced +49:55.000 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:55.000 [INFO] TestAgent: Synced node info +49:55.000 [DEBUG] TestAgent: Node info in sync +49:55.035 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:43592 latency=176.193µs +49:55.035 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/service/foo-proxy?stale= from=127.0.0.1:43594 latency=369.465µs +49:55.036 [DEBUG] TestAgent: Node info in sync +49:55.036 [INFO] TestAgent: Synced service: service=foo-proxy +49:55.036 [DEBUG] TestAgent: Check in sync: check=foo-proxy-ttl +49:55.036 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:43594 latency=1.249003ms +49:55.037 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +49:55.037 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +49:55.037 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +49:55.037 [ERROR] TestAgent.proxycfg: watch error: service_id=foo-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +49:55.061 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:43594 latency=318.866µs +49:55.061 [DEBUG] TestAgent: removed check: check=foo-proxy-ttl +49:55.061 [DEBUG] TestAgent: removed service: service=foo-proxy +49:55.061 [DEBUG] TestAgent: Node info in sync +49:55.062 [INFO] TestAgent: Deregistered service: service=foo-proxy +49:55.062 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/foo-proxy from=127.0.0.1:43594 latency=700.57µs +49:55.063 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:43594 latency=204.303µs +49:55.063 [INFO] TestAgent: Requesting shutdown +49:55.063 [INFO] TestAgent.server: shutting down server +49:55.063 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:55.063 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:55.063 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:55.063 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:55.063 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:55.063 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:55.063 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:55.063 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:55.063 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:55.063 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:55.063 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:55.063Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:55.063Z [INFO] agent.router.manager: shutting down +49:55.063 [INFO] TestAgent: consul server down +49:55.063 [INFO] TestAgent: shutdown complete +49:55.063 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27372 network=tcp +49:55.063 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27372 network=udp +49:55.063 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27373 network=tcp +49:55.563 [INFO] TestAgent: Waiting for endpoints to shut down +49:55.563 [INFO] TestAgent: Endpoints down +--- PASS: TestRegisterMonitor_good (1.02s) +2022-12-04T05:49:55.447Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:55.467Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:55.469 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:de36460b-4ad2-735d-4dcc-8ed3cc5f3b75 Address:127.0.0.1:27384}]" +49:55.469 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27384 [Follower]" leader= +49:55.470 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-de36460b-4ad2-735d-4dcc-8ed3cc5f3b75.dc1 127.0.0.1 +49:55.476 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-de36460b-4ad2-735d-4dcc-8ed3cc5f3b75 127.0.0.1 +2022-12-04T05:49:55.476Z [INFO] agent.router: Initializing LAN area manager +49:55.477 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27379 network=udp +49:55.477 [INFO] TestAgent.server: Adding LAN server: server="Node-de36460b-4ad2-735d-4dcc-8ed3cc5f3b75 (Addr: tcp/127.0.0.1:27384) (DC: dc1)" +49:55.477 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-de36460b-4ad2-735d-4dcc-8ed3cc5f3b75.dc1 area=wan +49:55.477 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27379 network=tcp +49:55.478 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27380 network=tcp +49:55.478 [INFO] TestAgent: started state syncer +49:55.478 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27385 network=tcp +49:55.510 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:55.510 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27384 [Candidate]" term=2 +49:55.510 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:55.510 [DEBUG] TestAgent.server.raft: vote granted: from=de36460b-4ad2-735d-4dcc-8ed3cc5f3b75 term=2 tally=1 +49:55.510 [INFO] TestAgent.server.raft: election won: tally=1 +49:55.510 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27384 [Leader]" +49:55.510 [INFO] TestAgent.server: cluster leadership acquired +49:55.510 [INFO] TestAgent.server: New leader elected: payload=Node-de36460b-4ad2-735d-4dcc-8ed3cc5f3b75 +49:55.510 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27384 +49:55.511 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:55.512 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:55.512 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:55.512 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:55.512 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:55.512 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:55.513 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-de36460b-4ad2-735d-4dcc-8ed3cc5f3b75 +49:55.513 [INFO] TestAgent.server: member joined, marking health alive: member=Node-de36460b-4ad2-735d-4dcc-8ed3cc5f3b75 +49:55.513 [INFO] TestAgent.server: federation state anti-entropy synced +49:55.515 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:55.515 [INFO] TestAgent: Synced node info +49:55.515 [INFO] TestAgent: Synced service: service=no-sidecar +49:55.516 [INFO] TestAgent: Synced service: service=one-sidecar +49:55.516 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy +49:55.516 [INFO] TestAgent: Synced service: service=two-sidecars +49:55.517 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy +49:55.517 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars +49:55.517 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 +49:55.517 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 +49:55.517 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 +49:55.517 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 +49:55.517 [DEBUG] TestAgent: Node info in sync +49:55.517 [DEBUG] TestAgent: Service in sync: service=other-sidecar-for-two-sidecars +49:55.517 [DEBUG] TestAgent: Service in sync: service=no-sidecar +49:55.517 [DEBUG] TestAgent: Service in sync: service=one-sidecar +49:55.517 [DEBUG] TestAgent: Service in sync: service=one-sidecar-sidecar-proxy +49:55.517 [DEBUG] TestAgent: Service in sync: service=two-sidecars +49:55.517 [DEBUG] TestAgent: Service in sync: service=two-sidecars-sidecar-proxy +49:55.517 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 +49:55.517 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 +49:55.517 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 +49:55.517 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 +49:55.578 [INFO] TestAgent: Requesting shutdown +49:55.578 [INFO] TestAgent.server: shutting down server +49:55.578 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:55.578 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:55.578 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:55.578 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:55.578 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:55.578 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:55.578 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:55.578 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:55.578 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:55.578 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:55.578 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:55.578Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:55.578Z [INFO] agent.router.manager: shutting down +49:55.578 [INFO] TestAgent: consul server down +49:55.578 [INFO] TestAgent: shutdown complete +49:55.578 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27379 network=tcp +49:55.579 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27379 network=udp +49:55.579 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27380 network=tcp +49:56.079 [INFO] TestAgent: Waiting for endpoints to shut down +49:56.079 [INFO] TestAgent: Endpoints down === RUN TestCommandConfigWatcher/-service_flag_with_-service-addr -2021-10-31T23:16:51.565Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:51.604Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:51.614 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:59b3a93d-43a5-c835-5f3c-c29eeeec4666 Address:127.0.0.1:11055}]" -16:51.615 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-59b3a93d-43a5-c835-5f3c-c29eeeec4666.dc1 127.0.0.1 -16:51.616 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-59b3a93d-43a5-c835-5f3c-c29eeeec4666 127.0.0.1 -2021-10-31T23:16:51.616Z [INFO] agent.router: Initializing LAN area manager -16:51.616 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11050 network=udp -16:51.616 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11055 [Follower]" leader= -16:51.616 [INFO] TestAgent.server: Adding LAN server: server="Node-59b3a93d-43a5-c835-5f3c-c29eeeec4666 (Addr: tcp/127.0.0.1:11055) (DC: dc1)" -16:51.616 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-59b3a93d-43a5-c835-5f3c-c29eeeec4666.dc1 area=wan -16:51.617 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11050 network=tcp -16:51.617 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11051 network=tcp -16:51.617 [INFO] TestAgent: started state syncer -16:51.617 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11056 network=tcp -16:51.677 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:51.677 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11055 [Candidate]" term=2 -16:51.678 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:51.678 [DEBUG] TestAgent.server.raft: vote granted: from=59b3a93d-43a5-c835-5f3c-c29eeeec4666 term=2 tally=1 -16:51.678 [INFO] TestAgent.server.raft: election won: tally=1 -16:51.678 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11055 [Leader]" -16:51.678 [INFO] TestAgent.server: cluster leadership acquired -16:51.678 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11055 -16:51.679 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:51.680 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:51.680 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:51.680 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:51.680 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:51.680 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:51.680 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-59b3a93d-43a5-c835-5f3c-c29eeeec4666 -16:51.680 [INFO] TestAgent.server: member joined, marking health alive: member=Node-59b3a93d-43a5-c835-5f3c-c29eeeec4666 -16:51.681 [INFO] TestAgent.server: New leader elected: payload=Node-59b3a93d-43a5-c835-5f3c-c29eeeec4666 -16:51.681 [INFO] TestAgent.server: federation state anti-entropy synced -16:51.838 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:51.838 [INFO] TestAgent: Synced node info -16:51.838 [INFO] TestAgent: Synced service: service=two-sidecars -16:51.839 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy -16:51.839 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars -16:51.839 [INFO] TestAgent: Synced service: service=no-sidecar -16:51.840 [INFO] TestAgent: Synced service: service=one-sidecar -16:51.840 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy -16:51.840 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 -16:51.840 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 -16:51.840 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 -16:51.840 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 -16:51.840 [DEBUG] TestAgent: Node info in sync -16:51.840 [DEBUG] TestAgent: Service in sync: service=two-sidecars -16:51.840 [DEBUG] TestAgent: Service in sync: service=two-sidecars-sidecar-proxy -16:51.840 [DEBUG] TestAgent: Service in sync: service=other-sidecar-for-two-sidecars -16:51.840 [DEBUG] TestAgent: Service in sync: service=no-sidecar -16:51.840 [DEBUG] TestAgent: Service in sync: service=one-sidecar -16:51.840 [DEBUG] TestAgent: Service in sync: service=one-sidecar-sidecar-proxy -16:51.840 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 -16:51.840 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 -16:51.840 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 -16:51.840 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 -16:51.854 [INFO] TestAgent: Requesting shutdown -16:51.854 [INFO] TestAgent.server: shutting down server -16:51.854 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:51.854 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:51.854 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:51.854 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:51.855 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:51.855 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:51.855 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:51.855 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:51.855 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:51.855 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:51.855 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:51.855Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:51.855Z [INFO] agent.router.manager: shutting down -16:51.855 [INFO] TestAgent: consul server down -16:51.855 [INFO] TestAgent: shutdown complete -16:51.855 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11050 network=tcp -16:51.855 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11050 network=udp -16:51.855 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11051 network=tcp -16:52.357 [INFO] TestAgent: Waiting for endpoints to shut down -16:52.357 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:56.092Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:56.105Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:56.107 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:22565505-296f-da32-19c7-3a3d1a516fe7 Address:127.0.0.1:27391}]" +49:56.107 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27391 [Follower]" leader= +49:56.107 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-22565505-296f-da32-19c7-3a3d1a516fe7.dc1 127.0.0.1 +49:56.109 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-22565505-296f-da32-19c7-3a3d1a516fe7 127.0.0.1 +2022-12-04T05:49:56.109Z [INFO] agent.router: Initializing LAN area manager +49:56.109 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-22565505-296f-da32-19c7-3a3d1a516fe7.dc1 area=wan +49:56.110 [INFO] TestAgent.server: Adding LAN server: server="Node-22565505-296f-da32-19c7-3a3d1a516fe7 (Addr: tcp/127.0.0.1:27391) (DC: dc1)" +49:56.110 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27386 network=udp +49:56.110 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27386 network=tcp +49:56.110 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27387 network=tcp +49:56.110 [INFO] TestAgent: started state syncer +49:56.111 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27392 network=tcp +49:56.151 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:56.151 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27391 [Candidate]" term=2 +49:56.152 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:56.152 [DEBUG] TestAgent.server.raft: vote granted: from=22565505-296f-da32-19c7-3a3d1a516fe7 term=2 tally=1 +49:56.152 [INFO] TestAgent.server.raft: election won: tally=1 +49:56.152 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27391 [Leader]" +49:56.152 [INFO] TestAgent.server: cluster leadership acquired +49:56.152 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27391 +49:56.152 [INFO] TestAgent.server: New leader elected: payload=Node-22565505-296f-da32-19c7-3a3d1a516fe7 +49:56.153 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:56.154 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:56.154 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:56.154 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:56.154 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:56.154 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:56.154 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-22565505-296f-da32-19c7-3a3d1a516fe7 +49:56.154 [INFO] TestAgent.server: member joined, marking health alive: member=Node-22565505-296f-da32-19c7-3a3d1a516fe7 +49:56.155 [INFO] TestAgent.server: federation state anti-entropy synced +49:56.207 [INFO] TestAgent: Requesting shutdown +49:56.207 [INFO] TestAgent.server: shutting down server +49:56.207 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:56.207 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:56.207 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:56.207 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:56.207 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:56.207 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:56.207 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:56.207 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:56.207 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:56.207 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:56.207 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:56.207 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:56.207Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:56.207Z [INFO] agent.router.manager: shutting down +49:56.207 [INFO] TestAgent: consul server down +49:56.207 [INFO] TestAgent: shutdown complete +49:56.207 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27386 network=tcp +49:56.207 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27386 network=udp +49:56.207 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27387 network=tcp +49:56.708 [INFO] TestAgent: Waiting for endpoints to shut down +49:56.708 [INFO] TestAgent: Endpoints down === RUN TestCommandConfigWatcher/-service,_-service-addr,_-listen -2021-10-31T23:16:52.393Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:52.464Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:52.475 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b7e33eb5-54af-85ae-2e67-7dfef8fa2e06 Address:127.0.0.1:11062}]" -16:52.475 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b7e33eb5-54af-85ae-2e67-7dfef8fa2e06.dc1 127.0.0.1 -16:52.476 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b7e33eb5-54af-85ae-2e67-7dfef8fa2e06 127.0.0.1 -2021-10-31T23:16:52.476Z [INFO] agent.router: Initializing LAN area manager -16:52.476 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11057 network=udp -16:52.477 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11062 [Follower]" leader= -16:52.477 [INFO] TestAgent.server: Adding LAN server: server="Node-b7e33eb5-54af-85ae-2e67-7dfef8fa2e06 (Addr: tcp/127.0.0.1:11062) (DC: dc1)" -16:52.477 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b7e33eb5-54af-85ae-2e67-7dfef8fa2e06.dc1 area=wan -16:52.481 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11057 network=tcp -16:52.482 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11058 network=tcp -16:52.482 [INFO] TestAgent: started state syncer -16:52.482 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11063 network=tcp -16:52.517 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:52.518 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11062 [Candidate]" term=2 -16:52.518 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:52.518 [DEBUG] TestAgent.server.raft: vote granted: from=b7e33eb5-54af-85ae-2e67-7dfef8fa2e06 term=2 tally=1 -16:52.518 [INFO] TestAgent.server.raft: election won: tally=1 -16:52.518 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11062 [Leader]" -16:52.518 [INFO] TestAgent.server: cluster leadership acquired -16:52.518 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11062 -16:52.519 [INFO] TestAgent.server: New leader elected: payload=Node-b7e33eb5-54af-85ae-2e67-7dfef8fa2e06 -16:52.519 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:52.520 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:52.520 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:52.520 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:52.520 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:52.520 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:52.520 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b7e33eb5-54af-85ae-2e67-7dfef8fa2e06 -16:52.520 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b7e33eb5-54af-85ae-2e67-7dfef8fa2e06 -16:52.521 [INFO] TestAgent.server: federation state anti-entropy synced -16:52.649 [INFO] TestAgent: Requesting shutdown -16:52.650 [INFO] TestAgent.server: shutting down server -16:52.650 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:52.650 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:52.650 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:52.650 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:52.650 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:52.650 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:52.650 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:52.650 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -16:52.650 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:52.650 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:52.650 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:52.650 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:52.650Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:52.650Z [INFO] agent.router.manager: shutting down -16:52.650 [INFO] TestAgent: consul server down -16:52.650 [INFO] TestAgent: shutdown complete -16:52.650 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11057 network=tcp -16:52.650 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11057 network=udp -16:52.650 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11058 network=tcp -16:53.153 [INFO] TestAgent: Waiting for endpoints to shut down -16:53.153 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:56.727Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:56.741Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:56.743 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f0a1feac-0052-984d-949d-e4a78571af65 Address:127.0.0.1:27398}]" +49:56.743 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27398 [Follower]" leader= +49:56.743 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f0a1feac-0052-984d-949d-e4a78571af65.dc1 127.0.0.1 +49:56.743 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f0a1feac-0052-984d-949d-e4a78571af65 127.0.0.1 +2022-12-04T05:49:56.744Z [INFO] agent.router: Initializing LAN area manager +49:56.744 [INFO] TestAgent.server: Adding LAN server: server="Node-f0a1feac-0052-984d-949d-e4a78571af65 (Addr: tcp/127.0.0.1:27398) (DC: dc1)" +49:56.744 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f0a1feac-0052-984d-949d-e4a78571af65.dc1 area=wan +49:56.744 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27393 network=tcp +49:56.744 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27393 network=udp +49:56.745 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27394 network=tcp +49:56.745 [INFO] TestAgent: started state syncer +49:56.745 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27399 network=tcp +49:56.805 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:56.805 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27398 [Candidate]" term=2 +49:56.805 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:56.805 [DEBUG] TestAgent.server.raft: vote granted: from=f0a1feac-0052-984d-949d-e4a78571af65 term=2 tally=1 +49:56.805 [INFO] TestAgent.server.raft: election won: tally=1 +49:56.805 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27398 [Leader]" +49:56.805 [INFO] TestAgent.server: cluster leadership acquired +49:56.805 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27398 +49:56.805 [INFO] TestAgent.server: New leader elected: payload=Node-f0a1feac-0052-984d-949d-e4a78571af65 +49:56.806 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:56.807 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:56.807 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:56.807 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:56.807 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:56.807 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:56.807 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f0a1feac-0052-984d-949d-e4a78571af65 +49:56.807 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f0a1feac-0052-984d-949d-e4a78571af65 +49:56.808 [INFO] TestAgent.server: federation state anti-entropy synced +49:56.960 [INFO] TestAgent: Requesting shutdown +49:56.960 [INFO] TestAgent.server: shutting down server +49:56.960 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:56.960 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:56.960 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:56.960 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:56.960 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:56.960 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:56.960 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:56.960 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:56.960 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:56.960 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +49:56.960 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:56.960 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:49:56.960Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:56.960Z [INFO] agent.router.manager: shutting down +49:56.960 [INFO] TestAgent: consul server down +49:56.960 [INFO] TestAgent: shutdown complete +49:56.960 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27393 network=tcp +49:56.960 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27393 network=udp +49:56.960 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27394 network=tcp +49:57.460 [INFO] TestAgent: Waiting for endpoints to shut down +49:57.460 [INFO] TestAgent: Endpoints down === RUN TestCommandConfigWatcher/-sidecar-for,_no_sidecar -2021-10-31T23:16:53.179Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:53.232Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:53.242 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:adda3ccc-6b90-f298-b2a9-da91d654bba2 Address:127.0.0.1:11069}]" -16:53.243 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-adda3ccc-6b90-f298-b2a9-da91d654bba2.dc1 127.0.0.1 -16:53.244 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-adda3ccc-6b90-f298-b2a9-da91d654bba2 127.0.0.1 -2021-10-31T23:16:53.244Z [INFO] agent.router: Initializing LAN area manager -16:53.244 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11064 network=udp -16:53.244 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11069 [Follower]" leader= -16:53.244 [INFO] TestAgent.server: Adding LAN server: server="Node-adda3ccc-6b90-f298-b2a9-da91d654bba2 (Addr: tcp/127.0.0.1:11069) (DC: dc1)" -16:53.244 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-adda3ccc-6b90-f298-b2a9-da91d654bba2.dc1 area=wan -16:53.245 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11064 network=tcp -16:53.250 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11065 network=tcp -16:53.250 [INFO] TestAgent: started state syncer -16:53.250 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11070 network=tcp -16:53.305 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:53.305 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11069 [Candidate]" term=2 -16:53.306 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:53.306 [DEBUG] TestAgent.server.raft: vote granted: from=adda3ccc-6b90-f298-b2a9-da91d654bba2 term=2 tally=1 -16:53.306 [INFO] TestAgent.server.raft: election won: tally=1 -16:53.306 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11069 [Leader]" -16:53.306 [INFO] TestAgent.server: cluster leadership acquired -16:53.306 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11069 -16:53.307 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:53.308 [INFO] TestAgent.server: New leader elected: payload=Node-adda3ccc-6b90-f298-b2a9-da91d654bba2 -16:53.308 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:53.308 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:53.308 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:53.308 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:53.308 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:53.308 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-adda3ccc-6b90-f298-b2a9-da91d654bba2 -16:53.308 [INFO] TestAgent.server: member joined, marking health alive: member=Node-adda3ccc-6b90-f298-b2a9-da91d654bba2 -16:53.309 [INFO] TestAgent.server: federation state anti-entropy synced -16:53.486 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:53.486 [INFO] TestAgent: Synced node info -16:53.487 [INFO] TestAgent: Synced service: service=one-sidecar -16:53.487 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy -16:53.488 [INFO] TestAgent: Synced service: service=two-sidecars -16:53.488 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy -16:53.489 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars -16:53.489 [INFO] TestAgent: Synced service: service=no-sidecar -16:53.489 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 -16:53.489 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 -16:53.489 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 -16:53.489 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 -16:53.535 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:51836 latency=467.036µs -16:53.535 [INFO] TestAgent: Requesting shutdown -16:53.535 [INFO] TestAgent.server: shutting down server -16:53.535 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:53.535 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:53.535 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:53.535 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:53.535 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:53.535 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:53.535 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:53.536 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:53.536 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:53.536 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:53.536 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:53.536Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:53.536Z [INFO] agent.router.manager: shutting down -16:53.536 [INFO] TestAgent: consul server down -16:53.536 [INFO] TestAgent: shutdown complete -16:53.536 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11064 network=tcp -16:53.536 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11064 network=udp -16:53.536 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11065 network=tcp -16:54.037 [INFO] TestAgent: Waiting for endpoints to shut down -16:54.037 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:57.472Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:57.490Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:57.491 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6ba861d8-7ff7-eb9e-7c8d-a52d845437bc Address:127.0.0.1:27405}]" +49:57.491 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27405 [Follower]" leader= +49:57.492 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6ba861d8-7ff7-eb9e-7c8d-a52d845437bc.dc1 127.0.0.1 +49:57.493 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6ba861d8-7ff7-eb9e-7c8d-a52d845437bc 127.0.0.1 +2022-12-04T05:49:57.493Z [INFO] agent.router: Initializing LAN area manager +49:57.493 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6ba861d8-7ff7-eb9e-7c8d-a52d845437bc.dc1 area=wan +49:57.493 [INFO] TestAgent.server: Adding LAN server: server="Node-6ba861d8-7ff7-eb9e-7c8d-a52d845437bc (Addr: tcp/127.0.0.1:27405) (DC: dc1)" +49:57.493 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27400 network=udp +49:57.493 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27400 network=tcp +49:57.493 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27401 network=tcp +49:57.493 [INFO] TestAgent: started state syncer +49:57.494 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27406 network=tcp +49:57.549 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:57.549 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27405 [Candidate]" term=2 +49:57.549 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:57.549 [DEBUG] TestAgent.server.raft: vote granted: from=6ba861d8-7ff7-eb9e-7c8d-a52d845437bc term=2 tally=1 +49:57.549 [INFO] TestAgent.server.raft: election won: tally=1 +49:57.549 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27405 [Leader]" +49:57.550 [INFO] TestAgent.server: cluster leadership acquired +49:57.550 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27405 +49:57.550 [INFO] TestAgent.server: New leader elected: payload=Node-6ba861d8-7ff7-eb9e-7c8d-a52d845437bc +49:57.550 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:57.551 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:57.551 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:57.551 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:57.551 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:57.551 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:57.551 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6ba861d8-7ff7-eb9e-7c8d-a52d845437bc +49:57.551 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6ba861d8-7ff7-eb9e-7c8d-a52d845437bc +49:57.552 [INFO] TestAgent.server: federation state anti-entropy synced +49:57.755 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:40816 latency=384.471µs +49:57.755 [INFO] TestAgent: Requesting shutdown +49:57.755 [INFO] TestAgent.server: shutting down server +49:57.755 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:57.755 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:57.755 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:57.755 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:57.755 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:57.756 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:57.756 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:57.756 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:57.756 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:57.756 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:57.756 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:57.756 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:57.756Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:57.756Z [INFO] agent.router.manager: shutting down +49:57.756 [INFO] TestAgent: consul server down +49:57.756 [INFO] TestAgent: shutdown complete +49:57.756 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27400 network=tcp +49:57.756 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27400 network=udp +49:57.756 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27401 network=tcp +49:58.256 [INFO] TestAgent: Waiting for endpoints to shut down +49:58.256 [INFO] TestAgent: Endpoints down === RUN TestCommandConfigWatcher/-sidecar-for,_multiple_sidecars -2021-10-31T23:16:54.078Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:54.132Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:54.143 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c0462c05-cf13-9bed-be08-bca4f4505e71 Address:127.0.0.1:11076}]" -16:54.144 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c0462c05-cf13-9bed-be08-bca4f4505e71.dc1 127.0.0.1 -16:54.145 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c0462c05-cf13-9bed-be08-bca4f4505e71 127.0.0.1 -2021-10-31T23:16:54.145Z [INFO] agent.router: Initializing LAN area manager -16:54.145 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11076 [Follower]" leader= -16:54.146 [INFO] TestAgent.server: Adding LAN server: server="Node-c0462c05-cf13-9bed-be08-bca4f4505e71 (Addr: tcp/127.0.0.1:11076) (DC: dc1)" -16:54.146 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c0462c05-cf13-9bed-be08-bca4f4505e71.dc1 area=wan -16:54.149 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11071 network=udp -16:54.149 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11071 network=tcp -16:54.158 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11072 network=tcp -16:54.158 [INFO] TestAgent: started state syncer -16:54.158 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11077 network=tcp -16:54.216 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:54.216 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11076 [Candidate]" term=2 -16:54.217 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:54.217 [DEBUG] TestAgent.server.raft: vote granted: from=c0462c05-cf13-9bed-be08-bca4f4505e71 term=2 tally=1 -16:54.217 [INFO] TestAgent.server.raft: election won: tally=1 -16:54.217 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11076 [Leader]" -16:54.217 [INFO] TestAgent.server: cluster leadership acquired -16:54.217 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11076 -16:54.218 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:54.220 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:54.220 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:54.220 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:54.220 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:54.220 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:54.220 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c0462c05-cf13-9bed-be08-bca4f4505e71 -16:54.220 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c0462c05-cf13-9bed-be08-bca4f4505e71 -16:54.220 [INFO] TestAgent.server: New leader elected: payload=Node-c0462c05-cf13-9bed-be08-bca4f4505e71 -16:54.220 [INFO] TestAgent.server: federation state anti-entropy synced -16:54.436 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:49788 latency=1.905112ms -16:54.437 [INFO] TestAgent: Requesting shutdown -16:54.437 [INFO] TestAgent.server: shutting down server -16:54.437 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:54.437 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:54.437 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:54.437 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:54.437 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:54.437 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:54.437 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:54.437 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -16:54.437 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:54.437 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:54.437 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:54.446 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:54.446Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:54.446Z [INFO] agent.router.manager: shutting down -16:54.447 [INFO] TestAgent: consul server down -16:54.447 [INFO] TestAgent: shutdown complete -16:54.447 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11071 network=tcp -16:54.447 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11071 network=udp -16:54.447 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11072 network=tcp -16:54.949 [INFO] TestAgent: Waiting for endpoints to shut down -16:54.949 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:58.267Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:58.285Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:58.286 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:64193535-22fd-2add-8a97-62cc096b2637 Address:127.0.0.1:27412}]" +49:58.286 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27412 [Follower]" leader= +49:58.287 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-64193535-22fd-2add-8a97-62cc096b2637.dc1 127.0.0.1 +49:58.287 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-64193535-22fd-2add-8a97-62cc096b2637 127.0.0.1 +2022-12-04T05:49:58.287Z [INFO] agent.router: Initializing LAN area manager +49:58.287 [INFO] TestAgent.server: Adding LAN server: server="Node-64193535-22fd-2add-8a97-62cc096b2637 (Addr: tcp/127.0.0.1:27412) (DC: dc1)" +49:58.287 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-64193535-22fd-2add-8a97-62cc096b2637.dc1 area=wan +49:58.287 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27407 network=udp +49:58.287 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27407 network=tcp +49:58.288 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27408 network=tcp +49:58.288 [INFO] TestAgent: started state syncer +49:58.288 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27413 network=tcp +49:58.356 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:58.356 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27412 [Candidate]" term=2 +49:58.356 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:58.356 [DEBUG] TestAgent.server.raft: vote granted: from=64193535-22fd-2add-8a97-62cc096b2637 term=2 tally=1 +49:58.356 [INFO] TestAgent.server.raft: election won: tally=1 +49:58.356 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27412 [Leader]" +49:58.356 [INFO] TestAgent.server: cluster leadership acquired +49:58.356 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27412 +49:58.356 [INFO] TestAgent.server: New leader elected: payload=Node-64193535-22fd-2add-8a97-62cc096b2637 +49:58.357 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:58.358 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:58.358 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:58.358 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:58.358 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:58.358 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:58.358 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-64193535-22fd-2add-8a97-62cc096b2637 +49:58.358 [INFO] TestAgent.server: member joined, marking health alive: member=Node-64193535-22fd-2add-8a97-62cc096b2637 +49:58.359 [INFO] TestAgent.server: federation state anti-entropy synced +49:58.565 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:58.565 [INFO] TestAgent: Synced node info +49:58.565 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars +49:58.566 [INFO] TestAgent: Synced service: service=no-sidecar +49:58.566 [INFO] TestAgent: Synced service: service=one-sidecar +49:58.566 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy +49:58.567 [INFO] TestAgent: Synced service: service=two-sidecars +49:58.567 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy +49:58.567 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 +49:58.567 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 +49:58.567 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 +49:58.567 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 +49:58.567 [DEBUG] TestAgent: Node info in sync +49:58.567 [DEBUG] TestAgent: Service in sync: service=other-sidecar-for-two-sidecars +49:58.567 [DEBUG] TestAgent: Service in sync: service=no-sidecar +49:58.567 [DEBUG] TestAgent: Service in sync: service=one-sidecar +49:58.567 [DEBUG] TestAgent: Service in sync: service=one-sidecar-sidecar-proxy +49:58.567 [DEBUG] TestAgent: Service in sync: service=two-sidecars +49:58.567 [DEBUG] TestAgent: Service in sync: service=two-sidecars-sidecar-proxy +49:58.567 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 +49:58.567 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 +49:58.567 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 +49:58.567 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 +49:58.685 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:40276 latency=384.777µs +49:58.685 [INFO] TestAgent: Requesting shutdown +49:58.686 [INFO] TestAgent.server: shutting down server +49:58.686 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:58.686 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:58.686 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:58.686 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:58.686 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:58.686 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:58.686 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:58.686 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:58.686 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:58.686 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:58.686 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:58.686Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:58.686Z [INFO] agent.router.manager: shutting down +49:58.686 [INFO] TestAgent: consul server down +49:58.686 [INFO] TestAgent: shutdown complete +49:58.686 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27407 network=tcp +49:58.686 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27407 network=udp +49:58.686 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27408 network=tcp +49:59.186 [INFO] TestAgent: Waiting for endpoints to shut down +49:59.186 [INFO] TestAgent: Endpoints down === RUN TestCommandConfigWatcher/-sidecar-for,_non-existent -2021-10-31T23:16:54.985Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:55.023Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:55.025 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:316cd7d3-b7b6-1fe3-4e06-f728d67e575e Address:127.0.0.1:11083}]" -16:55.025 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11083 [Follower]" leader= -16:55.038 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-316cd7d3-b7b6-1fe3-4e06-f728d67e575e.dc1 127.0.0.1 -16:55.038 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-316cd7d3-b7b6-1fe3-4e06-f728d67e575e 127.0.0.1 -2021-10-31T23:16:55.038Z [INFO] agent.router: Initializing LAN area manager -16:55.039 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11078 network=udp -16:55.039 [INFO] TestAgent.server: Adding LAN server: server="Node-316cd7d3-b7b6-1fe3-4e06-f728d67e575e (Addr: tcp/127.0.0.1:11083) (DC: dc1)" -16:55.039 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-316cd7d3-b7b6-1fe3-4e06-f728d67e575e.dc1 area=wan -16:55.039 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11078 network=tcp -16:55.040 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11079 network=tcp -16:55.040 [INFO] TestAgent: started state syncer -16:55.040 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11084 network=tcp -16:55.063 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:55.063 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11083 [Candidate]" term=2 -16:55.063 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:55.063 [DEBUG] TestAgent.server.raft: vote granted: from=316cd7d3-b7b6-1fe3-4e06-f728d67e575e term=2 tally=1 -16:55.063 [INFO] TestAgent.server.raft: election won: tally=1 -16:55.063 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11083 [Leader]" -16:55.064 [INFO] TestAgent.server: cluster leadership acquired -16:55.064 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11083 -16:55.064 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:55.065 [INFO] TestAgent.server: New leader elected: payload=Node-316cd7d3-b7b6-1fe3-4e06-f728d67e575e -16:55.066 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:55.066 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:55.066 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:55.066 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:55.066 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:55.070 [INFO] TestAgent.server: federation state anti-entropy synced -16:55.073 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-316cd7d3-b7b6-1fe3-4e06-f728d67e575e -16:55.074 [INFO] TestAgent.server: member joined, marking health alive: member=Node-316cd7d3-b7b6-1fe3-4e06-f728d67e575e -16:55.198 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:55.198 [INFO] TestAgent: Synced node info -16:55.199 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy -16:55.199 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars -16:55.199 [INFO] TestAgent: Synced service: service=no-sidecar -16:55.200 [INFO] TestAgent: Synced service: service=one-sidecar -16:55.200 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy -16:55.201 [INFO] TestAgent: Synced service: service=two-sidecars -16:55.201 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 -16:55.201 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 -16:55.201 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 -16:55.201 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 -16:55.201 [DEBUG] TestAgent: Node info in sync -16:55.201 [DEBUG] TestAgent: Service in sync: service=other-sidecar-for-two-sidecars -16:55.201 [DEBUG] TestAgent: Service in sync: service=no-sidecar -16:55.201 [DEBUG] TestAgent: Service in sync: service=one-sidecar -16:55.201 [DEBUG] TestAgent: Service in sync: service=one-sidecar-sidecar-proxy -16:55.201 [DEBUG] TestAgent: Service in sync: service=two-sidecars -16:55.201 [DEBUG] TestAgent: Service in sync: service=two-sidecars-sidecar-proxy -16:55.201 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 -16:55.201 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 -16:55.201 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 -16:55.201 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 -16:55.267 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:48990 latency=227.913µs -16:55.268 [INFO] TestAgent: Requesting shutdown -16:55.268 [INFO] TestAgent.server: shutting down server -16:55.268 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:55.268 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:55.268 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:55.268 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:55.268 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:55.268 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:55.268 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:55.268 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:55.268 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:55.268 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:55.268 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:55.268Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:55.268Z [INFO] agent.router.manager: shutting down -16:55.268 [INFO] TestAgent: consul server down -16:55.268 [INFO] TestAgent: shutdown complete -16:55.268 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11078 network=tcp -16:55.268 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11078 network=udp -16:55.268 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11079 network=tcp -16:55.773 [INFO] TestAgent: Waiting for endpoints to shut down -16:55.773 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:59.203Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:59.213Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:59.215 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:14f02ce5-b3ef-0244-1cb1-987e532b5a93 Address:127.0.0.1:27419}]" +49:59.215 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27419 [Follower]" leader= +49:59.215 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-14f02ce5-b3ef-0244-1cb1-987e532b5a93.dc1 127.0.0.1 +49:59.215 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-14f02ce5-b3ef-0244-1cb1-987e532b5a93 127.0.0.1 +2022-12-04T05:49:59.215Z [INFO] agent.router: Initializing LAN area manager +49:59.215 [INFO] TestAgent.server: Adding LAN server: server="Node-14f02ce5-b3ef-0244-1cb1-987e532b5a93 (Addr: tcp/127.0.0.1:27419) (DC: dc1)" +49:59.216 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-14f02ce5-b3ef-0244-1cb1-987e532b5a93.dc1 area=wan +49:59.216 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27414 network=udp +49:59.216 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27414 network=tcp +49:59.216 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27415 network=tcp +49:59.216 [INFO] TestAgent: started state syncer +49:59.216 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27420 network=tcp +49:59.261 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:59.261 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27419 [Candidate]" term=2 +49:59.261 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:59.261 [DEBUG] TestAgent.server.raft: vote granted: from=14f02ce5-b3ef-0244-1cb1-987e532b5a93 term=2 tally=1 +49:59.261 [INFO] TestAgent.server.raft: election won: tally=1 +49:59.261 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27419 [Leader]" +49:59.261 [INFO] TestAgent.server: cluster leadership acquired +49:59.261 [INFO] TestAgent.server: New leader elected: payload=Node-14f02ce5-b3ef-0244-1cb1-987e532b5a93 +49:59.261 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27419 +49:59.262 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:59.263 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:59.263 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:59.263 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:59.263 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:59.263 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:59.263 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-14f02ce5-b3ef-0244-1cb1-987e532b5a93 +49:59.263 [INFO] TestAgent.server: member joined, marking health alive: member=Node-14f02ce5-b3ef-0244-1cb1-987e532b5a93 +49:59.264 [INFO] TestAgent.server: federation state anti-entropy synced +49:59.516 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:59.516 [INFO] TestAgent: Synced node info +49:59.516 [INFO] TestAgent: Synced service: service=no-sidecar +49:59.517 [INFO] TestAgent: Synced service: service=one-sidecar +49:59.517 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy +49:59.517 [INFO] TestAgent: Synced service: service=two-sidecars +49:59.518 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy +49:59.518 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars +49:59.518 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 +49:59.518 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 +49:59.518 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 +49:59.518 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 +49:59.518 [DEBUG] TestAgent: Node info in sync +49:59.518 [DEBUG] TestAgent: Service in sync: service=no-sidecar +49:59.518 [DEBUG] TestAgent: Service in sync: service=one-sidecar +49:59.518 [DEBUG] TestAgent: Service in sync: service=one-sidecar-sidecar-proxy +49:59.518 [DEBUG] TestAgent: Service in sync: service=two-sidecars +49:59.518 [DEBUG] TestAgent: Service in sync: service=two-sidecars-sidecar-proxy +49:59.518 [DEBUG] TestAgent: Service in sync: service=other-sidecar-for-two-sidecars +49:59.518 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 +49:59.518 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 +49:59.518 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 +49:59.518 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 +49:59.535 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:58618 latency=395.004µs +49:59.535 [INFO] TestAgent: Requesting shutdown +49:59.536 [INFO] TestAgent.server: shutting down server +49:59.536 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:59.536 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:59.536 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:59.536 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:59.536 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:59.536 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:59.536 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:59.536 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:59.536 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:59.536 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:59.536 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:59.536Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:59.536Z [INFO] agent.router.manager: shutting down +49:59.536 [INFO] TestAgent: consul server down +49:59.536 [INFO] TestAgent: shutdown complete +49:59.536 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27414 network=tcp +49:59.536 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27414 network=udp +49:59.536 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27415 network=tcp +50:00.036 [INFO] TestAgent: Waiting for endpoints to shut down +50:00.036 [INFO] TestAgent: Endpoints down === RUN TestCommandConfigWatcher/-sidecar-for,_one_sidecar -2021-10-31T23:16:55.825Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:55.887Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:55.889 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bf3f9db7-663e-bada-a56d-159118a73515 Address:127.0.0.1:11090}]" -16:55.889 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bf3f9db7-663e-bada-a56d-159118a73515.dc1 127.0.0.1 -16:55.889 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11090 [Follower]" leader= -16:55.899 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bf3f9db7-663e-bada-a56d-159118a73515 127.0.0.1 -2021-10-31T23:16:55.899Z [INFO] agent.router: Initializing LAN area manager -16:55.899 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11085 network=udp -16:55.899 [INFO] TestAgent.server: Adding LAN server: server="Node-bf3f9db7-663e-bada-a56d-159118a73515 (Addr: tcp/127.0.0.1:11090) (DC: dc1)" -16:55.899 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bf3f9db7-663e-bada-a56d-159118a73515.dc1 area=wan -16:55.900 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11085 network=tcp -16:55.900 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11086 network=tcp -16:55.901 [INFO] TestAgent: started state syncer -16:55.901 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11091 network=tcp -16:55.933 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:55.934 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11090 [Candidate]" term=2 -16:55.934 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:55.934 [DEBUG] TestAgent.server.raft: vote granted: from=bf3f9db7-663e-bada-a56d-159118a73515 term=2 tally=1 -16:55.934 [INFO] TestAgent.server.raft: election won: tally=1 -16:55.934 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11090 [Leader]" -16:55.934 [INFO] TestAgent.server: cluster leadership acquired -16:55.934 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11090 -16:55.935 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:55.936 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:55.936 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:55.936 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:55.936 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:55.936 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:55.936 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bf3f9db7-663e-bada-a56d-159118a73515 -16:55.936 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bf3f9db7-663e-bada-a56d-159118a73515 -16:55.937 [INFO] TestAgent.server: New leader elected: payload=Node-bf3f9db7-663e-bada-a56d-159118a73515 -16:55.937 [INFO] TestAgent.server: federation state anti-entropy synced -16:56.006 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:56.006 [INFO] TestAgent: Synced node info -16:56.007 [INFO] TestAgent: Synced service: service=one-sidecar -16:56.007 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy -16:56.008 [INFO] TestAgent: Synced service: service=two-sidecars -16:56.008 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy -16:56.009 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars -16:56.009 [INFO] TestAgent: Synced service: service=no-sidecar -16:56.009 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 -16:56.009 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 -16:56.009 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 -16:56.009 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 -16:56.009 [DEBUG] TestAgent: Node info in sync -16:56.009 [DEBUG] TestAgent: Service in sync: service=two-sidecars -16:56.009 [DEBUG] TestAgent: Service in sync: service=two-sidecars-sidecar-proxy -16:56.009 [DEBUG] TestAgent: Service in sync: service=other-sidecar-for-two-sidecars -16:56.009 [DEBUG] TestAgent: Service in sync: service=no-sidecar -16:56.009 [DEBUG] TestAgent: Service in sync: service=one-sidecar -16:56.009 [DEBUG] TestAgent: Service in sync: service=one-sidecar-sidecar-proxy -16:56.009 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 -16:56.009 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 -16:56.009 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 -16:56.009 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 -16:56.170 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:55698 latency=479.774µs -16:56.175 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/one-sidecar-sidecar-proxy from=127.0.0.1:55698 latency=437.546µs -16:56.176 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/one-sidecar-sidecar-proxy from=127.0.0.1:55700 latency=676.561µs -16:56.178 [INFO] TestAgent: Requesting shutdown -16:56.178 [INFO] TestAgent.server: shutting down server -16:56.178 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:56.178 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:56.178 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:56.178 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:56.178 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:56.179 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:56.179 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:56.179 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:56.179 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:56.179 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:56.179 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:56.179Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:56.179Z [INFO] agent.router.manager: shutting down -16:56.179 [INFO] TestAgent: consul server down -16:56.179 [INFO] TestAgent: shutdown complete -16:56.179 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11085 network=tcp -16:56.179 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11085 network=udp -16:56.179 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11086 network=tcp -16:56.681 [INFO] TestAgent: Waiting for endpoints to shut down -16:56.681 [INFO] TestAgent: Endpoints down ---- PASS: TestCommandConfigWatcher (7.13s) - --- PASS: TestCommandConfigWatcher/-service_flag_only (0.99s) - --- PASS: TestCommandConfigWatcher/-service_flag_with_upstreams (0.96s) - --- PASS: TestCommandConfigWatcher/-service_flag_with_-service-addr (0.86s) - --- PASS: TestCommandConfigWatcher/-service,_-service-addr,_-listen (0.80s) - --- PASS: TestCommandConfigWatcher/-sidecar-for,_no_sidecar (0.88s) - --- PASS: TestCommandConfigWatcher/-sidecar-for,_multiple_sidecars (0.91s) - --- PASS: TestCommandConfigWatcher/-sidecar-for,_non-existent (0.82s) - --- PASS: TestCommandConfigWatcher/-sidecar-for,_one_sidecar (0.91s) +2022-12-04T05:50:00.070Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:00.080Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:00.081 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:62aef5e0-a053-4860-de9f-7d61c39a8411 Address:127.0.0.1:27426}]" +50:00.081 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27426 [Follower]" leader= +50:00.082 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-62aef5e0-a053-4860-de9f-7d61c39a8411.dc1 127.0.0.1 +50:00.082 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-62aef5e0-a053-4860-de9f-7d61c39a8411 127.0.0.1 +2022-12-04T05:50:00.082Z [INFO] agent.router: Initializing LAN area manager +50:00.082 [INFO] TestAgent.server: Adding LAN server: server="Node-62aef5e0-a053-4860-de9f-7d61c39a8411 (Addr: tcp/127.0.0.1:27426) (DC: dc1)" +50:00.082 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-62aef5e0-a053-4860-de9f-7d61c39a8411.dc1 area=wan +50:00.082 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27421 network=udp +50:00.083 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27421 network=tcp +50:00.083 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27422 network=tcp +50:00.083 [INFO] TestAgent: started state syncer +50:00.083 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27427 network=tcp +50:00.122 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:00.122 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27426 [Candidate]" term=2 +50:00.122 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:00.122 [DEBUG] TestAgent.server.raft: vote granted: from=62aef5e0-a053-4860-de9f-7d61c39a8411 term=2 tally=1 +50:00.122 [INFO] TestAgent.server.raft: election won: tally=1 +50:00.122 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27426 [Leader]" +50:00.122 [INFO] TestAgent.server: cluster leadership acquired +50:00.123 [INFO] TestAgent.server: New leader elected: payload=Node-62aef5e0-a053-4860-de9f-7d61c39a8411 +50:00.123 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27426 +50:00.126 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:00.127 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:00.127 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:00.127 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:00.127 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:00.127 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:00.127 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-62aef5e0-a053-4860-de9f-7d61c39a8411 +50:00.127 [INFO] TestAgent.server: member joined, marking health alive: member=Node-62aef5e0-a053-4860-de9f-7d61c39a8411 +50:00.128 [INFO] TestAgent.server: federation state anti-entropy synced +50:00.200 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:00.200 [INFO] TestAgent: Synced node info +50:00.201 [INFO] TestAgent: Synced service: service=two-sidecars-sidecar-proxy +50:00.201 [INFO] TestAgent: Synced service: service=other-sidecar-for-two-sidecars +50:00.201 [INFO] TestAgent: Synced service: service=no-sidecar +50:00.202 [INFO] TestAgent: Synced service: service=one-sidecar +50:00.202 [INFO] TestAgent: Synced service: service=one-sidecar-sidecar-proxy +50:00.202 [INFO] TestAgent: Synced service: service=two-sidecars +50:00.202 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 +50:00.202 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 +50:00.202 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 +50:00.202 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 +50:00.202 [DEBUG] TestAgent: Node info in sync +50:00.202 [DEBUG] TestAgent: Service in sync: service=one-sidecar +50:00.202 [DEBUG] TestAgent: Service in sync: service=one-sidecar-sidecar-proxy +50:00.202 [DEBUG] TestAgent: Service in sync: service=two-sidecars +50:00.202 [DEBUG] TestAgent: Service in sync: service=two-sidecars-sidecar-proxy +50:00.202 [DEBUG] TestAgent: Service in sync: service=other-sidecar-for-two-sidecars +50:00.202 [DEBUG] TestAgent: Service in sync: service=no-sidecar +50:00.202 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:2 +50:00.202 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:1 +50:00.202 [DEBUG] TestAgent: Check in sync: check=service:one-sidecar-sidecar-proxy:2 +50:00.202 [DEBUG] TestAgent: Check in sync: check=service:two-sidecars-sidecar-proxy:1 +50:00.384 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:60340 latency=438.53µs +50:00.386 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/one-sidecar-sidecar-proxy from=127.0.0.1:60340 latency=213.172µs +50:00.386 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/one-sidecar-sidecar-proxy from=127.0.0.1:60342 latency=576.17µs +50:00.387 [INFO] TestAgent: Requesting shutdown +50:00.387 [INFO] TestAgent.server: shutting down server +50:00.387 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:00.387 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:00.387 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:00.387 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:00.387 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:00.387 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:00.387 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:00.387 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:00.387 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:00.392 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:00.393 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:00.393Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:00.393Z [INFO] agent.router.manager: shutting down +50:00.393 [INFO] TestAgent: consul server down +50:00.393 [INFO] TestAgent: shutdown complete +50:00.393 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27421 network=tcp +50:00.393 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27421 network=udp +50:00.393 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27422 network=tcp +50:00.394 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/service/one-sidecar-sidecar-proxy?hash=3bafbcac3734b91 from=127.0.0.1:60342 latency=2.23116ms +50:00.894 [INFO] TestAgent: Waiting for endpoints to shut down +50:00.894 [INFO] TestAgent: Endpoints down +--- PASS: TestCommandConfigWatcher (6.35s) + --- PASS: TestCommandConfigWatcher/-service_flag_only (0.89s) + --- PASS: TestCommandConfigWatcher/-service_flag_with_upstreams (0.65s) + --- PASS: TestCommandConfigWatcher/-service_flag_with_-service-addr (0.63s) + --- PASS: TestCommandConfigWatcher/-service,_-service-addr,_-listen (0.75s) + --- PASS: TestCommandConfigWatcher/-sidecar-for,_no_sidecar (0.80s) + --- PASS: TestCommandConfigWatcher/-sidecar-for,_multiple_sidecars (0.93s) + --- PASS: TestCommandConfigWatcher/-sidecar-for,_non-existent (0.85s) + --- PASS: TestCommandConfigWatcher/-sidecar-for,_one_sidecar (0.86s) PASS -ok github.com/hashicorp/consul/command/connect/proxy 7.418s +ok github.com/hashicorp/consul/command/connect/proxy 6.399s === RUN TestEventCommand_noTabs === PAUSE TestEventCommand_noTabs === RUN TestEventCommand === PAUSE TestEventCommand === CONT TestEventCommand_noTabs ---- PASS: TestEventCommand_noTabs (0.00s) === CONT TestEventCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestEventCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:16:51.727Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:51.792Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:51.799 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d14de545-d4d8-d5ed-b51f-2a57de0a05d8 Address:127.0.0.1:26342}]" -16:51.800 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d14de545-d4d8-d5ed-b51f-2a57de0a05d8.dc1 127.0.0.1 -16:51.800 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:26342 [Follower]" leader= -16:51.800 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d14de545-d4d8-d5ed-b51f-2a57de0a05d8 127.0.0.1 -2021-10-31T23:16:51.800Z [INFO] agent.router: Initializing LAN area manager -16:51.801 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26337 network=udp -16:51.801 [INFO] TestAgent.server: Adding LAN server: server="Node-d14de545-d4d8-d5ed-b51f-2a57de0a05d8 (Addr: tcp/127.0.0.1:26342) (DC: dc1)" -16:51.801 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d14de545-d4d8-d5ed-b51f-2a57de0a05d8.dc1 area=wan -16:51.801 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26337 network=tcp -16:51.810 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:26338 network=tcp -16:51.810 [INFO] TestAgent: started state syncer -16:51.810 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:26343 network=tcp -16:51.862 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:51.862 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:26342 [Candidate]" term=2 -16:51.862 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:51.862 [DEBUG] TestAgent.server.raft: vote granted: from=d14de545-d4d8-d5ed-b51f-2a57de0a05d8 term=2 tally=1 -16:51.862 [INFO] TestAgent.server.raft: election won: tally=1 -16:51.862 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:26342 [Leader]" -16:51.862 [INFO] TestAgent.server: cluster leadership acquired -16:51.862 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:26342 -16:51.863 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:51.865 [INFO] TestAgent.server: New leader elected: payload=Node-d14de545-d4d8-d5ed-b51f-2a57de0a05d8 -16:51.866 [INFO] TestAgent: Synced node info -16:51.918 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:51.918 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:51.918 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:51.918 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:51.918 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:51.918 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d14de545-d4d8-d5ed-b51f-2a57de0a05d8 -16:51.918 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d14de545-d4d8-d5ed-b51f-2a57de0a05d8 -16:51.919 [INFO] TestAgent.server: federation state anti-entropy synced -16:52.032 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:33448 latency=5.069909ms -16:52.047 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/cmd from=127.0.0.1:33448 latency=1.585124ms -16:52.047 [DEBUG] TestAgent.server: User event: event=cmd -16:52.048 [DEBUG] TestAgent: new event: event_name=cmd event_id=dc149108-9245-977a-6f10-97e2fc2138e7 -16:52.048 [INFO] TestAgent: Requesting shutdown -16:52.048 [INFO] TestAgent.server: shutting down server -16:52.048 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:52.048 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:52.048 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:52.048 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:52.048 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:52.048 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:52.048 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:52.048 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:52.048 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:52.048 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:52.048 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:52.048Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:52.048Z [INFO] agent.router.manager: shutting down -16:52.048 [INFO] TestAgent: consul server down -16:52.048 [INFO] TestAgent: shutdown complete -16:52.048 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26337 network=tcp -16:52.048 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26337 network=udp -16:52.048 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:26338 network=tcp -16:52.549 [INFO] TestAgent: Waiting for endpoints to shut down -16:52.549 [INFO] TestAgent: Endpoints down ---- PASS: TestEventCommand (0.93s) +2022-12-04T05:49:55.667Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:55.681Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:55.683 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:56a1ff19-7e6c-a77c-df71-4e8fce3bf44c Address:127.0.0.1:13069}]" +49:55.683 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= +49:55.684 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-56a1ff19-7e6c-a77c-df71-4e8fce3bf44c.dc1 127.0.0.1 +49:55.684 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-56a1ff19-7e6c-a77c-df71-4e8fce3bf44c 127.0.0.1 +2022-12-04T05:49:55.684Z [INFO] agent.router: Initializing LAN area manager +49:55.685 [INFO] TestAgent.server: Adding LAN server: server="Node-56a1ff19-7e6c-a77c-df71-4e8fce3bf44c (Addr: tcp/127.0.0.1:13069) (DC: dc1)" +49:55.685 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-56a1ff19-7e6c-a77c-df71-4e8fce3bf44c.dc1 area=wan +49:55.685 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp +49:55.685 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp +49:55.686 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp +49:55.686 [INFO] TestAgent: started state syncer +49:55.686 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp +49:55.747 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:55.747 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 +49:55.747 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:55.747 [DEBUG] TestAgent.server.raft: vote granted: from=56a1ff19-7e6c-a77c-df71-4e8fce3bf44c term=2 tally=1 +49:55.747 [INFO] TestAgent.server.raft: election won: tally=1 +49:55.747 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" +49:55.747 [INFO] TestAgent.server: cluster leadership acquired +49:55.748 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13069 +49:55.749 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:55.752 [INFO] TestAgent.server: New leader elected: payload=Node-56a1ff19-7e6c-a77c-df71-4e8fce3bf44c +49:55.769 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:54422 latency=2.469008ms +49:55.770 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:55.770 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:55.770 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:55.770 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:55.770 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:55.770 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-56a1ff19-7e6c-a77c-df71-4e8fce3bf44c +49:55.771 [INFO] TestAgent.server: member joined, marking health alive: member=Node-56a1ff19-7e6c-a77c-df71-4e8fce3bf44c +49:55.771 [INFO] TestAgent.server: federation state anti-entropy synced +49:55.773 [DEBUG] TestAgent.server: User event: event=cmd +49:55.773 [DEBUG] TestAgent: new event: event_name=cmd event_id=6668ef31-1042-a1c7-201d-3eb6f3999bc2 +49:55.774 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/cmd from=127.0.0.1:54422 latency=1.021116ms +49:55.774 [INFO] TestAgent: Requesting shutdown +49:55.774 [INFO] TestAgent.server: shutting down server +49:55.774 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:55.774 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:55.774 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:55.774 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:55.774 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:55.774 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:55.774 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:55.774 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:55.774 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:55.774 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:55.774 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:55.774 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:55.775Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:55.775Z [INFO] agent.router.manager: shutting down +49:55.775 [INFO] TestAgent: consul server down +49:55.775 [INFO] TestAgent: shutdown complete +49:55.775 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp +49:55.775 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp +49:55.775 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp +49:56.275 [INFO] TestAgent: Waiting for endpoints to shut down +49:56.275 [INFO] TestAgent: Endpoints down +--- PASS: TestEventCommand (0.64s) PASS -ok github.com/hashicorp/consul/command/event 1.176s +ok github.com/hashicorp/consul/command/event 0.684s === RUN TestExecCommand_noTabs === PAUSE TestExecCommand_noTabs === RUN TestExecCommand @@ -83940,569 +84119,572 @@ === RUN TestExecCommand_StreamResults === PAUSE TestExecCommand_StreamResults === CONT TestExecCommand_noTabs -=== CONT TestExecCommand_UploadDestroy -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestExecCommand_noTabs (0.00s) +=== CONT TestExecCommand_StreamResults === CONT TestExecCommand_Sessions_Foreign === CONT TestExecCommand_Sessions === CONT TestExecCommand_Validate ---- PASS: TestExecCommand_Validate (0.00s) -=== CONT TestExecCommand_CrossDC +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestExecCommand_NoShell +=== CONT TestExecCommand_UploadDestroy === CONT TestExecCommand -=== CONT TestExecCommand_StreamResults +=== CONT TestExecCommand_CrossDC +--- PASS: TestExecCommand_noTabs (0.00s) +--- PASS: TestExecCommand_Validate (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:16:59.228Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:59.412Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:59.415 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d20f48cc-06f0-b047-eef2-5ffe3658d594 Address:127.0.0.1:15111}]" -16:59.422 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15111 [Follower]" leader= -16:59.454 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d20f48cc-06f0-b047-eef2-5ffe3658d594.dc1 127.0.0.1 -16:59.454 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d20f48cc-06f0-b047-eef2-5ffe3658d594 127.0.0.1 -2021-10-31T23:16:59.455Z [INFO] agent.router: Initializing LAN area manager -16:59.455 [INFO] TestAgent.server: Adding LAN server: server="Node-d20f48cc-06f0-b047-eef2-5ffe3658d594 (Addr: tcp/127.0.0.1:15111) (DC: dc1)" -16:59.455 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d20f48cc-06f0-b047-eef2-5ffe3658d594.dc1 area=wan -16:59.458 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=udp -16:59.458 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=tcp -16:59.458 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15107 network=tcp -16:59.458 [INFO] TestAgent: started state syncer -16:59.459 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15112 network=tcp -16:59.481 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:59.481 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15111 [Candidate]" term=2 -16:59.482 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:59.482 [DEBUG] TestAgent.server.raft: vote granted: from=d20f48cc-06f0-b047-eef2-5ffe3658d594 term=2 tally=1 -16:59.482 [INFO] TestAgent.server.raft: election won: tally=1 -16:59.482 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15111 [Leader]" -16:59.482 [INFO] TestAgent.server: cluster leadership acquired -16:59.482 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15111 -16:59.483 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:59.484 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:59.484 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:59.484 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:59.484 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:59.484 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:59.484 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d20f48cc-06f0-b047-eef2-5ffe3658d594 -16:59.484 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d20f48cc-06f0-b047-eef2-5ffe3658d594 -16:59.485 [INFO] TestAgent.server: New leader elected: payload=Node-d20f48cc-06f0-b047-eef2-5ffe3658d594 -16:59.490 [INFO] TestAgent.server: federation state anti-entropy synced -16:59.493 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:32790 latency=1.091562ms -16:59.495 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/444b5e49-d1fd-fb72-f7d1-e19863c09724/job?acquire=444b5e49-d1fd-fb72-f7d1-e19863c09724 from=127.0.0.1:32790 latency=1.134602ms -16:59.496 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/444b5e49-d1fd-fb72-f7d1-e19863c09724/job from=127.0.0.1:32792 latency=305.965µs -16:59.497 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/_rexec/444b5e49-d1fd-fb72-f7d1-e19863c09724?recurse= from=127.0.0.1:32790 latency=364.73µs -16:59.510 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/444b5e49-d1fd-fb72-f7d1-e19863c09724/job from=127.0.0.1:32794 latency=96.619µs -16:59.510 [INFO] TestAgent: Requesting shutdown -16:59.510 [INFO] TestAgent.server: shutting down server -16:59.510 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:59.510 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:59.510 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:59.510 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:59.510 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:59.510 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:59.510 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:59.510 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -16:59.510 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:59.510 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:59.510 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:59.510 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:59.510Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:59.510Z [INFO] agent.router.manager: shutting down -16:59.511 [INFO] TestAgent: consul server down -16:59.511 [INFO] TestAgent: shutdown complete -16:59.511 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=tcp -16:59.511 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=udp -16:59.511 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15107 network=tcp -17:00.011 [INFO] TestAgent: Waiting for endpoints to shut down -17:00.011 [INFO] TestAgent: Endpoints down ---- PASS: TestExecCommand_UploadDestroy (1.03s) -2021-10-31T23:16:59.067Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:59.121Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:59.124 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c7e3a84e-ad2a-f51d-56c5-d906bdc3ca20 Address:127.0.0.1:15125}]" -16:59.125 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c7e3a84e-ad2a-f51d-56c5-d906bdc3ca20.dc1 127.0.0.1 -16:59.125 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15125 [Follower]" leader= -16:59.170 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c7e3a84e-ad2a-f51d-56c5-d906bdc3ca20 127.0.0.1 -2021-10-31T23:16:59.170Z [INFO] agent.router: Initializing LAN area manager -16:59.175 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:59.175 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15125 [Candidate]" term=2 -16:59.177 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15120 network=udp -16:59.177 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15120 network=tcp -16:59.204 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c7e3a84e-ad2a-f51d-56c5-d906bdc3ca20.dc1 area=wan -16:59.206 [INFO] TestAgent.server: Adding LAN server: server="Node-c7e3a84e-ad2a-f51d-56c5-d906bdc3ca20 (Addr: tcp/127.0.0.1:15125) (DC: dc1)" -16:59.236 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15121 network=tcp -16:59.237 [INFO] TestAgent: started state syncer -16:59.237 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15126 network=tcp -16:59.237 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:59.237 [DEBUG] TestAgent.server.raft: vote granted: from=c7e3a84e-ad2a-f51d-56c5-d906bdc3ca20 term=2 tally=1 -16:59.237 [INFO] TestAgent.server.raft: election won: tally=1 -16:59.237 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15125 [Leader]" -16:59.237 [INFO] TestAgent.server: cluster leadership acquired -16:59.238 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15125 -16:59.239 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:59.239 [INFO] TestAgent.server: New leader elected: payload=Node-c7e3a84e-ad2a-f51d-56c5-d906bdc3ca20 -16:59.292 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:59.292 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:59.292 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:59.292 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:59.292 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:59.292 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c7e3a84e-ad2a-f51d-56c5-d906bdc3ca20 -16:59.292 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c7e3a84e-ad2a-f51d-56c5-d906bdc3ca20 -16:59.293 [INFO] TestAgent.server: federation state anti-entropy synced -16:59.353 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:59.354 [INFO] TestAgent: Synced node info -16:59.539 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:52902 latency=1.010687ms -16:59.541 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/3eb67cfa-f4cc-67c8-8d47-50fcc0960d0b from=127.0.0.1:52904 latency=859.025µs -16:59.542 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/3eb67cfa-f4cc-67c8-8d47-50fcc0960d0b from=127.0.0.1:52902 latency=513.865µs -16:59.553 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/3eb67cfa-f4cc-67c8-8d47-50fcc0960d0b from=127.0.0.1:52906 latency=6.803078ms -16:59.558 [INFO] TestAgent: Requesting shutdown -16:59.558 [INFO] TestAgent.server: shutting down server -16:59.558 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:59.558 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:59.558 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:59.558 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:59.558 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:59.558 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:59.558 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:59.559 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:59.559 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:59.559 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:59.559 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:59.559Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:59.559Z [INFO] agent.router.manager: shutting down -16:59.559 [INFO] TestAgent: consul server down -16:59.559 [INFO] TestAgent: shutdown complete -16:59.559 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15120 network=tcp -16:59.559 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15120 network=udp -16:59.559 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15121 network=tcp -17:00.060 [INFO] TestAgent: Waiting for endpoints to shut down -17:00.060 [INFO] TestAgent: Endpoints down ---- PASS: TestExecCommand_Sessions (1.07s) -2021-10-31T23:16:59.156Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:59.289Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:59.346 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9441b836-09e2-eaad-63d7-6288a42fa48d Address:127.0.0.1:15153}]" -16:59.347 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9441b836-09e2-eaad-63d7-6288a42fa48d.dc1 127.0.0.1 -16:59.347 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9441b836-09e2-eaad-63d7-6288a42fa48d 127.0.0.1 -2021-10-31T23:16:59.347Z [INFO] agent.router: Initializing LAN area manager -16:59.348 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15148 network=udp -16:59.348 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15153 [Follower]" leader= -16:59.348 [INFO] TestAgent.server: Adding LAN server: server="Node-9441b836-09e2-eaad-63d7-6288a42fa48d (Addr: tcp/127.0.0.1:15153) (DC: dc1)" -16:59.348 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9441b836-09e2-eaad-63d7-6288a42fa48d.dc1 area=wan -16:59.410 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15148 network=tcp -16:59.411 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15149 network=tcp -16:59.411 [INFO] TestAgent: started state syncer -16:59.411 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15154 network=tcp -16:59.413 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:59.413 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15153 [Candidate]" term=2 -16:59.413 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:59.413 [DEBUG] TestAgent.server.raft: vote granted: from=9441b836-09e2-eaad-63d7-6288a42fa48d term=2 tally=1 -16:59.413 [INFO] TestAgent.server.raft: election won: tally=1 -16:59.413 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15153 [Leader]" -16:59.415 [INFO] TestAgent.server: cluster leadership acquired -16:59.415 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15153 -16:59.417 [INFO] TestAgent.server: New leader elected: payload=Node-9441b836-09e2-eaad-63d7-6288a42fa48d -16:59.437 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:59.453 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:59.453 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:59.453 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:59.453 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:59.453 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:59.453 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9441b836-09e2-eaad-63d7-6288a42fa48d -16:59.453 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9441b836-09e2-eaad-63d7-6288a42fa48d -16:59.454 [INFO] TestAgent.server: federation state anti-entropy synced -16:59.558 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:59.559 [INFO] TestAgent: Synced node info -16:59.559 [DEBUG] TestAgent: Node info in sync -16:59.652 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:52784 latency=1.923172ms -16:59.654 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/d6f707f9-b74a-c5a2-0e9b-1b6681c36c04/?keys=&wait=2000ms from=127.0.0.1:52784 latency=103.941µs -16:59.655 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/d6f707f9-b74a-c5a2-0e9b-1b6681c36c04/foo/ack?acquire=d6f707f9-b74a-c5a2-0e9b-1b6681c36c04 from=127.0.0.1:52786 latency=634.397µs -16:59.656 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/d6f707f9-b74a-c5a2-0e9b-1b6681c36c04/?index=1&keys=&wait=2000ms from=127.0.0.1:52784 latency=364.322µs -16:59.658 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/d6f707f9-b74a-c5a2-0e9b-1b6681c36c04/foo/exit?acquire=d6f707f9-b74a-c5a2-0e9b-1b6681c36c04 from=127.0.0.1:52788 latency=650.867µs -16:59.658 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/d6f707f9-b74a-c5a2-0e9b-1b6681c36c04/?index=14&keys=&wait=2000ms from=127.0.0.1:52784 latency=376.202µs -16:59.659 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/d6f707f9-b74a-c5a2-0e9b-1b6681c36c04/foo/exit from=127.0.0.1:52784 latency=172.272µs -16:59.660 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/d6f707f9-b74a-c5a2-0e9b-1b6681c36c04/foo/random?acquire=d6f707f9-b74a-c5a2-0e9b-1b6681c36c04 from=127.0.0.1:52790 latency=454.128µs -16:59.660 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/d6f707f9-b74a-c5a2-0e9b-1b6681c36c04/?index=15&keys=&wait=2000ms from=127.0.0.1:52784 latency=163.149µs -16:59.662 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/d6f707f9-b74a-c5a2-0e9b-1b6681c36c04/foo/out/00000?acquire=d6f707f9-b74a-c5a2-0e9b-1b6681c36c04 from=127.0.0.1:52792 latency=835.033µs -16:59.662 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/d6f707f9-b74a-c5a2-0e9b-1b6681c36c04/?index=16&keys=&wait=2000ms from=127.0.0.1:52784 latency=1.491227ms -16:59.670 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/d6f707f9-b74a-c5a2-0e9b-1b6681c36c04/foo/out/00000 from=127.0.0.1:52784 latency=301.525µs -16:59.672 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/d6f707f9-b74a-c5a2-0e9b-1b6681c36c04/foo/out/00001?acquire=d6f707f9-b74a-c5a2-0e9b-1b6681c36c04 from=127.0.0.1:52794 latency=536.623µs -16:59.672 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/d6f707f9-b74a-c5a2-0e9b-1b6681c36c04/?index=17&keys=&wait=2000ms from=127.0.0.1:52784 latency=1.540122ms -16:59.675 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/d6f707f9-b74a-c5a2-0e9b-1b6681c36c04/foo/out/00001 from=127.0.0.1:52784 latency=390.329µs -16:59.676 [INFO] TestAgent: Requesting shutdown -16:59.676 [INFO] TestAgent.server: shutting down server -16:59.676 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:59.676 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:59.676 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:59.676 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:59.676 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:59.676 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:59.676 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:59.676 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:59.676 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:59.676 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:59.676 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:59.676Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:59.676Z [INFO] agent.router.manager: shutting down -16:59.677 [INFO] TestAgent: consul server down -16:59.677 [INFO] TestAgent: shutdown complete -16:59.677 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15148 network=tcp -16:59.677 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15148 network=udp -16:59.677 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15149 network=tcp -17:00.178 [INFO] TestAgent: Waiting for endpoints to shut down -17:00.178 [INFO] TestAgent: Endpoints down ---- PASS: TestExecCommand_StreamResults (1.19s) -2021-10-31T23:16:59.160Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:59.294Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:59.319 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6e546b49-9e7f-72b0-c35d-183880e060aa Address:127.0.0.1:15118}]" -16:59.342 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6e546b49-9e7f-72b0-c35d-183880e060aa.dc1 127.0.0.1 -16:59.342 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6e546b49-9e7f-72b0-c35d-183880e060aa 127.0.0.1 -2021-10-31T23:16:59.342Z [INFO] agent.router: Initializing LAN area manager -16:59.343 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=udp -16:59.343 [INFO] TestAgent.server: Adding LAN server: server="Node-6e546b49-9e7f-72b0-c35d-183880e060aa (Addr: tcp/127.0.0.1:15118) (DC: dc1)" -16:59.343 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6e546b49-9e7f-72b0-c35d-183880e060aa.dc1 area=wan -16:59.343 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=tcp -16:59.343 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15114 network=tcp -16:59.343 [INFO] TestAgent: started state syncer -16:59.344 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15119 network=tcp -16:59.344 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15118 [Follower]" leader= -16:59.397 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:59.397 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15118 [Candidate]" term=2 -16:59.398 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:59.398 [DEBUG] TestAgent.server.raft: vote granted: from=6e546b49-9e7f-72b0-c35d-183880e060aa term=2 tally=1 -16:59.398 [INFO] TestAgent.server.raft: election won: tally=1 -16:59.398 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15118 [Leader]" -16:59.398 [INFO] TestAgent.server: cluster leadership acquired -16:59.398 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15118 -16:59.413 [INFO] TestAgent.server: New leader elected: payload=Node-6e546b49-9e7f-72b0-c35d-183880e060aa -16:59.414 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:59.435 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:59.435 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:59.435 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:59.435 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:59.435 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:59.435 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6e546b49-9e7f-72b0-c35d-183880e060aa -16:59.435 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6e546b49-9e7f-72b0-c35d-183880e060aa -16:59.454 [INFO] TestAgent.server: federation state anti-entropy synced -16:59.770 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:59.770 [INFO] TestAgent: Synced node info -16:59.775 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/consul?passing=1 from=127.0.0.1:42220 latency=1.119293ms -16:59.777 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:42220 latency=438.734µs -16:59.778 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/4f1b4fad-4094-3303-9e3e-90283e7a74b0 from=127.0.0.1:42222 latency=386.296µs -16:59.782 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/4f1b4fad-4094-3303-9e3e-90283e7a74b0 from=127.0.0.1:42220 latency=567.999µs -16:59.783 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/4f1b4fad-4094-3303-9e3e-90283e7a74b0 from=127.0.0.1:42224 latency=183.069µs -16:59.783 [INFO] TestAgent: Requesting shutdown -16:59.783 [INFO] TestAgent.server: shutting down server -16:59.783 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -16:59.783 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -16:59.783 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -16:59.783 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -16:59.783 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -16:59.784 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -16:59.784 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -16:59.784 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -16:59.784 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -16:59.784 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -16:59.784 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:16:59.784Z [INFO] agent.router.manager: shutting down -2021-10-31T23:16:59.784Z [INFO] agent.router.manager: shutting down -16:59.784 [INFO] TestAgent: consul server down -16:59.784 [INFO] TestAgent: shutdown complete -16:59.784 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=tcp -16:59.784 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=udp -16:59.784 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15114 network=tcp -17:00.285 [INFO] TestAgent: Waiting for endpoints to shut down -17:00.285 [INFO] TestAgent: Endpoints down ---- PASS: TestExecCommand_Sessions_Foreign (1.30s) -2021-10-31T23:16:59.172Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:59.295Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:59.311 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4fda79dd-0515-d0fc-be24-f0481eea4f42 Address:127.0.0.1:15146}]" -16:59.312 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15146 [Follower]" leader= -16:59.313 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4fda79dd-0515-d0fc-be24-f0481eea4f42.dc1 127.0.0.1 -16:59.318 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4fda79dd-0515-d0fc-be24-f0481eea4f42 127.0.0.1 -2021-10-31T23:16:59.319Z [INFO] agent.router: Initializing LAN area manager -16:59.320 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15141 network=udp -16:59.344 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15141 network=tcp -16:59.345 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15142 network=tcp -16:59.345 [INFO] TestAgent: started state syncer -16:59.345 [INFO] TestAgent.server: Adding LAN server: server="Node-4fda79dd-0515-d0fc-be24-f0481eea4f42 (Addr: tcp/127.0.0.1:15146) (DC: dc1)" -16:59.345 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15147 network=tcp -16:59.345 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4fda79dd-0515-d0fc-be24-f0481eea4f42.dc1 area=wan -16:59.378 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:59.378 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15146 [Candidate]" term=2 -16:59.378 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:59.378 [DEBUG] TestAgent.server.raft: vote granted: from=4fda79dd-0515-d0fc-be24-f0481eea4f42 term=2 tally=1 -16:59.378 [INFO] TestAgent.server.raft: election won: tally=1 -16:59.378 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15146 [Leader]" -16:59.378 [INFO] TestAgent.server: cluster leadership acquired -16:59.378 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15146 -16:59.378 [INFO] TestAgent.server: New leader elected: payload=Node-4fda79dd-0515-d0fc-be24-f0481eea4f42 -16:59.379 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:59.380 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:59.380 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:59.380 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:59.380 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:59.380 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:59.380 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4fda79dd-0515-d0fc-be24-f0481eea4f42 -16:59.380 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4fda79dd-0515-d0fc-be24-f0481eea4f42 -16:59.381 [INFO] TestAgent.server: federation state anti-entropy synced -16:59.575 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:33930 latency=1.681411ms -16:59.584 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:33930 latency=860.747µs -16:59.585 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/eb5a09a3-f76a-00f3-a1ae-67ee0625a9cd/job?acquire=eb5a09a3-f76a-00f3-a1ae-67ee0625a9cd from=127.0.0.1:33930 latency=396.014µs -16:59.605 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:59.605 [INFO] TestAgent: Synced node info -16:59.790 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/_rexec from=127.0.0.1:33930 latency=427.948µs -16:59.790 [DEBUG] TestAgent.server: User event: event=_rexec -16:59.790 [DEBUG] TestAgent: received remote exec event: id=607189c0-75da-4656-0cb4-6e1176a42213 -16:59.791 [INFO] TestAgent: remote exec script: script=uptime -16:59.798 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/eb5a09a3-f76a-00f3-a1ae-67ee0625a9cd/?keys=&wait=1000ms from=127.0.0.1:33930 latency=285.095µs -16:59.802 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/eb5a09a3-f76a-00f3-a1ae-67ee0625a9cd/?index=15&keys=&wait=1000ms from=127.0.0.1:33930 latency=1.901402ms -16:59.804 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/eb5a09a3-f76a-00f3-a1ae-67ee0625a9cd/Node-4fda79dd-0515-d0fc-be24-f0481eea4f42/exit from=127.0.0.1:33930 latency=227.948µs -16:59.805 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/eb5a09a3-f76a-00f3-a1ae-67ee0625a9cd/Node-4fda79dd-0515-d0fc-be24-f0481eea4f42/out/00000 from=127.0.0.1:33930 latency=150.916µs -17:00.621 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:00.621 [DEBUG] TestAgent: Node info in sync -17:00.621 [DEBUG] TestAgent: Node info in sync -17:00.811 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/eb5a09a3-f76a-00f3-a1ae-67ee0625a9cd from=127.0.0.1:33960 latency=610.651µs -17:00.811 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/eb5a09a3-f76a-00f3-a1ae-67ee0625a9cd/?index=17&keys=&wait=1000ms from=127.0.0.1:33930 latency=1.005409822s -17:00.811 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/_rexec/eb5a09a3-f76a-00f3-a1ae-67ee0625a9cd?recurse= from=127.0.0.1:33960 latency=190.283µs -17:00.812 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/eb5a09a3-f76a-00f3-a1ae-67ee0625a9cd from=127.0.0.1:33960 latency=162.464µs -17:00.812 [INFO] TestAgent: Requesting shutdown -17:00.812 [INFO] TestAgent.server: shutting down server -17:00.812 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:00.812 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:00.812 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:00.812 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:00.812 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:00.812 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:00.812 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:00.812 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:00.812 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:00.812 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:00.812 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:00.812Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:00.812Z [INFO] agent.router.manager: shutting down -17:00.812 [INFO] TestAgent: consul server down -17:00.812 [INFO] TestAgent: shutdown complete -17:00.812 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15141 network=tcp -17:00.812 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15141 network=udp -17:00.812 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15142 network=tcp -17:01.312 [INFO] TestAgent: Waiting for endpoints to shut down -17:01.312 [INFO] TestAgent: Endpoints down ---- PASS: TestExecCommand (2.32s) -2021-10-31T23:16:59.238Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:59.411Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:59.413 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e401a75e-df75-0aca-934d-c6747282ed9e Address:127.0.0.1:15139}]" -16:59.413 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15139 [Follower]" leader= -16:59.454 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e401a75e-df75-0aca-934d-c6747282ed9e.dc1 127.0.0.1 -16:59.454 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e401a75e-df75-0aca-934d-c6747282ed9e 127.0.0.1 -2021-10-31T23:16:59.454Z [INFO] agent.router: Initializing LAN area manager -16:59.455 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15134 network=udp -16:59.455 [INFO] TestAgent.server: Adding LAN server: server="Node-e401a75e-df75-0aca-934d-c6747282ed9e (Addr: tcp/127.0.0.1:15139) (DC: dc1)" -16:59.455 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e401a75e-df75-0aca-934d-c6747282ed9e.dc1 area=wan -16:59.455 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15134 network=tcp -16:59.456 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15135 network=tcp -16:59.456 [INFO] TestAgent: started state syncer -16:59.456 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15140 network=tcp -16:59.469 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:59.469 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15139 [Candidate]" term=2 -16:59.469 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:59.469 [DEBUG] TestAgent.server.raft: vote granted: from=e401a75e-df75-0aca-934d-c6747282ed9e term=2 tally=1 -16:59.469 [INFO] TestAgent.server.raft: election won: tally=1 -16:59.469 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15139 [Leader]" -16:59.469 [INFO] TestAgent.server: cluster leadership acquired -16:59.469 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15139 -16:59.469 [INFO] TestAgent.server: New leader elected: payload=Node-e401a75e-df75-0aca-934d-c6747282ed9e -16:59.470 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:59.472 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:59.472 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:59.472 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:59.472 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:59.472 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:59.472 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e401a75e-df75-0aca-934d-c6747282ed9e -16:59.472 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e401a75e-df75-0aca-934d-c6747282ed9e -16:59.473 [INFO] TestAgent.server: federation state anti-entropy synced -16:59.506 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:59.506 [INFO] TestAgent: Synced node info -16:59.627 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:43750 latency=1.164166ms -16:59.638 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:43750 latency=792.681µs -16:59.643 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/2d979083-4355-ebe0-be7e-396d7debfb6a/job?acquire=2d979083-4355-ebe0-be7e-396d7debfb6a from=127.0.0.1:43750 latency=657.229µs -16:59.845 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/_rexec from=127.0.0.1:43750 latency=461.662µs -16:59.845 [DEBUG] TestAgent.server: User event: event=_rexec -16:59.845 [DEBUG] TestAgent: received remote exec event: id=45ae7c48-4c9b-8e7d-baf8-e6f32078fc4f -16:59.846 [INFO] TestAgent: remote exec script: script= -16:59.847 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/2d979083-4355-ebe0-be7e-396d7debfb6a/?keys=&wait=1000ms from=127.0.0.1:43750 latency=271.644µs -16:59.858 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/2d979083-4355-ebe0-be7e-396d7debfb6a/?index=15&keys=&wait=1000ms from=127.0.0.1:43750 latency=11.12214ms -16:59.862 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/2d979083-4355-ebe0-be7e-396d7debfb6a/Node-e401a75e-df75-0aca-934d-c6747282ed9e/exit from=127.0.0.1:43750 latency=284.793µs -16:59.863 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/2d979083-4355-ebe0-be7e-396d7debfb6a/Node-e401a75e-df75-0aca-934d-c6747282ed9e/out/00000 from=127.0.0.1:43750 latency=161.807µs -17:00.682 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:00.682 [DEBUG] TestAgent: Node info in sync -17:00.682 [DEBUG] TestAgent: Node info in sync -17:00.871 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/2d979083-4355-ebe0-be7e-396d7debfb6a from=127.0.0.1:43780 latency=897.014µs -17:00.872 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/2d979083-4355-ebe0-be7e-396d7debfb6a/?index=17&keys=&wait=1000ms from=127.0.0.1:43750 latency=1.00857396s -17:00.888 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/_rexec/2d979083-4355-ebe0-be7e-396d7debfb6a?recurse= from=127.0.0.1:43780 latency=628.76µs -17:00.895 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/2d979083-4355-ebe0-be7e-396d7debfb6a from=127.0.0.1:43780 latency=618.443µs -17:00.895 [INFO] TestAgent: Requesting shutdown -17:00.895 [INFO] TestAgent.server: shutting down server -17:00.895 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:00.895 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:00.895 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:00.895 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:00.895 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:00.896 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:00.896 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:00.896 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:00.896 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:00.896 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:00.896 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:00.896Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:00.896Z [INFO] agent.router.manager: shutting down -17:00.896 [INFO] TestAgent: consul server down -17:00.896 [INFO] TestAgent: shutdown complete -17:00.896 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15134 network=tcp -17:00.896 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15134 network=udp -17:00.896 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15135 network=tcp -17:01.397 [INFO] TestAgent: Waiting for endpoints to shut down -17:01.397 [INFO] TestAgent: Endpoints down ---- PASS: TestExecCommand_NoShell (2.41s) -2021-10-31T23:16:59.750Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:59.782Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:59.786 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:83ff2c8f-98c7-c993-0bf1-e702bda206b9 Address:127.0.0.1:15160}]" -16:59.787 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-83ff2c8f-98c7-c993-0bf1-e702bda206b9.dc2 127.0.0.1 -16:59.787 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-83ff2c8f-98c7-c993-0bf1-e702bda206b9 127.0.0.1 -2021-10-31T23:16:59.787Z [INFO] agent.router: Initializing LAN area manager -16:59.788 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15160 [Follower]" leader= -16:59.788 [INFO] TestAgent.server: Adding LAN server: server="Node-83ff2c8f-98c7-c993-0bf1-e702bda206b9 (Addr: tcp/127.0.0.1:15160) (DC: dc2)" -16:59.788 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-83ff2c8f-98c7-c993-0bf1-e702bda206b9.dc2 area=wan -16:59.788 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15155 network=tcp -16:59.788 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15155 network=udp -16:59.789 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15156 network=tcp -16:59.789 [INFO] TestAgent: started state syncer -16:59.789 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15161 network=tcp -16:59.856 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:59.856 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15160 [Candidate]" term=2 -16:59.856 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:59.856 [DEBUG] TestAgent.server.raft: vote granted: from=83ff2c8f-98c7-c993-0bf1-e702bda206b9 term=2 tally=1 -16:59.856 [INFO] TestAgent.server.raft: election won: tally=1 -16:59.856 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15160 [Leader]" -16:59.856 [INFO] TestAgent.server: cluster leadership acquired -16:59.857 [INFO] TestAgent.server: New leader elected: payload=Node-83ff2c8f-98c7-c993-0bf1-e702bda206b9 -16:59.857 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15160 -16:59.858 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:59.860 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:59.860 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:59.860 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:59.860 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:59.860 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:59.860 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-83ff2c8f-98c7-c993-0bf1-e702bda206b9 -16:59.860 [INFO] TestAgent.server: member joined, marking health alive: member=Node-83ff2c8f-98c7-c993-0bf1-e702bda206b9 -16:59.860 [INFO] TestAgent.server: federation state anti-entropy synced -16:59.960 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:15131] -16:59.962 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:15131 -16:59.968 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e2d587cd-07b9-2540-3b3c-ba1f0651cb0a.dc1 127.0.0.1 -16:59.969 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -16:59.969 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e2d587cd-07b9-2540-3b3c-ba1f0651cb0a.dc1 area=wan -17:00.078 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:00.079 [INFO] TestAgent: Synced node info -17:00.198 [DEBUG] TestAgent.server: User event: event=_rexec -17:00.198 [DEBUG] TestAgent: received remote exec event: id=074d237d-fd99-3bcf-a7c2-0d2bde6c300d -17:00.199 [INFO] TestAgent: remote exec script: script=uptime -17:00.723 [INFO] TestAgent: Requesting shutdown -17:00.723 [INFO] TestAgent.server: shutting down server -17:00.723 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:00.723 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:00.723 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:00.723 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:00.723 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:00.723 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:00.723 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:00.723 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:00.723 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:00.723 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:00.723 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:00.723Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:00.723Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:00.723Z [INFO] agent.router.manager: shutting down -17:00.723 [INFO] TestAgent: consul server down -17:00.723 [INFO] TestAgent: shutdown complete -17:00.723 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15155 network=tcp -17:00.723 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15155 network=udp -17:00.723 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15156 network=tcp -17:01.225 [INFO] TestAgent: Waiting for endpoints to shut down -17:01.225 [INFO] TestAgent: Endpoints down -2021-10-31T23:16:59.159Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:16:59.237Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -16:59.239 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e2d587cd-07b9-2540-3b3c-ba1f0651cb0a Address:127.0.0.1:15132}]" -16:59.254 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e2d587cd-07b9-2540-3b3c-ba1f0651cb0a.dc1 127.0.0.1 -16:59.255 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e2d587cd-07b9-2540-3b3c-ba1f0651cb0a 127.0.0.1 -2021-10-31T23:16:59.255Z [INFO] agent.router: Initializing LAN area manager -16:59.255 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15127 network=udp -16:59.256 [INFO] TestAgent.server: Adding LAN server: server="Node-e2d587cd-07b9-2540-3b3c-ba1f0651cb0a (Addr: tcp/127.0.0.1:15132) (DC: dc1)" -16:59.256 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e2d587cd-07b9-2540-3b3c-ba1f0651cb0a.dc1 area=wan -16:59.256 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15127 network=tcp -16:59.257 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15128 network=tcp -16:59.257 [INFO] TestAgent: started state syncer -16:59.257 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15133 network=tcp -16:59.260 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15132 [Follower]" leader= -16:59.348 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -16:59.348 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15132 [Candidate]" term=2 -16:59.348 [DEBUG] TestAgent.server.raft: votes: needed=1 -16:59.348 [DEBUG] TestAgent.server.raft: vote granted: from=e2d587cd-07b9-2540-3b3c-ba1f0651cb0a term=2 tally=1 -16:59.348 [INFO] TestAgent.server.raft: election won: tally=1 -16:59.348 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15132 [Leader]" -16:59.349 [INFO] TestAgent.server: cluster leadership acquired -16:59.350 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15132 -16:59.351 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -16:59.352 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -16:59.352 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -16:59.352 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -16:59.352 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -16:59.352 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -16:59.352 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e2d587cd-07b9-2540-3b3c-ba1f0651cb0a -16:59.352 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e2d587cd-07b9-2540-3b3c-ba1f0651cb0a -16:59.353 [INFO] TestAgent.server: New leader elected: payload=Node-e2d587cd-07b9-2540-3b3c-ba1f0651cb0a -16:59.353 [INFO] TestAgent.server: federation state anti-entropy synced -16:59.630 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:59.630 [INFO] TestAgent: Synced node info -16:59.717 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -16:59.717 [DEBUG] TestAgent: Node info in sync -16:59.717 [DEBUG] TestAgent: Node info in sync -16:59.963 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:37146 -16:59.963 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-83ff2c8f-98c7-c993-0bf1-e702bda206b9.dc2 127.0.0.1 -16:59.963 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-83ff2c8f-98c7-c993-0bf1-e702bda206b9.dc2 area=wan -16:59.980 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self?dc=dc2 from=127.0.0.1:48184 latency=1.483114ms -16:59.986 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/consul?dc=dc2&passing=1 from=127.0.0.1:48184 latency=3.662963ms -16:59.993 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create?dc=dc2 from=127.0.0.1:48184 latency=5.909531ms -16:59.994 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/b0e5df19-ce7a-f846-6539-e4621fd2811f/job?acquire=b0e5df19-ce7a-f846-6539-e4621fd2811f&dc=dc2 from=127.0.0.1:48184 latency=705.061µs -17:00.199 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/_rexec?dc=dc2 from=127.0.0.1:48184 latency=1.049384ms -17:00.210 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/b0e5df19-ce7a-f846-6539-e4621fd2811f/?dc=dc2&keys=&wait=500ms from=127.0.0.1:48184 latency=5.975605ms -17:00.211 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/b0e5df19-ce7a-f846-6539-e4621fd2811f/Node-83ff2c8f-98c7-c993-0bf1-e702bda206b9/exit?dc=dc2 from=127.0.0.1:48184 latency=351.233µs -17:00.212 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/b0e5df19-ce7a-f846-6539-e4621fd2811f/Node-83ff2c8f-98c7-c993-0bf1-e702bda206b9/out/00000?dc=dc2 from=127.0.0.1:48184 latency=303.64µs -17:00.294 [DEBUG] TestAgent.server.serf.wan: serf: messageJoinType: Node-83ff2c8f-98c7-c993-0bf1-e702bda206b9.dc2 -17:00.716 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/b0e5df19-ce7a-f846-6539-e4621fd2811f/?dc=dc2&index=17&keys=&wait=500ms from=127.0.0.1:48184 latency=504.207561ms -17:00.717 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/b0e5df19-ce7a-f846-6539-e4621fd2811f?dc=dc2 from=127.0.0.1:48186 latency=1.306361ms -17:00.718 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/_rexec/b0e5df19-ce7a-f846-6539-e4621fd2811f?dc=dc2&recurse= from=127.0.0.1:48184 latency=940.71µs -17:00.723 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/b0e5df19-ce7a-f846-6539-e4621fd2811f?dc=dc2 from=127.0.0.1:48184 latency=542.122µs -17:01.225 [INFO] TestAgent: Requesting shutdown -17:01.225 [INFO] TestAgent.server: shutting down server -17:01.225 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:01.225 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:01.226 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:01.226 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:01.226 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:01.225 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:01.226 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:01.226 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:01.226 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:01.226 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:01.226Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:01.226Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:01.226Z [INFO] agent.router.manager: shutting down -17:01.226 [INFO] TestAgent: consul server down -17:01.226 [INFO] TestAgent: shutdown complete -17:01.226 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15127 network=tcp -17:01.226 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15127 network=udp -17:01.226 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15128 network=tcp -17:01.727 [INFO] TestAgent: Waiting for endpoints to shut down -17:01.727 [INFO] TestAgent: Endpoints down ---- PASS: TestExecCommand_CrossDC (2.74s) +2022-12-04T05:49:56.538Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:56.586Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:56.621 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ce31e801-fcf1-298a-daa6-f7a79c3f1a9f Address:127.0.0.1:31454}]" +49:56.621 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31454 [Follower]" leader= +49:56.622 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ce31e801-fcf1-298a-daa6-f7a79c3f1a9f.dc1 127.0.0.1 +49:56.623 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ce31e801-fcf1-298a-daa6-f7a79c3f1a9f 127.0.0.1 +2022-12-04T05:49:56.623Z [INFO] agent.router: Initializing LAN area manager +49:56.624 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ce31e801-fcf1-298a-daa6-f7a79c3f1a9f.dc1 area=wan +49:56.624 [INFO] TestAgent.server: Adding LAN server: server="Node-ce31e801-fcf1-298a-daa6-f7a79c3f1a9f (Addr: tcp/127.0.0.1:31454) (DC: dc1)" +49:56.652 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31449 network=tcp +49:56.652 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31449 network=udp +49:56.653 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31450 network=tcp +49:56.653 [INFO] TestAgent: started state syncer +49:56.653 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31455 network=tcp +49:56.688 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:56.688 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31454 [Candidate]" term=2 +49:56.689 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:56.689 [DEBUG] TestAgent.server.raft: vote granted: from=ce31e801-fcf1-298a-daa6-f7a79c3f1a9f term=2 tally=1 +49:56.689 [INFO] TestAgent.server.raft: election won: tally=1 +49:56.689 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31454 [Leader]" +49:56.689 [INFO] TestAgent.server: cluster leadership acquired +49:56.689 [INFO] TestAgent.server: New leader elected: payload=Node-ce31e801-fcf1-298a-daa6-f7a79c3f1a9f +49:56.689 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31454 +49:56.690 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:56.693 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:56.693 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:56.693 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:56.693 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:56.693 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:56.693 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ce31e801-fcf1-298a-daa6-f7a79c3f1a9f +49:56.693 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ce31e801-fcf1-298a-daa6-f7a79c3f1a9f +49:56.695 [INFO] TestAgent.server: federation state anti-entropy synced +49:56.868 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:56.868 [INFO] TestAgent: Synced node info +49:56.868 [DEBUG] TestAgent: Node info in sync +49:56.893 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/consul?passing=1 from=127.0.0.1:33950 latency=1.805747ms +49:56.895 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:33950 latency=550.896µs +49:56.896 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/e6113c40-8d71-29a9-c69b-61d4492f3cff from=127.0.0.1:33952 latency=299.308µs +49:56.897 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/e6113c40-8d71-29a9-c69b-61d4492f3cff from=127.0.0.1:33950 latency=391.396µs +49:56.898 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/e6113c40-8d71-29a9-c69b-61d4492f3cff from=127.0.0.1:33954 latency=134.21µs +49:56.898 [INFO] TestAgent: Requesting shutdown +49:56.898 [INFO] TestAgent.server: shutting down server +49:56.898 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:56.898 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:56.898 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:56.898 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:56.898 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:56.898 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:56.898 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:56.898 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:56.898 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:56.898 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:56.898 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:56.899Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:56.899Z [INFO] agent.router.manager: shutting down +49:56.899 [INFO] TestAgent: consul server down +49:56.899 [INFO] TestAgent: shutdown complete +49:56.899 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31449 network=tcp +49:56.899 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31449 network=udp +49:56.899 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31450 network=tcp +49:57.399 [INFO] TestAgent: Waiting for endpoints to shut down +49:57.399 [INFO] TestAgent: Endpoints down +--- PASS: TestExecCommand_Sessions_Foreign (0.92s) +2022-12-04T05:49:56.540Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:56.564Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:56.569 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0d918b84-b77d-34fd-0b87-95557bfca261 Address:127.0.0.1:31447}]" +49:56.570 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0d918b84-b77d-34fd-0b87-95557bfca261.dc1 127.0.0.1 +49:56.571 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0d918b84-b77d-34fd-0b87-95557bfca261 127.0.0.1 +2022-12-04T05:49:56.571Z [INFO] agent.router: Initializing LAN area manager +49:56.571 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=udp +49:56.571 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= +49:56.571 [INFO] TestAgent.server: Adding LAN server: server="Node-0d918b84-b77d-34fd-0b87-95557bfca261 (Addr: tcp/127.0.0.1:31447) (DC: dc1)" +49:56.571 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0d918b84-b77d-34fd-0b87-95557bfca261.dc1 area=wan +49:56.571 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=tcp +49:56.572 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31443 network=tcp +49:56.572 [INFO] TestAgent: started state syncer +49:56.572 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31448 network=tcp +49:56.622 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:56.622 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 +49:56.623 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:56.623 [DEBUG] TestAgent.server.raft: vote granted: from=0d918b84-b77d-34fd-0b87-95557bfca261 term=2 tally=1 +49:56.623 [INFO] TestAgent.server.raft: election won: tally=1 +49:56.623 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" +49:56.623 [INFO] TestAgent.server: cluster leadership acquired +49:56.623 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31447 +49:56.624 [INFO] TestAgent.server: New leader elected: payload=Node-0d918b84-b77d-34fd-0b87-95557bfca261 +49:56.625 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:56.650 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:56.650 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:56.650 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:56.650 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:56.650 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:56.650 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d918b84-b77d-34fd-0b87-95557bfca261 +49:56.650 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0d918b84-b77d-34fd-0b87-95557bfca261 +49:56.652 [INFO] TestAgent.server: federation state anti-entropy synced +49:56.799 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:56.799 [INFO] TestAgent: Synced node info +49:56.799 [DEBUG] TestAgent: Node info in sync +49:56.969 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:36800 latency=528.631µs +49:56.970 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/b4461d19-9acf-ab5e-f381-d547f34f0d35 from=127.0.0.1:36804 latency=108.807µs +49:56.970 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/b4461d19-9acf-ab5e-f381-d547f34f0d35 from=127.0.0.1:36800 latency=384.517µs +49:56.971 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/session/info/b4461d19-9acf-ab5e-f381-d547f34f0d35 from=127.0.0.1:36806 latency=121.027µs +49:56.971 [INFO] TestAgent: Requesting shutdown +49:56.971 [INFO] TestAgent.server: shutting down server +49:56.971 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:56.971 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:56.971 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:56.971 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:56.971 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:56.971 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:56.971 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:56.971 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:56.971 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:56.971 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:56.971 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:56.971Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:56.971Z [INFO] agent.router.manager: shutting down +49:56.972 [INFO] TestAgent: consul server down +49:56.972 [INFO] TestAgent: shutdown complete +49:56.972 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp +49:56.972 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp +49:56.972 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp +49:57.472 [INFO] TestAgent: Waiting for endpoints to shut down +49:57.472 [INFO] TestAgent: Endpoints down +--- PASS: TestExecCommand_Sessions (1.00s) +2022-12-04T05:49:56.547Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:56.604Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:56.607 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2943378c-5def-595e-8e61-2137ef2212ea Address:127.0.0.1:31461}]" +49:56.608 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2943378c-5def-595e-8e61-2137ef2212ea.dc1 127.0.0.1 +49:56.608 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31461 [Follower]" leader= +49:56.614 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2943378c-5def-595e-8e61-2137ef2212ea 127.0.0.1 +2022-12-04T05:49:56.614Z [INFO] agent.router: Initializing LAN area manager +49:56.614 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2943378c-5def-595e-8e61-2137ef2212ea.dc1 area=wan +49:56.614 [INFO] TestAgent.server: Adding LAN server: server="Node-2943378c-5def-595e-8e61-2137ef2212ea (Addr: tcp/127.0.0.1:31461) (DC: dc1)" +49:56.614 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31456 network=udp +49:56.614 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31456 network=tcp +49:56.615 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31457 network=tcp +49:56.615 [INFO] TestAgent: started state syncer +49:56.615 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31462 network=tcp +49:56.671 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:56.671 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31461 [Candidate]" term=2 +49:56.672 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:56.672 [DEBUG] TestAgent.server.raft: vote granted: from=2943378c-5def-595e-8e61-2137ef2212ea term=2 tally=1 +49:56.672 [INFO] TestAgent.server.raft: election won: tally=1 +49:56.672 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31461 [Leader]" +49:56.672 [INFO] TestAgent.server: cluster leadership acquired +49:56.672 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31461 +49:56.672 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:56.673 [INFO] TestAgent.server: New leader elected: payload=Node-2943378c-5def-595e-8e61-2137ef2212ea +49:56.674 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:56.674 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:56.674 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:56.674 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:56.674 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:56.674 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2943378c-5def-595e-8e61-2137ef2212ea +49:56.674 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2943378c-5def-595e-8e61-2137ef2212ea +49:56.674 [INFO] TestAgent.server: federation state anti-entropy synced +49:56.974 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:57856 latency=541.284µs +49:56.975 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/346acaf1-a090-c4fc-e1ff-7bbeb29ae503/?keys=&wait=2000ms from=127.0.0.1:57856 latency=135.417µs +49:56.976 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/346acaf1-a090-c4fc-e1ff-7bbeb29ae503/foo/ack?acquire=346acaf1-a090-c4fc-e1ff-7bbeb29ae503 from=127.0.0.1:57860 latency=765.848µs +49:56.980 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/346acaf1-a090-c4fc-e1ff-7bbeb29ae503/?index=1&keys=&wait=2000ms from=127.0.0.1:57856 latency=1.053523ms +49:56.982 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/346acaf1-a090-c4fc-e1ff-7bbeb29ae503/foo/exit?acquire=346acaf1-a090-c4fc-e1ff-7bbeb29ae503 from=127.0.0.1:57862 latency=509.126µs +49:56.983 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/346acaf1-a090-c4fc-e1ff-7bbeb29ae503/?index=13&keys=&wait=2000ms from=127.0.0.1:57856 latency=1.221649ms +49:56.983 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/346acaf1-a090-c4fc-e1ff-7bbeb29ae503/foo/exit from=127.0.0.1:57856 latency=224.959µs +49:56.985 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/346acaf1-a090-c4fc-e1ff-7bbeb29ae503/?index=14&keys=&wait=2000ms from=127.0.0.1:57856 latency=487.473µs +49:56.985 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/346acaf1-a090-c4fc-e1ff-7bbeb29ae503/foo/random?acquire=346acaf1-a090-c4fc-e1ff-7bbeb29ae503 from=127.0.0.1:57864 latency=474.097µs +49:56.986 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/346acaf1-a090-c4fc-e1ff-7bbeb29ae503/?index=15&keys=&wait=2000ms from=127.0.0.1:57856 latency=809.608µs +49:56.986 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/346acaf1-a090-c4fc-e1ff-7bbeb29ae503/foo/out/00000 from=127.0.0.1:57856 latency=114.252µs +49:56.986 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/346acaf1-a090-c4fc-e1ff-7bbeb29ae503/foo/out/00000?acquire=346acaf1-a090-c4fc-e1ff-7bbeb29ae503 from=127.0.0.1:57866 latency=1.216781ms +49:56.987 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/346acaf1-a090-c4fc-e1ff-7bbeb29ae503/foo/out/00001?acquire=346acaf1-a090-c4fc-e1ff-7bbeb29ae503 from=127.0.0.1:57868 latency=423.47µs +49:56.988 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/346acaf1-a090-c4fc-e1ff-7bbeb29ae503/?index=16&keys=&wait=2000ms from=127.0.0.1:57856 latency=1.166048ms +49:56.988 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/346acaf1-a090-c4fc-e1ff-7bbeb29ae503/foo/out/00001 from=127.0.0.1:57856 latency=127.68µs +49:56.988 [INFO] TestAgent: Requesting shutdown +49:56.988 [INFO] TestAgent.server: shutting down server +49:56.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:56.988 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:56.989 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:56.989 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:56.989 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:56.989 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:56.989 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:56.989 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:56.989 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:56.989 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:56.989 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:56.989 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:56.989Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:56.989Z [INFO] agent.router.manager: shutting down +49:56.989 [INFO] TestAgent: consul server down +49:56.989 [INFO] TestAgent: shutdown complete +49:56.989 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31456 network=tcp +49:56.989 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31456 network=udp +49:56.989 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31457 network=tcp +49:56.989 [ERROR] TestAgent.http: Request error: method=GET url=/v1/kv/_rexec/346acaf1-a090-c4fc-e1ff-7bbeb29ae503/?index=17&keys=&wait=2000ms from=127.0.0.1:57856 error="No cluster leader" +49:56.989 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/346acaf1-a090-c4fc-e1ff-7bbeb29ae503/?index=17&keys=&wait=2000ms from=127.0.0.1:57856 latency=203.163µs +49:57.489 [INFO] TestAgent: Waiting for endpoints to shut down +49:57.489 [INFO] TestAgent: Endpoints down +--- PASS: TestExecCommand_StreamResults (1.01s) +2022-12-04T05:49:56.541Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:56.572Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:56.585 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b0434cb0-d29e-27dd-4f1c-0d86f81235e7 Address:127.0.0.1:31475}]" +49:56.585 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31475 [Follower]" leader= +49:56.586 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b0434cb0-d29e-27dd-4f1c-0d86f81235e7.dc1 127.0.0.1 +49:56.587 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b0434cb0-d29e-27dd-4f1c-0d86f81235e7 127.0.0.1 +2022-12-04T05:49:56.587Z [INFO] agent.router: Initializing LAN area manager +49:56.588 [INFO] TestAgent.server: Adding LAN server: server="Node-b0434cb0-d29e-27dd-4f1c-0d86f81235e7 (Addr: tcp/127.0.0.1:31475) (DC: dc1)" +49:56.588 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b0434cb0-d29e-27dd-4f1c-0d86f81235e7.dc1 area=wan +49:56.588 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31470 network=udp +49:56.589 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31470 network=tcp +49:56.609 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31471 network=tcp +49:56.609 [INFO] TestAgent: started state syncer +49:56.610 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31476 network=tcp +49:56.646 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:56.646 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31475 [Candidate]" term=2 +49:56.646 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:56.646 [DEBUG] TestAgent.server.raft: vote granted: from=b0434cb0-d29e-27dd-4f1c-0d86f81235e7 term=2 tally=1 +49:56.646 [INFO] TestAgent.server.raft: election won: tally=1 +49:56.646 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31475 [Leader]" +49:56.646 [INFO] TestAgent.server: cluster leadership acquired +49:56.647 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31475 +49:56.647 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:56.647 [INFO] TestAgent.server: New leader elected: payload=Node-b0434cb0-d29e-27dd-4f1c-0d86f81235e7 +49:56.652 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:56.652 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:56.652 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:56.652 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:56.652 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:56.652 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b0434cb0-d29e-27dd-4f1c-0d86f81235e7 +49:56.652 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b0434cb0-d29e-27dd-4f1c-0d86f81235e7 +49:56.653 [INFO] TestAgent.server: federation state anti-entropy synced +49:56.947 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:56.947 [INFO] TestAgent: Synced node info +49:56.947 [DEBUG] TestAgent: Node info in sync +49:57.052 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:59656 latency=563.15µs +49:57.053 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/d28a1f07-fda8-8a0d-7860-23c535b65e12/job?acquire=d28a1f07-fda8-8a0d-7860-23c535b65e12 from=127.0.0.1:59656 latency=419.381µs +49:57.054 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/d28a1f07-fda8-8a0d-7860-23c535b65e12/job from=127.0.0.1:59658 latency=247.939µs +49:57.054 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/_rexec/d28a1f07-fda8-8a0d-7860-23c535b65e12?recurse= from=127.0.0.1:59656 latency=413.71µs +49:57.056 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/d28a1f07-fda8-8a0d-7860-23c535b65e12/job from=127.0.0.1:59660 latency=43.904µs +49:57.056 [INFO] TestAgent: Requesting shutdown +49:57.056 [INFO] TestAgent.server: shutting down server +49:57.056 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:57.056 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:57.056 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:57.056 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:57.056 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:57.056 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:57.056 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:57.056 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:57.056 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:57.056 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:57.056 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:57.056Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:57.056Z [INFO] agent.router.manager: shutting down +49:57.057 [INFO] TestAgent: consul server down +49:57.057 [INFO] TestAgent: shutdown complete +49:57.057 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31470 network=tcp +49:57.057 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31470 network=udp +49:57.057 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31471 network=tcp +49:57.557 [INFO] TestAgent: Waiting for endpoints to shut down +49:57.557 [INFO] TestAgent: Endpoints down +--- PASS: TestExecCommand_UploadDestroy (1.08s) +2022-12-04T05:49:56.561Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:56.657Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:56.658 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:172e13be-5e7f-256c-f290-fee038c99214 Address:127.0.0.1:31482}]" +49:56.658 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31482 [Follower]" leader= +49:56.658 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-172e13be-5e7f-256c-f290-fee038c99214.dc1 127.0.0.1 +49:56.659 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-172e13be-5e7f-256c-f290-fee038c99214 127.0.0.1 +2022-12-04T05:49:56.659Z [INFO] agent.router: Initializing LAN area manager +49:56.659 [INFO] TestAgent.server: Adding LAN server: server="Node-172e13be-5e7f-256c-f290-fee038c99214 (Addr: tcp/127.0.0.1:31482) (DC: dc1)" +49:56.659 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-172e13be-5e7f-256c-f290-fee038c99214.dc1 area=wan +49:56.659 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31477 network=udp +49:56.659 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31477 network=tcp +49:56.660 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31478 network=tcp +49:56.660 [INFO] TestAgent: started state syncer +49:56.660 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31483 network=tcp +49:56.695 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:56.704 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31482 [Candidate]" term=2 +49:56.705 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:56.705 [DEBUG] TestAgent.server.raft: vote granted: from=172e13be-5e7f-256c-f290-fee038c99214 term=2 tally=1 +49:56.705 [INFO] TestAgent.server.raft: election won: tally=1 +49:56.705 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31482 [Leader]" +49:56.705 [INFO] TestAgent.server: cluster leadership acquired +49:56.705 [INFO] TestAgent.server: New leader elected: payload=Node-172e13be-5e7f-256c-f290-fee038c99214 +49:56.705 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31482 +49:56.706 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:56.708 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:56.708 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:56.708 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:56.708 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:56.708 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:56.708 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-172e13be-5e7f-256c-f290-fee038c99214 +49:56.708 [INFO] TestAgent.server: member joined, marking health alive: member=Node-172e13be-5e7f-256c-f290-fee038c99214 +49:56.708 [INFO] TestAgent.server: federation state anti-entropy synced +49:56.818 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:41230 latency=1.367025ms +49:56.821 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:41230 latency=844.227µs +49:56.822 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/afc98bab-d46a-cad5-f10a-e9f0c4006488/job?acquire=afc98bab-d46a-cad5-f10a-e9f0c4006488 from=127.0.0.1:41230 latency=441.118µs +49:56.992 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:56.992 [INFO] TestAgent: Synced node info +49:57.023 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/_rexec from=127.0.0.1:41230 latency=441.876µs +49:57.023 [DEBUG] TestAgent.server: User event: event=_rexec +49:57.023 [DEBUG] TestAgent: received remote exec event: id=5c1bbcb8-0465-fe7b-afc1-6cd4ac80ab85 +49:57.023 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/afc98bab-d46a-cad5-f10a-e9f0c4006488/?keys=&wait=1000ms from=127.0.0.1:41230 latency=234.736µs +49:57.024 [INFO] TestAgent: remote exec script: script=uptime +49:57.024 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/afc98bab-d46a-cad5-f10a-e9f0c4006488/?index=13&keys=&wait=1000ms from=127.0.0.1:41230 latency=720.195µs +49:57.029 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/afc98bab-d46a-cad5-f10a-e9f0c4006488/?index=15&keys=&wait=1000ms from=127.0.0.1:41230 latency=4.265478ms +49:57.030 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/afc98bab-d46a-cad5-f10a-e9f0c4006488/Node-172e13be-5e7f-256c-f290-fee038c99214/out/00000 from=127.0.0.1:41230 latency=225.032µs +49:57.030 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/afc98bab-d46a-cad5-f10a-e9f0c4006488/?index=16&keys=&wait=1000ms from=127.0.0.1:41230 latency=235.385µs +49:57.031 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/afc98bab-d46a-cad5-f10a-e9f0c4006488/Node-172e13be-5e7f-256c-f290-fee038c99214/exit from=127.0.0.1:41230 latency=176.775µs +49:58.033 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/afc98bab-d46a-cad5-f10a-e9f0c4006488/?index=17&keys=&wait=1000ms from=127.0.0.1:41230 latency=1.001385963s +49:58.033 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/afc98bab-d46a-cad5-f10a-e9f0c4006488 from=127.0.0.1:41232 latency=730.815µs +49:58.034 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/_rexec/afc98bab-d46a-cad5-f10a-e9f0c4006488?recurse= from=127.0.0.1:41232 latency=384.644µs +49:58.034 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/afc98bab-d46a-cad5-f10a-e9f0c4006488 from=127.0.0.1:41232 latency=219.096µs +49:58.034 [INFO] TestAgent: Requesting shutdown +49:58.034 [INFO] TestAgent.server: shutting down server +49:58.034 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:58.034 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:58.034 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:58.034 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:58.034 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:58.034 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:58.034 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:58.034 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:58.035 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:58.035 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:58.035 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:58.035Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:58.035Z [INFO] agent.router.manager: shutting down +49:58.035 [INFO] TestAgent: consul server down +49:58.035 [INFO] TestAgent: shutdown complete +49:58.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31477 network=tcp +49:58.035 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31477 network=udp +49:58.035 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31478 network=tcp +49:58.535 [INFO] TestAgent: Waiting for endpoints to shut down +49:58.535 [INFO] TestAgent: Endpoints down +--- PASS: TestExecCommand (2.06s) +2022-12-04T05:49:56.541Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:56.621Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:56.653 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5bd56560-7ee1-2bcc-640f-7150f0d18185 Address:127.0.0.1:31468}]" +49:56.653 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31468 [Follower]" leader= +49:56.653 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5bd56560-7ee1-2bcc-640f-7150f0d18185.dc1 127.0.0.1 +49:56.654 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5bd56560-7ee1-2bcc-640f-7150f0d18185 127.0.0.1 +2022-12-04T05:49:56.654Z [INFO] agent.router: Initializing LAN area manager +49:56.654 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31463 network=udp +49:56.654 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5bd56560-7ee1-2bcc-640f-7150f0d18185.dc1 area=wan +49:56.654 [INFO] TestAgent.server: Adding LAN server: server="Node-5bd56560-7ee1-2bcc-640f-7150f0d18185 (Addr: tcp/127.0.0.1:31468) (DC: dc1)" +49:56.654 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31463 network=tcp +49:56.655 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31464 network=tcp +49:56.655 [INFO] TestAgent: started state syncer +49:56.655 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31469 network=tcp +49:56.692 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:56.692 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31468 [Candidate]" term=2 +49:56.692 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:56.692 [DEBUG] TestAgent.server.raft: vote granted: from=5bd56560-7ee1-2bcc-640f-7150f0d18185 term=2 tally=1 +49:56.692 [INFO] TestAgent.server.raft: election won: tally=1 +49:56.692 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31468 [Leader]" +49:56.692 [INFO] TestAgent.server: cluster leadership acquired +49:56.693 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31468 +49:56.694 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:56.694 [INFO] TestAgent.server: New leader elected: payload=Node-5bd56560-7ee1-2bcc-640f-7150f0d18185 +49:56.697 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:56.697 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:56.697 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:56.697 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:56.697 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:56.697 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5bd56560-7ee1-2bcc-640f-7150f0d18185 +49:56.697 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5bd56560-7ee1-2bcc-640f-7150f0d18185 +49:56.966 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:57192 latency=961.153µs +49:56.968 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:57192 latency=532.073µs +49:56.969 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/82d6f562-ace8-097d-cae3-af60ffdf2742/job?acquire=82d6f562-ace8-097d-cae3-af60ffdf2742 from=127.0.0.1:57192 latency=401.499µs +49:57.005 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:57.006 [INFO] TestAgent: Synced node info +49:57.093 [INFO] TestAgent.server: federation state anti-entropy synced +49:57.170 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/_rexec from=127.0.0.1:57192 latency=270.092µs +49:57.171 [DEBUG] TestAgent.server: User event: event=_rexec +49:57.171 [DEBUG] TestAgent: received remote exec event: id=f20a77ef-2923-b170-6f51-09bc550ce462 +49:57.171 [INFO] TestAgent: remote exec script: script= +49:57.171 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/82d6f562-ace8-097d-cae3-af60ffdf2742/?keys=&wait=1000ms from=127.0.0.1:57192 latency=204.2µs +49:57.174 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/82d6f562-ace8-097d-cae3-af60ffdf2742/?index=15&keys=&wait=1000ms from=127.0.0.1:57192 latency=1.739639ms +49:57.175 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/82d6f562-ace8-097d-cae3-af60ffdf2742/Node-5bd56560-7ee1-2bcc-640f-7150f0d18185/exit from=127.0.0.1:57192 latency=110.789µs +49:57.175 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/82d6f562-ace8-097d-cae3-af60ffdf2742/Node-5bd56560-7ee1-2bcc-640f-7150f0d18185/out/00000 from=127.0.0.1:57192 latency=190.375µs +49:58.177 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/82d6f562-ace8-097d-cae3-af60ffdf2742/?index=17&keys=&wait=1000ms from=127.0.0.1:57192 latency=1.001284641s +49:58.177 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/82d6f562-ace8-097d-cae3-af60ffdf2742 from=127.0.0.1:57196 latency=708.61µs +49:58.178 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/_rexec/82d6f562-ace8-097d-cae3-af60ffdf2742?recurse= from=127.0.0.1:57196 latency=298.667µs +49:58.178 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/82d6f562-ace8-097d-cae3-af60ffdf2742 from=127.0.0.1:57196 latency=205.55µs +49:58.178 [INFO] TestAgent: Requesting shutdown +49:58.178 [INFO] TestAgent.server: shutting down server +49:58.178 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:58.178 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:58.178 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:58.178 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:58.178 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:58.178 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:58.178 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:58.178 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:58.178 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:58.178 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:58.178 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:58.179Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:58.179Z [INFO] agent.router.manager: shutting down +49:58.179 [INFO] TestAgent: consul server down +49:58.179 [INFO] TestAgent: shutdown complete +49:58.179 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31463 network=tcp +49:58.179 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31463 network=udp +49:58.179 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31464 network=tcp +49:58.679 [INFO] TestAgent: Waiting for endpoints to shut down +49:58.679 [INFO] TestAgent: Endpoints down +--- PASS: TestExecCommand_NoShell (2.20s) +2022-12-04T05:49:56.998Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:57.008Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:57.009 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9fe1ae12-a21d-0273-bf40-f2a85eacf51f Address:127.0.0.1:31496}]" +49:57.009 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31496 [Follower]" leader= +49:57.010 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9fe1ae12-a21d-0273-bf40-f2a85eacf51f.dc2 127.0.0.1 +49:57.010 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9fe1ae12-a21d-0273-bf40-f2a85eacf51f 127.0.0.1 +2022-12-04T05:49:57.010Z [INFO] agent.router: Initializing LAN area manager +49:57.010 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31491 network=udp +49:57.010 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9fe1ae12-a21d-0273-bf40-f2a85eacf51f.dc2 area=wan +49:57.010 [INFO] TestAgent.server: Adding LAN server: server="Node-9fe1ae12-a21d-0273-bf40-f2a85eacf51f (Addr: tcp/127.0.0.1:31496) (DC: dc2)" +49:57.010 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31491 network=tcp +49:57.011 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31492 network=tcp +49:57.011 [INFO] TestAgent: started state syncer +49:57.011 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31497 network=tcp +49:57.072 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:57.072 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31496 [Candidate]" term=2 +49:57.073 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:57.073 [DEBUG] TestAgent.server.raft: vote granted: from=9fe1ae12-a21d-0273-bf40-f2a85eacf51f term=2 tally=1 +49:57.073 [INFO] TestAgent.server.raft: election won: tally=1 +49:57.073 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31496 [Leader]" +49:57.073 [INFO] TestAgent.server: cluster leadership acquired +49:57.073 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31496 +49:57.073 [INFO] TestAgent.server: New leader elected: payload=Node-9fe1ae12-a21d-0273-bf40-f2a85eacf51f +49:57.073 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:57.075 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:57.075 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:57.075 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:57.075 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:57.075 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:57.075 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9fe1ae12-a21d-0273-bf40-f2a85eacf51f +49:57.075 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9fe1ae12-a21d-0273-bf40-f2a85eacf51f +49:57.075 [INFO] TestAgent.server: federation state anti-entropy synced +49:57.280 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:31488] +49:57.280 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:31488 +49:57.281 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-50b5a69b-e305-0350-ac4f-26fd58cecda0.dc1 127.0.0.1 +49:57.281 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +49:57.281 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-50b5a69b-e305-0350-ac4f-26fd58cecda0.dc1 area=wan +49:57.392 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:57.393 [INFO] TestAgent: Synced node info +49:57.393 [DEBUG] TestAgent: Node info in sync +49:57.491 [DEBUG] TestAgent.server: User event: event=_rexec +49:57.491 [DEBUG] TestAgent: received remote exec event: id=7fad32a6-9c83-d8c0-ec40-3ebf21bfd07b +49:57.492 [INFO] TestAgent: remote exec script: script=uptime +49:57.608 [DEBUG] TestAgent.server.serf.wan: serf: messageJoinType: Node-9fe1ae12-a21d-0273-bf40-f2a85eacf51f.dc2 +49:58.003 [INFO] TestAgent: Requesting shutdown +49:58.003 [INFO] TestAgent.server: shutting down server +49:58.003 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:58.003 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:58.003 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:58.003 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:58.003 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:58.003 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:58.003 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:58.003 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:58.003 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:58.003 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:58.003 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:58.003Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:58.003Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:58.003Z [INFO] agent.router.manager: shutting down +49:58.003 [INFO] TestAgent: consul server down +49:58.003 [INFO] TestAgent: shutdown complete +49:58.003 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31491 network=tcp +49:58.003 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31491 network=udp +49:58.003 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31492 network=tcp +49:58.504 [INFO] TestAgent: Waiting for endpoints to shut down +49:58.504 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:56.552Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:56.591Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:56.606 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:50b5a69b-e305-0350-ac4f-26fd58cecda0 Address:127.0.0.1:31489}]" +49:56.607 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-50b5a69b-e305-0350-ac4f-26fd58cecda0.dc1 127.0.0.1 +49:56.608 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31489 [Follower]" leader= +49:56.608 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-50b5a69b-e305-0350-ac4f-26fd58cecda0 127.0.0.1 +2022-12-04T05:49:56.609Z [INFO] agent.router: Initializing LAN area manager +49:56.609 [INFO] TestAgent.server: Adding LAN server: server="Node-50b5a69b-e305-0350-ac4f-26fd58cecda0 (Addr: tcp/127.0.0.1:31489) (DC: dc1)" +49:56.609 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-50b5a69b-e305-0350-ac4f-26fd58cecda0.dc1 area=wan +49:56.610 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31484 network=udp +49:56.610 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31484 network=tcp +49:56.610 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31485 network=tcp +49:56.610 [INFO] TestAgent: started state syncer +49:56.611 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31490 network=tcp +49:56.676 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:56.676 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31489 [Candidate]" term=2 +49:56.676 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:56.676 [DEBUG] TestAgent.server.raft: vote granted: from=50b5a69b-e305-0350-ac4f-26fd58cecda0 term=2 tally=1 +49:56.676 [INFO] TestAgent.server.raft: election won: tally=1 +49:56.676 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31489 [Leader]" +49:56.676 [INFO] TestAgent.server: cluster leadership acquired +49:56.676 [INFO] TestAgent.server: New leader elected: payload=Node-50b5a69b-e305-0350-ac4f-26fd58cecda0 +49:56.676 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31489 +49:56.677 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:56.678 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:56.678 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:56.678 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:56.678 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:56.678 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:56.678 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-50b5a69b-e305-0350-ac4f-26fd58cecda0 +49:56.678 [INFO] TestAgent.server: member joined, marking health alive: member=Node-50b5a69b-e305-0350-ac4f-26fd58cecda0 +49:56.679 [INFO] TestAgent.server: federation state anti-entropy synced +49:56.892 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:56.892 [INFO] TestAgent: Synced node info +49:56.892 [DEBUG] TestAgent: Node info in sync +49:57.280 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:35920 +49:57.281 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9fe1ae12-a21d-0273-bf40-f2a85eacf51f.dc2 127.0.0.1 +49:57.281 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9fe1ae12-a21d-0273-bf40-f2a85eacf51f.dc2 area=wan +49:57.283 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self?dc=dc2 from=127.0.0.1:45206 latency=1.1776ms +49:57.286 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/health/service/consul?dc=dc2&passing=1 from=127.0.0.1:45206 latency=1.303349ms +49:57.287 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create?dc=dc2 from=127.0.0.1:45206 latency=630.086µs +49:57.288 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/_rexec/cf39554b-9c4a-109c-5347-ef1883966255/job?acquire=cf39554b-9c4a-109c-5347-ef1883966255&dc=dc2 from=127.0.0.1:45206 latency=487.405µs +49:57.492 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/event/fire/_rexec?dc=dc2 from=127.0.0.1:45206 latency=1.029459ms +49:57.493 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/cf39554b-9c4a-109c-5347-ef1883966255/?dc=dc2&keys=&wait=500ms from=127.0.0.1:45206 latency=459.704µs +49:57.496 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/cf39554b-9c4a-109c-5347-ef1883966255/?dc=dc2&index=15&keys=&wait=500ms from=127.0.0.1:45206 latency=2.401929ms +49:57.498 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/cf39554b-9c4a-109c-5347-ef1883966255/Node-9fe1ae12-a21d-0273-bf40-f2a85eacf51f/out/00000?dc=dc2 from=127.0.0.1:45206 latency=583.339µs +49:57.498 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/cf39554b-9c4a-109c-5347-ef1883966255/?dc=dc2&index=16&keys=&wait=500ms from=127.0.0.1:45206 latency=286.876µs +49:57.499 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/cf39554b-9c4a-109c-5347-ef1883966255/Node-9fe1ae12-a21d-0273-bf40-f2a85eacf51f/exit?dc=dc2 from=127.0.0.1:45206 latency=250.082µs +49:57.510 [DEBUG] TestAgent.server.serf.wan: serf: messageJoinType: Node-9fe1ae12-a21d-0273-bf40-f2a85eacf51f.dc2 +49:58.001 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/_rexec/cf39554b-9c4a-109c-5347-ef1883966255/?dc=dc2&index=17&keys=&wait=500ms from=127.0.0.1:45206 latency=502.063152ms +49:58.001 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/cf39554b-9c4a-109c-5347-ef1883966255?dc=dc2 from=127.0.0.1:45208 latency=1.232406ms +49:58.002 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/_rexec/cf39554b-9c4a-109c-5347-ef1883966255?dc=dc2&recurse= from=127.0.0.1:45208 latency=578.332µs +49:58.003 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/cf39554b-9c4a-109c-5347-ef1883966255?dc=dc2 from=127.0.0.1:45208 latency=505.427µs +49:58.504 [INFO] TestAgent: Requesting shutdown +49:58.504 [INFO] TestAgent.server: shutting down server +49:58.504 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:58.504 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:58.504 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:58.504 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:58.504 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:58.504 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:58.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:58.504 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:58.504 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:58.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:58.504 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:58.504Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:58.504Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:58.504Z [INFO] agent.router.manager: shutting down +49:58.504 [INFO] TestAgent: consul server down +49:58.504 [INFO] TestAgent: shutdown complete +49:58.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31484 network=tcp +49:58.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31484 network=udp +49:58.504 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31485 network=tcp +49:59.005 [INFO] TestAgent: Waiting for endpoints to shut down +49:59.005 [INFO] TestAgent: Endpoints down +--- PASS: TestExecCommand_CrossDC (2.53s) PASS -ok github.com/hashicorp/consul/command/exec 2.966s +ok github.com/hashicorp/consul/command/exec 2.577s === RUN TestForceLeaveCommand_noTabs === PAUSE TestForceLeaveCommand_noTabs === RUN TestForceLeaveCommand @@ -84514,378 +84696,368 @@ === RUN TestForceLeaveCommand_noAddrs === PAUSE TestForceLeaveCommand_noAddrs === CONT TestForceLeaveCommand_noTabs ---- PASS: TestForceLeaveCommand_noTabs (0.00s) -=== CONT TestForceLeaveCommand_noAddrs ---- PASS: TestForceLeaveCommand_noAddrs (0.00s) === CONT TestForceLeaveCommand_prune [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestForceLeaveCommand_noTabs (0.00s) === CONT TestForceLeaveCommand_NoNodeWithName === CONT TestForceLeaveCommand +=== CONT TestForceLeaveCommand_noAddrs +--- PASS: TestForceLeaveCommand_noAddrs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:17:01.169Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:01.221Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:01.240 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8f8f1328-32b2-fdaa-a131-71c7d9bc8427 Address:127.0.0.1:16139}]" -17:01.240 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8f8f1328-32b2-fdaa-a131-71c7d9bc8427.dc1 127.0.0.1 -17:01.241 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8f8f1328-32b2-fdaa-a131-71c7d9bc8427 127.0.0.1 -2021-10-31T23:17:01.241Z [INFO] agent.router: Initializing LAN area manager -17:01.241 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=udp -17:01.262 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=tcp -17:01.262 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16135 network=tcp -17:01.262 [INFO] TestAgent: started state syncer -17:01.263 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8f8f1328-32b2-fdaa-a131-71c7d9bc8427.dc1 area=wan -17:01.263 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16140 network=tcp -17:01.263 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16139 [Follower]" leader= -17:01.263 [INFO] TestAgent.server: Adding LAN server: server="Node-8f8f1328-32b2-fdaa-a131-71c7d9bc8427 (Addr: tcp/127.0.0.1:16139) (DC: dc1)" -17:01.327 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:01.327 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16139 [Candidate]" term=2 -17:01.327 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:01.327 [DEBUG] TestAgent.server.raft: vote granted: from=8f8f1328-32b2-fdaa-a131-71c7d9bc8427 term=2 tally=1 -17:01.327 [INFO] TestAgent.server.raft: election won: tally=1 -17:01.327 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16139 [Leader]" -17:01.327 [INFO] TestAgent.server: cluster leadership acquired -17:01.328 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16139 -17:01.328 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:01.328 [INFO] TestAgent.server: New leader elected: payload=Node-8f8f1328-32b2-fdaa-a131-71c7d9bc8427 -17:01.330 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:01.330 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:01.330 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:01.330 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:01.330 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:01.330 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8f8f1328-32b2-fdaa-a131-71c7d9bc8427 -17:01.330 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8f8f1328-32b2-fdaa-a131-71c7d9bc8427 -17:01.331 [INFO] TestAgent.server: federation state anti-entropy synced -17:01.350 [INFO] TestAgent: Force leaving node: node=garbage-name -17:01.350 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/force-leave/garbage-name from=127.0.0.1:50830 error="agent: No node found with name 'garbage-name'" -17:01.362 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/force-leave/garbage-name from=127.0.0.1:50830 latency=11.477913ms -17:01.366 [INFO] TestAgent: Requesting shutdown -17:01.366 [INFO] TestAgent.server: shutting down server -17:01.366 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:01.366 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:01.366 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:01.366 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:01.366 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:01.366 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:01.366 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:01.366 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:01.367 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:01.367 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:01.367 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:01.367 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:01.367Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:01.367Z [INFO] agent.router.manager: shutting down -17:01.367 [INFO] TestAgent: consul server down -17:01.367 [INFO] TestAgent: shutdown complete -17:01.367 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=tcp -17:01.367 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=udp -17:01.367 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16135 network=tcp -17:01.869 [INFO] TestAgent: Waiting for endpoints to shut down -17:01.869 [INFO] TestAgent: Endpoints down ---- PASS: TestForceLeaveCommand_NoNodeWithName (0.79s) -2021-10-31T23:17:01.633Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:01.666Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:01.670 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8b517d6e-533a-ca16-73b5-e131f0500d65 Address:127.0.0.1:16153}]" -17:01.671 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8b517d6e-533a-ca16-73b5-e131f0500d65.dc1 127.0.0.1 -17:01.671 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16153 [Follower]" leader= -17:01.672 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8b517d6e-533a-ca16-73b5-e131f0500d65 127.0.0.1 -2021-10-31T23:17:01.672Z [INFO] agent.router: Initializing LAN area manager -17:01.672 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16148 network=udp -17:01.672 [INFO] TestAgent.server: Adding LAN server: server="Node-8b517d6e-533a-ca16-73b5-e131f0500d65 (Addr: tcp/127.0.0.1:16153) (DC: dc1)" -17:01.672 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8b517d6e-533a-ca16-73b5-e131f0500d65.dc1 area=wan -17:01.672 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16148 network=tcp -17:01.691 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16149 network=tcp -17:01.691 [INFO] TestAgent: started state syncer -17:01.692 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16154 network=tcp -17:01.725 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:01.725 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16153 [Candidate]" term=2 -17:01.725 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:01.725 [DEBUG] TestAgent.server.raft: vote granted: from=8b517d6e-533a-ca16-73b5-e131f0500d65 term=2 tally=1 -17:01.725 [INFO] TestAgent.server.raft: election won: tally=1 -17:01.725 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16153 [Leader]" -17:01.725 [INFO] TestAgent.server: cluster leadership acquired -17:01.725 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16153 -17:01.725 [INFO] TestAgent.server: New leader elected: payload=Node-8b517d6e-533a-ca16-73b5-e131f0500d65 -17:01.726 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:01.727 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:01.727 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:01.727 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:01.727 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:01.727 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:01.728 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8b517d6e-533a-ca16-73b5-e131f0500d65 -17:01.728 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8b517d6e-533a-ca16-73b5-e131f0500d65 -17:01.728 [INFO] TestAgent.server: federation state anti-entropy synced -17:01.863 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:16144] -17:01.866 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:16144 -17:01.870 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:55846 -17:01.870 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b9b43a23-5c67-618c-584d-f2f0208f4aa5 127.0.0.1 -17:01.870 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -17:01.870 [DEBUG] TestAgent: systemd notify failed: error="No socket" -17:01.870 [INFO] TestAgent: Requesting shutdown -17:01.870 [INFO] TestAgent.server: shutting down server -17:01.870 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:01.870 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:01.870 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:01.870 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:01.870 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:01.871 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:01.871 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:01.871 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:01.871 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:01.871 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:01.871 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:01.871 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -17:01.871 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b9b43a23-5c67-618c-584d-f2f0208f4aa5.dc1 127.0.0.1 -2021-10-31T23:17:01.871Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:01.871Z [INFO] agent.router.manager: shutting down -17:01.871 [INFO] TestAgent: consul server down -17:01.871 [INFO] TestAgent: shutdown complete -17:01.871 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16148 network=tcp -17:01.871 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16148 network=udp -17:01.871 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16149 network=tcp -17:02.372 [INFO] TestAgent: Waiting for endpoints to shut down -17:02.372 [INFO] TestAgent: Endpoints down -2021-10-31T23:17:01.211Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:01.254Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:01.264 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b9b43a23-5c67-618c-584d-f2f0208f4aa5 Address:127.0.0.1:16146}]" -17:01.265 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b9b43a23-5c67-618c-584d-f2f0208f4aa5.dc1 127.0.0.1 -17:01.265 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b9b43a23-5c67-618c-584d-f2f0208f4aa5 127.0.0.1 -2021-10-31T23:17:01.266Z [INFO] agent.router: Initializing LAN area manager -17:01.266 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16141 network=udp -17:01.266 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16146 [Follower]" leader= -17:01.266 [INFO] TestAgent.server: Adding LAN server: server="Node-b9b43a23-5c67-618c-584d-f2f0208f4aa5 (Addr: tcp/127.0.0.1:16146) (DC: dc1)" -17:01.266 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b9b43a23-5c67-618c-584d-f2f0208f4aa5.dc1 area=wan -17:01.267 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16141 network=tcp -17:01.267 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16142 network=tcp -17:01.267 [INFO] TestAgent: started state syncer -17:01.267 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16147 network=tcp -17:01.332 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:01.332 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16146 [Candidate]" term=2 -17:01.332 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:01.332 [DEBUG] TestAgent.server.raft: vote granted: from=b9b43a23-5c67-618c-584d-f2f0208f4aa5 term=2 tally=1 -17:01.332 [INFO] TestAgent.server.raft: election won: tally=1 -17:01.332 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16146 [Leader]" -17:01.332 [INFO] TestAgent.server: cluster leadership acquired -17:01.332 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16146 -17:01.333 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:01.333 [INFO] TestAgent.server: New leader elected: payload=Node-b9b43a23-5c67-618c-584d-f2f0208f4aa5 -17:01.334 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:01.334 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:01.334 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:01.334 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:01.334 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:01.334 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b9b43a23-5c67-618c-584d-f2f0208f4aa5 -17:01.334 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b9b43a23-5c67-618c-584d-f2f0208f4aa5 -17:01.335 [INFO] TestAgent.server: federation state anti-entropy synced -17:01.526 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:01.527 [INFO] TestAgent: Synced node info -17:01.527 [DEBUG] TestAgent: Node info in sync -17:01.866 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:34262 -17:01.869 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8b517d6e-533a-ca16-73b5-e131f0500d65 127.0.0.1 -17:01.869 [INFO] TestAgent.server: Adding LAN server: server="Node-8b517d6e-533a-ca16-73b5-e131f0500d65 (Addr: tcp/127.0.0.1:16153) (DC: dc1)" -17:01.869 [INFO] TestAgent.server: New leader elected: payload=Node-8b517d6e-533a-ca16-73b5-e131f0500d65 -17:01.869 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-8b517d6e-533a-ca16-73b5-e131f0500d65 other=Node-b9b43a23-5c67-618c-584d-f2f0208f4aa5 -17:01.869 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8b517d6e-533a-ca16-73b5-e131f0500d65 -17:01.870 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-8b517d6e-533a-ca16-73b5-e131f0500d65.dc1 127.0.0.1:16152 -17:01.871 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8b517d6e-533a-ca16-73b5-e131f0500d65.dc1 127.0.0.1 -17:01.871 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-8b517d6e-533a-ca16-73b5-e131f0500d65.dc1 address=127.0.0.1:16152 -17:01.871 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8b517d6e-533a-ca16-73b5-e131f0500d65.dc1 area=wan -17:02.190 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:02.190 [DEBUG] TestAgent: Node info in sync -17:02.262 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:02.262 [DEBUG] TestAgent: Node info in sync -17:02.386 [INFO] TestAgent: Force leaving node: node=Node-8b517d6e-533a-ca16-73b5-e131f0500d65 -17:02.386 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/force-leave/Node-8b517d6e-533a-ca16-73b5-e131f0500d65 from=127.0.0.1:53404 latency=40.266µs -17:02.769 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-8b517d6e-533a-ca16-73b5-e131f0500d65 (timeout reached) -17:03.268 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-8b517d6e-533a-ca16-73b5-e131f0500d65 has failed, no acks received -17:03.338 [WARN] TestAgent: error getting server health from server: server=Node-8b517d6e-533a-ca16-73b5-e131f0500d65 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:16153: connect: connection refused" -17:04.338 [WARN] TestAgent: error getting server health from server: server=Node-8b517d6e-533a-ca16-73b5-e131f0500d65 error="context deadline exceeded" -17:04.770 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-8b517d6e-533a-ca16-73b5-e131f0500d65 (timeout reached) -17:05.335 [WARN] TestAgent: error getting server health from server: server=Node-8b517d6e-533a-ca16-73b5-e131f0500d65 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:16153: connect: connection refused" -17:06.270 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-8b517d6e-533a-ca16-73b5-e131f0500d65 has failed, no acks received -17:06.338 [WARN] TestAgent: error getting server health from server: server=Node-8b517d6e-533a-ca16-73b5-e131f0500d65 error="context deadline exceeded" -17:06.773 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-8b517d6e-533a-ca16-73b5-e131f0500d65 (timeout reached) -17:07.270 [INFO] TestAgent.server.memberlist.lan: memberlist: Marking Node-8b517d6e-533a-ca16-73b5-e131f0500d65 as failed, suspect timeout reached (0 peer confirmations) -17:07.270 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-8b517d6e-533a-ca16-73b5-e131f0500d65 127.0.0.1 -17:07.270 [INFO] TestAgent.server: Removing LAN server: server="Node-8b517d6e-533a-ca16-73b5-e131f0500d65 (Addr: tcp/127.0.0.1:16153) (DC: dc1)" -17:07.270 [INFO] TestAgent.server: deregistering member: member=Node-8b517d6e-533a-ca16-73b5-e131f0500d65 reason=left -17:07.285 [INFO] TestAgent: Requesting shutdown -17:07.286 [INFO] TestAgent.server: shutting down server -17:07.286 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:07.286 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:07.286 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:07.286 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:07.286 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:07.286 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:07.286 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:07.286 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:07.286 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:07.286 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:07.286 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:07.286Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:07.286Z [INFO] agent.router.manager: shutting down -17:07.286 [INFO] TestAgent: consul server down -17:07.286 [INFO] TestAgent: shutdown complete -17:07.286 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16141 network=tcp -17:07.286 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16141 network=udp -17:07.286 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16142 network=tcp -17:07.791 [INFO] TestAgent: Waiting for endpoints to shut down -17:07.791 [INFO] TestAgent: Endpoints down ---- PASS: TestForceLeaveCommand (6.71s) -2021-10-31T23:17:01.669Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:01.706Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:01.708 [INFO] Agent2.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:90034491-dbe0-fc2d-371c-b7475a0e5125 Address:127.0.0.1:16160}]" -17:01.709 [INFO] Agent2.server.serf.wan: serf: EventMemberJoin: Node-90034491-dbe0-fc2d-371c-b7475a0e5125.dc1 127.0.0.1 -17:01.709 [INFO] Agent2.server.raft: entering follower state: follower="Node at 127.0.0.1:16160 [Follower]" leader= -17:01.709 [INFO] Agent2.server.serf.lan: serf: EventMemberJoin: Node-90034491-dbe0-fc2d-371c-b7475a0e5125 127.0.0.1 -2021-10-31T23:17:01.709Z [INFO] agent.router: Initializing LAN area manager -17:01.710 [INFO] Agent2: Started DNS server: address=127.0.0.1:16155 network=udp -17:01.710 [INFO] Agent2.server: Adding LAN server: server="Node-90034491-dbe0-fc2d-371c-b7475a0e5125 (Addr: tcp/127.0.0.1:16160) (DC: dc1)" -17:01.710 [INFO] Agent2.server: Handled event for server in area: event=member-join server=Node-90034491-dbe0-fc2d-371c-b7475a0e5125.dc1 area=wan -17:01.710 [INFO] Agent2: Started DNS server: address=127.0.0.1:16155 network=tcp -17:01.711 [INFO] Agent2: Started HTTP server: address=127.0.0.1:16156 network=tcp -17:01.711 [INFO] Agent2: started state syncer -17:01.711 [INFO] Agent2: Started gRPC server: address=127.0.0.1:16161 network=tcp -17:01.769 [WARN] Agent2.server.raft: heartbeat timeout reached, starting election: last-leader= -17:01.769 [INFO] Agent2.server.raft: entering candidate state: node="Node at 127.0.0.1:16160 [Candidate]" term=2 -17:01.770 [DEBUG] Agent2.server.raft: votes: needed=1 -17:01.770 [DEBUG] Agent2.server.raft: vote granted: from=90034491-dbe0-fc2d-371c-b7475a0e5125 term=2 tally=1 -17:01.770 [INFO] Agent2.server.raft: election won: tally=1 -17:01.770 [INFO] Agent2.server.raft: entering leader state: leader="Node at 127.0.0.1:16160 [Leader]" -17:01.770 [INFO] Agent2.server: cluster leadership acquired -17:01.770 [DEBUG] Agent2.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16160 -17:01.771 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:01.772 [INFO] Agent2.server.connect: initialized primary datacenter CA with provider: provider=consul -17:01.772 [INFO] Agent2.leader: started routine: routine="federation state anti-entropy" -17:01.772 [INFO] Agent2.leader: started routine: routine="federation state pruning" -17:01.772 [INFO] Agent2.leader: started routine: routine="intermediate cert renew watch" -17:01.772 [INFO] Agent2.leader: started routine: routine="CA root pruning" -17:01.772 [DEBUG] Agent2.server: Skipping self join check for node since the cluster is too small: node=Node-90034491-dbe0-fc2d-371c-b7475a0e5125 -17:01.772 [INFO] Agent2.server: member joined, marking health alive: member=Node-90034491-dbe0-fc2d-371c-b7475a0e5125 -17:01.773 [INFO] Agent2.server: New leader elected: payload=Node-90034491-dbe0-fc2d-371c-b7475a0e5125 -17:01.773 [INFO] Agent2.server: federation state anti-entropy synced -17:01.938 [DEBUG] Agent2: Skipping remote check since it is managed automatically: check=serfHealth -17:01.939 [INFO] Agent2: Synced node info -17:01.939 [DEBUG] Agent2: Node info in sync -17:01.953 [INFO] Agent2: (LAN) joining: lan_addresses=[127.0.0.1:16130] -17:01.954 [DEBUG] Agent2.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:16130 -17:01.958 [DEBUG] Agent2.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:54940 -17:01.958 [INFO] Agent2.server.serf.wan: serf: EventMemberJoin: Node-ff40d7c7-06d4-d1df-1e7a-07bf2b544363.dc1 127.0.0.1 -17:01.958 [INFO] Agent2.server: Handled event for server in area: event=member-join server=Node-ff40d7c7-06d4-d1df-1e7a-07bf2b544363.dc1 area=wan -17:01.959 [INFO] Agent2.server.serf.lan: serf: EventMemberJoin: Node-ff40d7c7-06d4-d1df-1e7a-07bf2b544363 127.0.0.1 -17:01.959 [INFO] Agent2: (LAN) joined: number_of_nodes=1 -17:01.959 [DEBUG] Agent2: systemd notify failed: error="No socket" -17:01.959 [INFO] Agent2: Requesting shutdown -17:01.959 [INFO] Agent2.server: shutting down server -17:01.959 [DEBUG] Agent2.leader: stopping routine: routine="federation state anti-entropy" -17:01.959 [DEBUG] Agent2.leader: stopping routine: routine="federation state pruning" -17:01.959 [DEBUG] Agent2.leader: stopping routine: routine="intermediate cert renew watch" -17:01.959 [DEBUG] Agent2.leader: stopping routine: routine="CA root pruning" -17:01.959 [WARN] Agent2.server.serf.lan: serf: Shutdown without a Leave -17:01.959 [ERROR] Agent2.server: error performing anti-entropy sync of federation state: error="context canceled" -17:01.959 [DEBUG] Agent2.leader: stopped routine: routine="federation state anti-entropy" -17:01.959 [DEBUG] Agent2.leader: stopped routine: routine="federation state pruning" -17:01.959 [DEBUG] Agent2.leader: stopped routine: routine="intermediate cert renew watch" -17:01.959 [DEBUG] Agent2.leader: stopped routine: routine="CA root pruning" -17:01.959 [WARN] Agent2.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:01.959Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:01.959Z [INFO] agent.router.manager: shutting down -17:01.959 [INFO] Agent2: consul server down -17:01.959 [INFO] Agent2: shutdown complete -17:01.959 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:16155 network=tcp -17:01.959 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:16155 network=udp -17:01.959 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:16156 network=tcp -17:02.460 [INFO] Agent2: Waiting for endpoints to shut down -17:02.460 [INFO] Agent2: Endpoints down -2021-10-31T23:17:01.163Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:01.217Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:01.237 [INFO] Agent1.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ff40d7c7-06d4-d1df-1e7a-07bf2b544363 Address:127.0.0.1:16132}]" -17:01.241 [INFO] Agent1.server.raft: entering follower state: follower="Node at 127.0.0.1:16132 [Follower]" leader= -17:01.246 [INFO] Agent1.server.serf.wan: serf: EventMemberJoin: Node-ff40d7c7-06d4-d1df-1e7a-07bf2b544363.dc1 127.0.0.1 -17:01.247 [INFO] Agent1.server.serf.lan: serf: EventMemberJoin: Node-ff40d7c7-06d4-d1df-1e7a-07bf2b544363 127.0.0.1 -2021-10-31T23:17:01.247Z [INFO] agent.router: Initializing LAN area manager -17:01.263 [INFO] Agent1: Started DNS server: address=127.0.0.1:16127 network=udp -17:01.263 [INFO] Agent1.server: Adding LAN server: server="Node-ff40d7c7-06d4-d1df-1e7a-07bf2b544363 (Addr: tcp/127.0.0.1:16132) (DC: dc1)" -17:01.263 [INFO] Agent1.server: Handled event for server in area: event=member-join server=Node-ff40d7c7-06d4-d1df-1e7a-07bf2b544363.dc1 area=wan -17:01.263 [INFO] Agent1: Started DNS server: address=127.0.0.1:16127 network=tcp -17:01.264 [INFO] Agent1: Started HTTP server: address=127.0.0.1:16128 network=tcp -17:01.264 [INFO] Agent1: started state syncer -17:01.266 [INFO] Agent1: Started gRPC server: address=127.0.0.1:16133 network=tcp -17:01.286 [WARN] Agent1.server.raft: heartbeat timeout reached, starting election: last-leader= -17:01.286 [INFO] Agent1.server.raft: entering candidate state: node="Node at 127.0.0.1:16132 [Candidate]" term=2 -17:01.286 [DEBUG] Agent1.server.raft: votes: needed=1 -17:01.286 [DEBUG] Agent1.server.raft: vote granted: from=ff40d7c7-06d4-d1df-1e7a-07bf2b544363 term=2 tally=1 -17:01.286 [INFO] Agent1.server.raft: election won: tally=1 -17:01.286 [INFO] Agent1.server.raft: entering leader state: leader="Node at 127.0.0.1:16132 [Leader]" -17:01.287 [INFO] Agent1.server: cluster leadership acquired -17:01.287 [DEBUG] Agent1.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16132 -17:01.287 [INFO] Agent1.server: New leader elected: payload=Node-ff40d7c7-06d4-d1df-1e7a-07bf2b544363 -17:01.288 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:01.307 [INFO] Agent1.server.connect: initialized primary datacenter CA with provider: provider=consul -17:01.307 [INFO] Agent1.leader: started routine: routine="federation state anti-entropy" -17:01.307 [INFO] Agent1.leader: started routine: routine="federation state pruning" -17:01.307 [INFO] Agent1.leader: started routine: routine="intermediate cert renew watch" -17:01.307 [INFO] Agent1.leader: started routine: routine="CA root pruning" -17:01.307 [DEBUG] Agent1.server: Skipping self join check for node since the cluster is too small: node=Node-ff40d7c7-06d4-d1df-1e7a-07bf2b544363 -17:01.307 [INFO] Agent1.server: member joined, marking health alive: member=Node-ff40d7c7-06d4-d1df-1e7a-07bf2b544363 -17:01.308 [INFO] Agent1.server: federation state anti-entropy synced -17:01.440 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth -17:01.440 [INFO] Agent1: Synced node info -17:01.954 [DEBUG] Agent1.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:55936 -17:01.956 [INFO] Agent1.server.serf.lan: serf: EventMemberJoin: Node-90034491-dbe0-fc2d-371c-b7475a0e5125 127.0.0.1 -17:01.956 [INFO] Agent1.server: Adding LAN server: server="Node-90034491-dbe0-fc2d-371c-b7475a0e5125 (Addr: tcp/127.0.0.1:16160) (DC: dc1)" -17:01.956 [INFO] Agent1.server: New leader elected: payload=Node-90034491-dbe0-fc2d-371c-b7475a0e5125 -17:01.956 [ERROR] Agent1.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-90034491-dbe0-fc2d-371c-b7475a0e5125 other=Node-ff40d7c7-06d4-d1df-1e7a-07bf2b544363 -17:01.956 [INFO] Agent1.server: member joined, marking health alive: member=Node-90034491-dbe0-fc2d-371c-b7475a0e5125 -17:01.958 [DEBUG] Agent1.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-90034491-dbe0-fc2d-371c-b7475a0e5125.dc1 127.0.0.1:16159 -17:01.960 [INFO] Agent1.server.serf.wan: serf: EventMemberJoin: Node-90034491-dbe0-fc2d-371c-b7475a0e5125.dc1 127.0.0.1 -17:01.960 [DEBUG] Agent1.server: Successfully performed flood-join for server at address: server=Node-90034491-dbe0-fc2d-371c-b7475a0e5125.dc1 address=127.0.0.1:16159 -17:01.960 [INFO] Agent1.server: Handled event for server in area: event=member-join server=Node-90034491-dbe0-fc2d-371c-b7475a0e5125.dc1 area=wan -17:02.001 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth -17:02.001 [DEBUG] Agent1: Node info in sync -17:02.462 [INFO] Agent1: Force leaving node: node=Node-90034491-dbe0-fc2d-371c-b7475a0e5125 -17:02.749 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-90034491-dbe0-fc2d-371c-b7475a0e5125 (timeout reached) -17:03.250 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-90034491-dbe0-fc2d-371c-b7475a0e5125 has failed, no acks received -17:04.751 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-90034491-dbe0-fc2d-371c-b7475a0e5125 (timeout reached) -17:06.250 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-90034491-dbe0-fc2d-371c-b7475a0e5125 has failed, no acks received -17:06.753 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-90034491-dbe0-fc2d-371c-b7475a0e5125 (timeout reached) -17:07.250 [INFO] Agent1.server.memberlist.lan: memberlist: Marking Node-90034491-dbe0-fc2d-371c-b7475a0e5125 as failed, suspect timeout reached (0 peer confirmations) -17:09.253 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-90034491-dbe0-fc2d-371c-b7475a0e5125 has failed, no acks received -17:09.253 [DEBUG] Agent1.server.memberlist.wan: memberlist: Failed ping: Node-90034491-dbe0-fc2d-371c-b7475a0e5125.dc1 (timeout reached) -17:10.466 [INFO] Agent1.server.serf.lan: serf: EventMemberReap (forced): Node-90034491-dbe0-fc2d-371c-b7475a0e5125 127.0.0.1 -17:10.466 [INFO] Agent1.server: Removing LAN server: server="Node-90034491-dbe0-fc2d-371c-b7475a0e5125 (Addr: tcp/127.0.0.1:16160) (DC: dc1)" -17:10.466 [INFO] Agent1.server: deregistering member: member=Node-90034491-dbe0-fc2d-371c-b7475a0e5125 reason=reaped -17:10.467 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth -17:10.467 [DEBUG] Agent1: Node info in sync -17:10.467 [DEBUG] Agent1: Node info in sync -17:11.250 [INFO] Agent1.server.memberlist.wan: memberlist: Suspect Node-90034491-dbe0-fc2d-371c-b7475a0e5125.dc1 has failed, no acks received -17:18.469 [INFO] Agent1.server.serf.wan: serf: EventMemberReap (forced): Node-90034491-dbe0-fc2d-371c-b7475a0e5125.dc1 127.0.0.1 -17:18.469 [DEBUG] Agent1.http: Request finished: method=PUT url=/v1/agent/force-leave/Node-90034491-dbe0-fc2d-371c-b7475a0e5125?prune=1 from=127.0.0.1:43564 latency=16.007049596s -17:18.469 [INFO] Agent1.server: Handled event for server in area: event=member-reap server=Node-90034491-dbe0-fc2d-371c-b7475a0e5125.dc1 area=wan -17:18.469 [INFO] Agent1: Requesting shutdown -17:18.469 [INFO] Agent1.server: shutting down server -17:18.469 [DEBUG] Agent1.leader: stopping routine: routine="federation state pruning" -17:18.469 [DEBUG] Agent1.leader: stopping routine: routine="intermediate cert renew watch" -17:18.469 [DEBUG] Agent1.leader: stopping routine: routine="CA root pruning" -17:18.469 [DEBUG] Agent1.leader: stopping routine: routine="federation state anti-entropy" -17:18.469 [WARN] Agent1.server.serf.lan: serf: Shutdown without a Leave -17:18.470 [ERROR] Agent1.server: error performing anti-entropy sync of federation state: error="context canceled" -17:18.470 [DEBUG] Agent1.leader: stopped routine: routine="federation state anti-entropy" -17:18.470 [DEBUG] Agent1.leader: stopped routine: routine="federation state pruning" -17:18.470 [DEBUG] Agent1.leader: stopped routine: routine="intermediate cert renew watch" -17:18.470 [DEBUG] Agent1.leader: stopped routine: routine="CA root pruning" -17:18.470 [WARN] Agent1.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:18.470Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:18.470Z [INFO] agent.router.manager: shutting down -17:18.470 [INFO] Agent1: consul server down -17:18.470 [INFO] Agent1: shutdown complete -17:18.470 [INFO] Agent1: Stopping server: protocol=DNS address=127.0.0.1:16127 network=tcp -17:18.470 [INFO] Agent1: Stopping server: protocol=DNS address=127.0.0.1:16127 network=udp -17:18.470 [INFO] Agent1: Stopping server: protocol=HTTP address=127.0.0.1:16128 network=tcp -17:18.972 [INFO] Agent1: Waiting for endpoints to shut down -17:18.972 [INFO] Agent1: Endpoints down ---- PASS: TestForceLeaveCommand_prune (17.89s) +2022-12-04T05:49:57.922Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:57.942Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:57.944 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d295e06f-8737-9a3d-eb3b-67bff9e1ee8f Address:127.0.0.1:25328}]" +49:57.945 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25328 [Follower]" leader= +49:57.945 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d295e06f-8737-9a3d-eb3b-67bff9e1ee8f.dc1 127.0.0.1 +49:57.949 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d295e06f-8737-9a3d-eb3b-67bff9e1ee8f 127.0.0.1 +2022-12-04T05:49:57.949Z [INFO] agent.router: Initializing LAN area manager +49:57.950 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d295e06f-8737-9a3d-eb3b-67bff9e1ee8f.dc1 area=wan +49:57.950 [INFO] TestAgent.server: Adding LAN server: server="Node-d295e06f-8737-9a3d-eb3b-67bff9e1ee8f (Addr: tcp/127.0.0.1:25328) (DC: dc1)" +49:57.950 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=udp +49:57.950 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=tcp +49:57.950 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25324 network=tcp +49:57.950 [INFO] TestAgent: started state syncer +49:57.950 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25329 network=tcp +49:58.004 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:58.004 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25328 [Candidate]" term=2 +49:58.004 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:58.004 [DEBUG] TestAgent.server.raft: vote granted: from=d295e06f-8737-9a3d-eb3b-67bff9e1ee8f term=2 tally=1 +49:58.004 [INFO] TestAgent.server.raft: election won: tally=1 +49:58.004 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25328 [Leader]" +49:58.004 [INFO] TestAgent.server: cluster leadership acquired +49:58.004 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25328 +49:58.004 [INFO] TestAgent.server: New leader elected: payload=Node-d295e06f-8737-9a3d-eb3b-67bff9e1ee8f +49:58.005 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:58.006 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:58.006 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:58.006 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:58.006 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:58.006 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:58.006 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d295e06f-8737-9a3d-eb3b-67bff9e1ee8f +49:58.006 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d295e06f-8737-9a3d-eb3b-67bff9e1ee8f +49:58.006 [INFO] TestAgent.server: federation state anti-entropy synced +49:58.089 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:58.090 [INFO] TestAgent: Synced node info +49:58.350 [INFO] TestAgent: Force leaving node: node=garbage-name +49:58.350 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/agent/force-leave/garbage-name from=127.0.0.1:49032 error="agent: No node found with name 'garbage-name'" +49:58.350 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/force-leave/garbage-name from=127.0.0.1:49032 latency=222.042µs +49:58.351 [INFO] TestAgent: Requesting shutdown +49:58.351 [INFO] TestAgent.server: shutting down server +49:58.351 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:58.351 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:58.351 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:58.351 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:58.351 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:58.351 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:58.351 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:58.351 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:58.351 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:58.351 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:58.351 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:58.351Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:58.351Z [INFO] agent.router.manager: shutting down +49:58.351 [INFO] TestAgent: consul server down +49:58.351 [INFO] TestAgent: shutdown complete +49:58.351 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=tcp +49:58.351 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=udp +49:58.351 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25324 network=tcp +49:58.852 [INFO] TestAgent: Waiting for endpoints to shut down +49:58.852 [INFO] TestAgent: Endpoints down +--- PASS: TestForceLeaveCommand_NoNodeWithName (0.97s) +2022-12-04T05:49:58.224Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:58.236Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:58.237 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2b57ce18-381b-9eea-b5c4-43b6b4994f72 Address:127.0.0.1:25342}]" +49:58.237 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25342 [Follower]" leader= +49:58.238 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72.dc1 127.0.0.1 +49:58.238 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 127.0.0.1 +2022-12-04T05:49:58.238Z [INFO] agent.router: Initializing LAN area manager +49:58.238 [INFO] TestAgent.server: Adding LAN server: server="Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 (Addr: tcp/127.0.0.1:25342) (DC: dc1)" +49:58.238 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25337 network=udp +49:58.238 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72.dc1 area=wan +49:58.238 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25337 network=tcp +49:58.239 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25338 network=tcp +49:58.239 [INFO] TestAgent: started state syncer +49:58.239 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25343 network=tcp +49:58.281 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:58.281 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25342 [Candidate]" term=2 +49:58.281 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:58.281 [DEBUG] TestAgent.server.raft: vote granted: from=2b57ce18-381b-9eea-b5c4-43b6b4994f72 term=2 tally=1 +49:58.281 [INFO] TestAgent.server.raft: election won: tally=1 +49:58.281 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25342 [Leader]" +49:58.281 [INFO] TestAgent.server: cluster leadership acquired +49:58.281 [INFO] TestAgent.server: New leader elected: payload=Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 +49:58.281 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25342 +49:58.282 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:58.283 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:58.283 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:58.283 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:58.283 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:58.283 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:58.283 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 +49:58.283 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 +49:58.283 [INFO] TestAgent.server: federation state anti-entropy synced +49:58.473 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:25333] +49:58.473 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:25333 +49:58.474 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-88aebfec-a192-1b49-23dc-6a9b34342a92 127.0.0.1 +49:58.474 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +49:58.474 [DEBUG] TestAgent: systemd notify failed: error="No socket" +49:58.474 [INFO] TestAgent: Requesting shutdown +49:58.474 [INFO] TestAgent.server: shutting down server +49:58.474 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +49:58.474 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +49:58.474 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +49:58.474 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +49:58.474 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +49:58.474 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +49:58.474 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +49:58.474 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +49:58.474 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +49:58.474 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +49:58.474 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +49:58.474 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:58.474Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:58.474Z [INFO] agent.router.manager: shutting down +49:58.474 [INFO] TestAgent: consul server down +49:58.474 [INFO] TestAgent: shutdown complete +49:58.474 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25337 network=tcp +49:58.474 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25337 network=udp +49:58.474 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25338 network=tcp +49:58.975 [INFO] TestAgent: Waiting for endpoints to shut down +49:58.975 [INFO] TestAgent: Endpoints down +2022-12-04T05:49:57.924Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:57.943Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:57.945 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:88aebfec-a192-1b49-23dc-6a9b34342a92 Address:127.0.0.1:25335}]" +49:57.945 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25335 [Follower]" leader= +49:57.948 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-88aebfec-a192-1b49-23dc-6a9b34342a92.dc1 127.0.0.1 +49:57.950 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-88aebfec-a192-1b49-23dc-6a9b34342a92 127.0.0.1 +2022-12-04T05:49:57.950Z [INFO] agent.router: Initializing LAN area manager +49:57.950 [INFO] TestAgent.server: Adding LAN server: server="Node-88aebfec-a192-1b49-23dc-6a9b34342a92 (Addr: tcp/127.0.0.1:25335) (DC: dc1)" +49:57.950 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-88aebfec-a192-1b49-23dc-6a9b34342a92.dc1 area=wan +49:57.950 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25330 network=udp +49:57.950 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25330 network=tcp +49:57.950 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25331 network=tcp +49:57.950 [INFO] TestAgent: started state syncer +49:57.950 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25336 network=tcp +49:57.983 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +49:57.983 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25335 [Candidate]" term=2 +49:57.983 [DEBUG] TestAgent.server.raft: votes: needed=1 +49:57.983 [DEBUG] TestAgent.server.raft: vote granted: from=88aebfec-a192-1b49-23dc-6a9b34342a92 term=2 tally=1 +49:57.983 [INFO] TestAgent.server.raft: election won: tally=1 +49:57.983 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25335 [Leader]" +49:57.983 [INFO] TestAgent.server: cluster leadership acquired +49:57.983 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25335 +49:57.983 [INFO] TestAgent.server: New leader elected: payload=Node-88aebfec-a192-1b49-23dc-6a9b34342a92 +49:57.984 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:57.999 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +49:57.999 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +49:57.999 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +49:57.999 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +49:57.999 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +49:57.999 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-88aebfec-a192-1b49-23dc-6a9b34342a92 +49:57.999 [INFO] TestAgent.server: member joined, marking health alive: member=Node-88aebfec-a192-1b49-23dc-6a9b34342a92 +49:58.000 [INFO] TestAgent.server: federation state anti-entropy synced +49:58.028 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +49:58.029 [INFO] TestAgent: Synced node info +49:58.473 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:46176 +49:58.474 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 127.0.0.1 +49:58.474 [INFO] TestAgent.server: Adding LAN server: server="Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 (Addr: tcp/127.0.0.1:25342) (DC: dc1)" +49:58.474 [INFO] TestAgent.server: New leader elected: payload=Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 +49:58.474 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 other=Node-88aebfec-a192-1b49-23dc-6a9b34342a92 +49:58.474 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 +49:58.475 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:25341: connect: connection refused +49:58.475 [DEBUG] TestAgent.server: Failed to flood-join server at address: server=Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72.dc1 address=127.0.0.1:25341 error="1 error occurred: + * Failed to join 127.0.0.1: dial tcp 127.0.0.1:25341: connect: connection refused + +" +49:58.975 [INFO] TestAgent: Force leaving node: node=Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 +49:58.975 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/force-leave/Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 from=127.0.0.1:48342 latency=19.099µs +49:59.450 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 (timeout reached) +49:59.951 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 has failed, no acks received +49:59.984 [WARN] TestAgent: error getting server health from server: server=Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:25342: connect: connection refused" +50:00.445 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:00.445 [DEBUG] TestAgent: Node info in sync +50:00.445 [DEBUG] TestAgent: Node info in sync +50:00.984 [WARN] TestAgent: error getting server health from server: server=Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 error="context deadline exceeded" +50:01.450 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 (timeout reached) +50:01.984 [WARN] TestAgent: error getting server health from server: server=Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 error="rpc error getting client: failed to get conn: dial tcp 127.0.0.1:0->127.0.0.1:25342: connect: connection refused" +50:02.952 [INFO] TestAgent.server.memberlist.lan: memberlist: Suspect Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 has failed, no acks received +50:02.984 [WARN] TestAgent: error getting server health from server: server=Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 error="context deadline exceeded" +50:03.241 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:03.241 [DEBUG] TestAgent: Node info in sync +50:03.951 [INFO] TestAgent.server.memberlist.lan: memberlist: Marking Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 as failed, suspect timeout reached (0 peer confirmations) +50:03.951 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 127.0.0.1 +50:03.951 [INFO] TestAgent.server: Removing LAN server: server="Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 (Addr: tcp/127.0.0.1:25342) (DC: dc1)" +50:03.951 [INFO] TestAgent.server: deregistering member: member=Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 reason=left +50:03.967 [INFO] TestAgent: Requesting shutdown +50:03.967 [INFO] TestAgent.server: shutting down server +50:03.967 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:03.967 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:03.967 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:03.967 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:03.967 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:03.967 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:03.967 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:03.967 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:03.967 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:03.967 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:03.967 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:03.967Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:03.967Z [INFO] agent.router.manager: shutting down +50:03.968 [INFO] TestAgent: consul server down +50:03.968 [INFO] TestAgent: shutdown complete +50:03.968 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25330 network=tcp +50:03.968 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25330 network=udp +50:03.968 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25331 network=tcp +50:04.450 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-2b57ce18-381b-9eea-b5c4-43b6b4994f72 (timeout reached) +50:04.469 [INFO] TestAgent: Waiting for endpoints to shut down +50:04.469 [INFO] TestAgent: Endpoints down +--- PASS: TestForceLeaveCommand (6.59s) +2022-12-04T05:49:58.375Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:58.385Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:58.387 [INFO] Agent2.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d0806e09-98b7-b66f-8c96-5b70fefd4511 Address:127.0.0.1:25349}]" +49:58.387 [INFO] Agent2.server.raft: entering follower state: follower="Node at 127.0.0.1:25349 [Follower]" leader= +49:58.387 [INFO] Agent2.server.serf.wan: serf: EventMemberJoin: Node-d0806e09-98b7-b66f-8c96-5b70fefd4511.dc1 127.0.0.1 +49:58.388 [INFO] Agent2.server.serf.lan: serf: EventMemberJoin: Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 127.0.0.1 +2022-12-04T05:49:58.388Z [INFO] agent.router: Initializing LAN area manager +49:58.388 [INFO] Agent2.server: Adding LAN server: server="Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 (Addr: tcp/127.0.0.1:25349) (DC: dc1)" +49:58.388 [INFO] Agent2.server: Handled event for server in area: event=member-join server=Node-d0806e09-98b7-b66f-8c96-5b70fefd4511.dc1 area=wan +49:58.388 [INFO] Agent2: Started DNS server: address=127.0.0.1:25344 network=udp +49:58.388 [INFO] Agent2: Started DNS server: address=127.0.0.1:25344 network=tcp +49:58.388 [INFO] Agent2: Started HTTP server: address=127.0.0.1:25345 network=tcp +49:58.388 [INFO] Agent2: started state syncer +49:58.388 [INFO] Agent2: Started gRPC server: address=127.0.0.1:25350 network=tcp +49:58.431 [WARN] Agent2.server.raft: heartbeat timeout reached, starting election: last-leader= +49:58.431 [INFO] Agent2.server.raft: entering candidate state: node="Node at 127.0.0.1:25349 [Candidate]" term=2 +49:58.431 [DEBUG] Agent2.server.raft: votes: needed=1 +49:58.431 [DEBUG] Agent2.server.raft: vote granted: from=d0806e09-98b7-b66f-8c96-5b70fefd4511 term=2 tally=1 +49:58.431 [INFO] Agent2.server.raft: election won: tally=1 +49:58.431 [INFO] Agent2.server.raft: entering leader state: leader="Node at 127.0.0.1:25349 [Leader]" +49:58.431 [INFO] Agent2.server: cluster leadership acquired +49:58.431 [DEBUG] Agent2.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25349 +49:58.431 [INFO] Agent2.server: New leader elected: payload=Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 +49:58.432 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:58.433 [INFO] Agent2.server.connect: initialized primary datacenter CA with provider: provider=consul +49:58.433 [INFO] Agent2.leader: started routine: routine="federation state anti-entropy" +49:58.433 [INFO] Agent2.leader: started routine: routine="federation state pruning" +49:58.433 [INFO] Agent2.leader: started routine: routine="intermediate cert renew watch" +49:58.433 [INFO] Agent2.leader: started routine: routine="CA root pruning" +49:58.433 [DEBUG] Agent2.server: Skipping self join check for node since the cluster is too small: node=Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 +49:58.433 [INFO] Agent2.server: member joined, marking health alive: member=Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 +49:58.433 [INFO] Agent2.server: federation state anti-entropy synced +49:58.469 [INFO] Agent2: (LAN) joining: lan_addresses=[127.0.0.1:25319] +49:58.469 [DEBUG] Agent2.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:25319 +49:58.470 [INFO] Agent2.server.serf.lan: serf: EventMemberJoin: Node-07306b86-52f8-8a8b-4048-ff1e25fddf46 127.0.0.1 +49:58.470 [INFO] Agent2: (LAN) joined: number_of_nodes=1 +49:58.470 [DEBUG] Agent2: systemd notify failed: error="No socket" +49:58.470 [INFO] Agent2: Requesting shutdown +49:58.470 [INFO] Agent2.server: shutting down server +49:58.470 [DEBUG] Agent2.leader: stopping routine: routine="federation state anti-entropy" +49:58.470 [DEBUG] Agent2.leader: stopping routine: routine="federation state pruning" +49:58.470 [DEBUG] Agent2.leader: stopping routine: routine="intermediate cert renew watch" +49:58.470 [DEBUG] Agent2.leader: stopping routine: routine="CA root pruning" +49:58.470 [WARN] Agent2.server.serf.lan: serf: Shutdown without a Leave +49:58.471 [ERROR] Agent2.server: error performing anti-entropy sync of federation state: error="context canceled" +49:58.471 [DEBUG] Agent2.leader: stopped routine: routine="federation state anti-entropy" +49:58.471 [ERROR] Agent2.anti_entropy: failed to sync remote state: error="No cluster leader" +49:58.471 [DEBUG] Agent2.leader: stopped routine: routine="federation state pruning" +49:58.471 [DEBUG] Agent2.leader: stopped routine: routine="intermediate cert renew watch" +49:58.471 [DEBUG] Agent2.leader: stopped routine: routine="CA root pruning" +49:58.471 [WARN] Agent2.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:49:58.471Z [INFO] agent.router.manager: shutting down +2022-12-04T05:49:58.471Z [INFO] agent.router.manager: shutting down +49:58.471 [INFO] Agent2: consul server down +49:58.471 [INFO] Agent2: shutdown complete +49:58.471 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:25344 network=tcp +49:58.471 [INFO] Agent2: Stopping server: protocol=DNS address=127.0.0.1:25344 network=udp +49:58.471 [INFO] Agent2: Stopping server: protocol=HTTP address=127.0.0.1:25345 network=tcp +49:58.971 [INFO] Agent2: Waiting for endpoints to shut down +49:58.971 [INFO] Agent2: Endpoints down +2022-12-04T05:49:57.922Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:49:57.937Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +49:57.942 [INFO] Agent1.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:07306b86-52f8-8a8b-4048-ff1e25fddf46 Address:127.0.0.1:25321}]" +49:57.942 [INFO] Agent1.server.raft: entering follower state: follower="Node at 127.0.0.1:25321 [Follower]" leader= +49:57.943 [INFO] Agent1.server.serf.wan: serf: EventMemberJoin: Node-07306b86-52f8-8a8b-4048-ff1e25fddf46.dc1 127.0.0.1 +49:57.944 [INFO] Agent1.server.serf.lan: serf: EventMemberJoin: Node-07306b86-52f8-8a8b-4048-ff1e25fddf46 127.0.0.1 +2022-12-04T05:49:57.944Z [INFO] agent.router: Initializing LAN area manager +49:57.944 [INFO] Agent1.server: Handled event for server in area: event=member-join server=Node-07306b86-52f8-8a8b-4048-ff1e25fddf46.dc1 area=wan +49:57.944 [INFO] Agent1: Started DNS server: address=127.0.0.1:25316 network=udp +49:57.944 [INFO] Agent1: Started DNS server: address=127.0.0.1:25316 network=tcp +49:57.944 [INFO] Agent1.server: Adding LAN server: server="Node-07306b86-52f8-8a8b-4048-ff1e25fddf46 (Addr: tcp/127.0.0.1:25321) (DC: dc1)" +49:57.945 [INFO] Agent1: Started HTTP server: address=127.0.0.1:25317 network=tcp +49:57.945 [INFO] Agent1: started state syncer +49:57.945 [INFO] Agent1: Started gRPC server: address=127.0.0.1:25322 network=tcp +49:58.012 [WARN] Agent1.server.raft: heartbeat timeout reached, starting election: last-leader= +49:58.012 [INFO] Agent1.server.raft: entering candidate state: node="Node at 127.0.0.1:25321 [Candidate]" term=2 +49:58.012 [DEBUG] Agent1.server.raft: votes: needed=1 +49:58.012 [DEBUG] Agent1.server.raft: vote granted: from=07306b86-52f8-8a8b-4048-ff1e25fddf46 term=2 tally=1 +49:58.012 [INFO] Agent1.server.raft: election won: tally=1 +49:58.012 [INFO] Agent1.server.raft: entering leader state: leader="Node at 127.0.0.1:25321 [Leader]" +49:58.012 [INFO] Agent1.server: cluster leadership acquired +49:58.012 [DEBUG] Agent1.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25321 +49:58.012 [INFO] Agent1.server: New leader elected: payload=Node-07306b86-52f8-8a8b-4048-ff1e25fddf46 +49:58.013 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +49:58.014 [INFO] Agent1.server.connect: initialized primary datacenter CA with provider: provider=consul +49:58.014 [INFO] Agent1.leader: started routine: routine="federation state anti-entropy" +49:58.014 [INFO] Agent1.leader: started routine: routine="federation state pruning" +49:58.014 [INFO] Agent1.leader: started routine: routine="intermediate cert renew watch" +49:58.014 [INFO] Agent1.leader: started routine: routine="CA root pruning" +49:58.014 [DEBUG] Agent1.server: Skipping self join check for node since the cluster is too small: node=Node-07306b86-52f8-8a8b-4048-ff1e25fddf46 +49:58.014 [INFO] Agent1.server: member joined, marking health alive: member=Node-07306b86-52f8-8a8b-4048-ff1e25fddf46 +49:58.014 [INFO] Agent1.server: federation state anti-entropy synced +49:58.327 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth +49:58.327 [INFO] Agent1: Synced node info +49:58.469 [DEBUG] Agent1.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:60018 +49:58.470 [INFO] Agent1.server.serf.lan: serf: EventMemberJoin: Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 127.0.0.1 +49:58.470 [INFO] Agent1.server: Adding LAN server: server="Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 (Addr: tcp/127.0.0.1:25349) (DC: dc1)" +49:58.470 [INFO] Agent1.server: New leader elected: payload=Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 +49:58.470 [ERROR] Agent1.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 other=Node-07306b86-52f8-8a8b-4048-ff1e25fddf46 +49:58.470 [INFO] Agent1.server: member joined, marking health alive: member=Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 +49:58.471 [DEBUG] Agent1.server.memberlist.wan: memberlist: Failed to join 127.0.0.1: dial tcp 127.0.0.1:25348: connect: connection refused +49:58.471 [DEBUG] Agent1.server: Failed to flood-join server at address: server=Node-d0806e09-98b7-b66f-8c96-5b70fefd4511.dc1 address=127.0.0.1:25348 error="1 error occurred: + * Failed to join 127.0.0.1: dial tcp 127.0.0.1:25348: connect: connection refused + +" +49:58.972 [INFO] Agent1: Force leaving node: node=Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 +49:59.445 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 (timeout reached) +49:59.944 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 has failed, no acks received +50:01.444 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 (timeout reached) +50:02.944 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 has failed, no acks received +50:03.944 [INFO] Agent1.server.memberlist.lan: memberlist: Marking Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 as failed, suspect timeout reached (0 peer confirmations) +50:04.444 [DEBUG] Agent1.server.memberlist.lan: memberlist: Failed ping: Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 (timeout reached) +50:06.973 [INFO] Agent1.server.serf.lan: serf: EventMemberReap (forced): Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 127.0.0.1 +50:06.973 [DEBUG] Agent1.http: Request finished: method=PUT url=/v1/agent/force-leave/Node-d0806e09-98b7-b66f-8c96-5b70fefd4511?prune=1 from=127.0.0.1:33962 latency=8.001298577s +50:06.973 [INFO] Agent1.server.memberlist.lan: memberlist: Suspect Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 has failed, no acks received +50:06.974 [INFO] Agent1.server: Removing LAN server: server="Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 (Addr: tcp/127.0.0.1:25349) (DC: dc1)" +50:06.974 [DEBUG] Agent1: Skipping remote check since it is managed automatically: check=serfHealth +50:06.974 [DEBUG] Agent1: Node info in sync +50:06.974 [INFO] Agent1.server: deregistering member: member=Node-d0806e09-98b7-b66f-8c96-5b70fefd4511 reason=reaped +50:06.975 [INFO] Agent1: Requesting shutdown +50:06.975 [INFO] Agent1.server: shutting down server +50:06.975 [DEBUG] Agent1.leader: stopping routine: routine="federation state anti-entropy" +50:06.975 [DEBUG] Agent1.leader: stopping routine: routine="federation state pruning" +50:06.975 [DEBUG] Agent1.leader: stopping routine: routine="intermediate cert renew watch" +50:06.975 [DEBUG] Agent1.leader: stopping routine: routine="CA root pruning" +50:06.975 [WARN] Agent1.server.serf.lan: serf: Shutdown without a Leave +50:06.975 [DEBUG] Agent1.leader: stopped routine: routine="intermediate cert renew watch" +50:06.975 [DEBUG] Agent1.leader: stopped routine: routine="federation state pruning" +50:06.975 [ERROR] Agent1.server: error performing anti-entropy sync of federation state: error="context canceled" +50:06.975 [DEBUG] Agent1.leader: stopped routine: routine="federation state anti-entropy" +50:06.975 [DEBUG] Agent1.leader: stopped routine: routine="CA root pruning" +50:06.975 [WARN] Agent1.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:06.975Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:06.975Z [INFO] agent.router.manager: shutting down +50:06.975 [INFO] Agent1: consul server down +50:06.975 [INFO] Agent1: shutdown complete +50:06.975 [INFO] Agent1: Stopping server: protocol=DNS address=127.0.0.1:25316 network=tcp +50:06.975 [WARN] Agent1: error getting server health from server: server=Node-07306b86-52f8-8a8b-4048-ff1e25fddf46 error="rpc error making call: EOF" +50:06.975 [INFO] Agent1: Stopping server: protocol=DNS address=127.0.0.1:25316 network=udp +50:06.975 [INFO] Agent1: Stopping server: protocol=HTTP address=127.0.0.1:25317 network=tcp +50:07.476 [INFO] Agent1: Waiting for endpoints to shut down +50:07.476 [INFO] Agent1: Endpoints down +--- PASS: TestForceLeaveCommand_prune (9.60s) PASS -ok github.com/hashicorp/consul/command/forceleave 18.017s +ok github.com/hashicorp/consul/command/forceleave 9.648s ? github.com/hashicorp/consul/command/helpers [no test files] === RUN TestInfoCommand_noTabs === PAUSE TestInfoCommand_noTabs @@ -84897,71 +85069,73 @@ [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:17:06.443Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:06.481Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:06.483 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cf2a03e7-3956-4d41-b0f3-b89e0d4a9696 Address:127.0.0.1:31447}]" -17:06.484 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= -17:06.484 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cf2a03e7-3956-4d41-b0f3-b89e0d4a9696.dc1 127.0.0.1 -17:06.485 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cf2a03e7-3956-4d41-b0f3-b89e0d4a9696 127.0.0.1 -2021-10-31T23:17:06.485Z [INFO] agent.router: Initializing LAN area manager -17:06.485 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=udp -17:06.485 [INFO] TestAgent.server: Adding LAN server: server="Node-cf2a03e7-3956-4d41-b0f3-b89e0d4a9696 (Addr: tcp/127.0.0.1:31447) (DC: dc1)" -17:06.485 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cf2a03e7-3956-4d41-b0f3-b89e0d4a9696.dc1 area=wan -17:06.485 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=tcp -17:06.486 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31443 network=tcp -17:06.486 [INFO] TestAgent: started state syncer -17:06.486 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31448 network=tcp -17:06.545 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:06.545 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 -17:06.546 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:06.546 [DEBUG] TestAgent.server.raft: vote granted: from=cf2a03e7-3956-4d41-b0f3-b89e0d4a9696 term=2 tally=1 -17:06.546 [INFO] TestAgent.server.raft: election won: tally=1 -17:06.546 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" -17:06.546 [INFO] TestAgent.server: cluster leadership acquired -17:06.546 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31447 -17:06.546 [INFO] TestAgent.server: New leader elected: payload=Node-cf2a03e7-3956-4d41-b0f3-b89e0d4a9696 -17:06.547 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:06.583 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:06.583 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:06.583 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:06.583 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:06.583 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:06.583 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cf2a03e7-3956-4d41-b0f3-b89e0d4a9696 -17:06.583 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cf2a03e7-3956-4d41-b0f3-b89e0d4a9696 -17:06.584 [INFO] TestAgent.server: federation state anti-entropy synced -17:06.764 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:54914 latency=3.410549ms -17:06.771 [INFO] TestAgent: Requesting shutdown -17:06.771 [INFO] TestAgent.server: shutting down server -17:06.771 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:06.771 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:06.771 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:06.771 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:06.771 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:06.771 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:06.771 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:06.771 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:06.771 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:06.771 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:06.771 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:06.771 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:06.771Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:06.771Z [INFO] agent.router.manager: shutting down -17:06.771 [INFO] TestAgent: consul server down -17:06.771 [INFO] TestAgent: shutdown complete -17:06.771 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp -17:06.771 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp -17:06.771 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp -17:07.277 [INFO] TestAgent: Waiting for endpoints to shut down -17:07.277 [INFO] TestAgent: Endpoints down ---- PASS: TestInfoCommand (0.92s) +2022-12-04T05:50:01.891Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:01.910Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:01.913 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d746d6ac-54c1-8879-ea00-b5365c616d17 Address:127.0.0.1:19195}]" +50:01.913 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d746d6ac-54c1-8879-ea00-b5365c616d17.dc1 127.0.0.1 +50:01.914 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= +50:01.914 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d746d6ac-54c1-8879-ea00-b5365c616d17 127.0.0.1 +2022-12-04T05:50:01.914Z [INFO] agent.router: Initializing LAN area manager +50:01.914 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp +50:01.914 [INFO] TestAgent.server: Adding LAN server: server="Node-d746d6ac-54c1-8879-ea00-b5365c616d17 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +50:01.914 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d746d6ac-54c1-8879-ea00-b5365c616d17.dc1 area=wan +50:01.915 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp +50:01.915 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp +50:01.915 [INFO] TestAgent: started state syncer +50:01.915 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp +50:01.949 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:01.949 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 +50:01.949 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:01.949 [DEBUG] TestAgent.server.raft: vote granted: from=d746d6ac-54c1-8879-ea00-b5365c616d17 term=2 tally=1 +50:01.949 [INFO] TestAgent.server.raft: election won: tally=1 +50:01.949 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" +50:01.949 [INFO] TestAgent.server: cluster leadership acquired +50:01.950 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 +50:01.950 [INFO] TestAgent.server: New leader elected: payload=Node-d746d6ac-54c1-8879-ea00-b5365c616d17 +50:01.950 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:01.969 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:01.969 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:01.969 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:01.969 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:01.969 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:01.969 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d746d6ac-54c1-8879-ea00-b5365c616d17 +50:01.969 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d746d6ac-54c1-8879-ea00-b5365c616d17 +50:01.970 [INFO] TestAgent.server: federation state anti-entropy synced +50:01.989 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:01.990 [INFO] TestAgent: Synced node info +50:01.990 [DEBUG] TestAgent: Node info in sync +50:02.227 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:40368 latency=1.20867ms +50:02.229 [INFO] TestAgent: Requesting shutdown +50:02.229 [INFO] TestAgent.server: shutting down server +50:02.229 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:02.229 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:02.229 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:02.229 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:02.229 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:02.229 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:02.229 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:02.229 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:02.229 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:02.229 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:02.229 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:02.229Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:02.229Z [INFO] agent.router.manager: shutting down +50:02.229 [INFO] TestAgent: consul server down +50:02.229 [INFO] TestAgent: shutdown complete +50:02.229 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp +50:02.229 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp +50:02.229 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp +50:02.730 [INFO] TestAgent: Waiting for endpoints to shut down +50:02.730 [INFO] TestAgent: Endpoints down +--- PASS: TestInfoCommand (0.89s) PASS -ok github.com/hashicorp/consul/command/info 1.227s +ok github.com/hashicorp/consul/command/info 0.948s === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === CONT TestCommand_noTabs --- PASS: TestCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/intention 0.045s +ok github.com/hashicorp/consul/command/intention 0.005s === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === RUN TestCommand_Validation @@ -84969,82 +85143,82 @@ === RUN TestCommand === PAUSE TestCommand === CONT TestCommand_noTabs ---- PASS: TestCommand_noTabs (0.00s) === CONT TestCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestCommand_Validation +--- PASS: TestCommand_noTabs (0.01s) +=== RUN TestCommand_Validation/3_args === RUN TestCommand_Validation/0_args +[INFO] freeport: detected ephemeral port range of [32768, 60999] +[INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestCommand_Validation/1_args -=== RUN TestCommand_Validation/3_args ---- PASS: TestCommand_Validation (0.01s) +--- PASS: TestCommand_Validation (0.00s) + --- PASS: TestCommand_Validation/3_args (0.00s) --- PASS: TestCommand_Validation/0_args (0.00s) --- PASS: TestCommand_Validation/1_args (0.00s) - --- PASS: TestCommand_Validation/3_args (0.00s) -[INFO] freeport: detected ephemeral port range of [32768, 60999] -[INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:17:12.172Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:12.199Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:12.202 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:09e53a9d-7aef-11c2-a0a8-4a0bdd356fc1 Address:127.0.0.1:12048}]" -17:12.202 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-09e53a9d-7aef-11c2-a0a8-4a0bdd356fc1.dc1 127.0.0.1 -17:12.203 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-09e53a9d-7aef-11c2-a0a8-4a0bdd356fc1 127.0.0.1 -2021-10-31T23:17:12.203Z [INFO] agent.router: Initializing LAN area manager -17:12.203 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=udp -17:12.203 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12048 [Follower]" leader= -17:12.204 [INFO] TestAgent.server: Adding LAN server: server="Node-09e53a9d-7aef-11c2-a0a8-4a0bdd356fc1 (Addr: tcp/127.0.0.1:12048) (DC: dc1)" -17:12.204 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-09e53a9d-7aef-11c2-a0a8-4a0bdd356fc1.dc1 area=wan -17:12.204 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=tcp -17:12.204 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12044 network=tcp -17:12.204 [INFO] TestAgent: started state syncer -17:12.205 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12049 network=tcp -17:12.251 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:12.251 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12048 [Candidate]" term=2 -17:12.251 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:12.251 [DEBUG] TestAgent.server.raft: vote granted: from=09e53a9d-7aef-11c2-a0a8-4a0bdd356fc1 term=2 tally=1 -17:12.251 [INFO] TestAgent.server.raft: election won: tally=1 -17:12.251 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12048 [Leader]" -17:12.252 [INFO] TestAgent.server: cluster leadership acquired -17:12.252 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12048 -17:12.253 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:12.253 [INFO] TestAgent.server: New leader elected: payload=Node-09e53a9d-7aef-11c2-a0a8-4a0bdd356fc1 -17:12.272 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:12.272 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:12.272 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:12.272 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:12.272 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:12.272 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-09e53a9d-7aef-11c2-a0a8-4a0bdd356fc1 -17:12.272 [INFO] TestAgent.server: member joined, marking health alive: member=Node-09e53a9d-7aef-11c2-a0a8-4a0bdd356fc1 -17:12.273 [INFO] TestAgent.server: federation state anti-entropy synced -17:12.280 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:12.281 [INFO] TestAgent: Synced node info -17:12.281 [DEBUG] TestAgent: Node info in sync -17:12.405 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:60536 latency=1.661488ms -17:12.412 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/check?destination=db&source=foo&source-type=consul from=127.0.0.1:60538 latency=4.065026ms -17:12.421 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/check?destination=db&source=web&source-type=consul from=127.0.0.1:60540 latency=1.086377ms -17:12.423 [INFO] TestAgent: Requesting shutdown -17:12.423 [INFO] TestAgent.server: shutting down server -17:12.423 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:12.423 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:12.423 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:12.423 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:12.423 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:12.423 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:12.423 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:12.423 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:12.423 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:12.423 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:12.423 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:12.423Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:12.423Z [INFO] agent.router.manager: shutting down -17:12.423 [INFO] TestAgent: consul server down -17:12.423 [INFO] TestAgent: shutdown complete -17:12.423 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=tcp -17:12.423 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=udp -17:12.423 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12044 network=tcp -17:12.926 [INFO] TestAgent: Waiting for endpoints to shut down -17:12.926 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand (0.82s) +2022-12-04T05:50:05.292Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:05.305Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:05.307 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2b808f30-571d-10a7-1aa9-fc8b00324b44 Address:127.0.0.1:28384}]" +50:05.307 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28384 [Follower]" leader= +50:05.307 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2b808f30-571d-10a7-1aa9-fc8b00324b44.dc1 127.0.0.1 +50:05.308 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2b808f30-571d-10a7-1aa9-fc8b00324b44 127.0.0.1 +2022-12-04T05:50:05.308Z [INFO] agent.router: Initializing LAN area manager +50:05.308 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2b808f30-571d-10a7-1aa9-fc8b00324b44.dc1 area=wan +50:05.308 [INFO] TestAgent.server: Adding LAN server: server="Node-2b808f30-571d-10a7-1aa9-fc8b00324b44 (Addr: tcp/127.0.0.1:28384) (DC: dc1)" +50:05.308 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=udp +50:05.308 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=tcp +50:05.309 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28380 network=tcp +50:05.309 [INFO] TestAgent: started state syncer +50:05.309 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28385 network=tcp +50:05.357 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:05.357 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28384 [Candidate]" term=2 +50:05.360 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:05.360 [DEBUG] TestAgent.server.raft: vote granted: from=2b808f30-571d-10a7-1aa9-fc8b00324b44 term=2 tally=1 +50:05.360 [INFO] TestAgent.server.raft: election won: tally=1 +50:05.360 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28384 [Leader]" +50:05.360 [INFO] TestAgent.server: cluster leadership acquired +50:05.360 [INFO] TestAgent.server: New leader elected: payload=Node-2b808f30-571d-10a7-1aa9-fc8b00324b44 +50:05.360 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28384 +50:05.361 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:05.377 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:05.377 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:05.377 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:05.377 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:05.377 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:05.378 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2b808f30-571d-10a7-1aa9-fc8b00324b44 +50:05.378 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2b808f30-571d-10a7-1aa9-fc8b00324b44 +50:05.378 [INFO] TestAgent.server: federation state anti-entropy synced +50:05.586 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:05.586 [INFO] TestAgent: Synced node info +50:05.586 [DEBUG] TestAgent: Node info in sync +50:05.727 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:54786 latency=1.083894ms +50:05.728 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/check?destination=db&source=foo&source-type=consul from=127.0.0.1:54788 latency=247.786µs +50:05.732 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/check?destination=db&source=web&source-type=consul from=127.0.0.1:54790 latency=2.565708ms +50:05.733 [INFO] TestAgent: Requesting shutdown +50:05.733 [INFO] TestAgent.server: shutting down server +50:05.733 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:05.733 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:05.733 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:05.733 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:05.733 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:05.733 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:05.733 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:05.733 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:05.733 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +50:05.733 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:05.733 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +2022-12-04T05:50:05.733Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:05.733Z [INFO] agent.router.manager: shutting down +50:05.733 [INFO] TestAgent: consul server down +50:05.733 [INFO] TestAgent: shutdown complete +50:05.733 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=tcp +50:05.733 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=udp +50:05.733 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28380 network=tcp +50:06.233 [INFO] TestAgent: Waiting for endpoints to shut down +50:06.233 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand (0.98s) PASS -ok github.com/hashicorp/consul/command/intention/check 0.919s +ok github.com/hashicorp/consul/command/intention/check 1.026s === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === RUN TestCommand_Validation @@ -85062,378 +85236,343 @@ === RUN TestCommand_replace === PAUSE TestCommand_replace === CONT TestCommand_noTabs ---- PASS: TestCommand_noTabs (0.00s) -=== CONT TestCommand_replace +=== CONT TestCommand_File +=== CONT TestCommand_deny [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestCommand_meta +=== CONT TestCommand_replace === CONT TestCommand_FileNoExist -=== CONT TestCommand_File +=== CONT TestCommand_meta === CONT TestCommand -=== CONT TestCommand_deny === CONT TestCommand_Validation +--- PASS: TestCommand_noTabs (0.00s) === RUN TestCommand_Validation/-allow_and_-deny --- PASS: TestCommand_Validation (0.00s) --- PASS: TestCommand_Validation/-allow_and_-deny (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:17:13.239Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:13.369Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:13.401 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:717b1ba1-1dee-4c39-19cc-cb16c27e689e Address:127.0.0.1:31468}]" -17:13.401 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31468 [Follower]" leader= -17:13.432 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-717b1ba1-1dee-4c39-19cc-cb16c27e689e.dc1 127.0.0.1 -17:13.433 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-717b1ba1-1dee-4c39-19cc-cb16c27e689e 127.0.0.1 -2021-10-31T23:17:13.433Z [INFO] agent.router: Initializing LAN area manager -17:13.433 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31463 network=udp -17:13.433 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-717b1ba1-1dee-4c39-19cc-cb16c27e689e.dc1 area=wan -17:13.433 [INFO] TestAgent.server: Adding LAN server: server="Node-717b1ba1-1dee-4c39-19cc-cb16c27e689e (Addr: tcp/127.0.0.1:31468) (DC: dc1)" -17:13.449 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:13.449 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31468 [Candidate]" term=2 -17:13.450 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:13.450 [DEBUG] TestAgent.server.raft: vote granted: from=717b1ba1-1dee-4c39-19cc-cb16c27e689e term=2 tally=1 -17:13.450 [INFO] TestAgent.server.raft: election won: tally=1 -17:13.450 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31468 [Leader]" -17:13.450 [INFO] TestAgent.server: cluster leadership acquired -17:13.450 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31468 -17:13.451 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:13.451 [INFO] TestAgent.server: New leader elected: payload=Node-717b1ba1-1dee-4c39-19cc-cb16c27e689e -17:13.452 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:13.452 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:13.452 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:13.452 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:13.452 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:13.452 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-717b1ba1-1dee-4c39-19cc-cb16c27e689e -17:13.452 [INFO] TestAgent.server: member joined, marking health alive: member=Node-717b1ba1-1dee-4c39-19cc-cb16c27e689e -17:13.453 [INFO] TestAgent.server: federation state anti-entropy synced -17:13.482 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31463 network=tcp -17:13.482 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31464 network=tcp -17:13.482 [INFO] TestAgent: started state syncer -17:13.482 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31469 network=tcp -17:13.483 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:13.483 [INFO] TestAgent: Synced node info -17:13.483 [DEBUG] TestAgent: Node info in sync -17:13.516 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:48162 latency=15.65012ms -17:13.524 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:48164 latency=3.685739ms -17:13.526 [INFO] TestAgent: Requesting shutdown -17:13.526 [INFO] TestAgent.server: shutting down server -17:13.526 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:13.526 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:13.526 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:13.526 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:13.526 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:13.526 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:13.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:13.526 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:13.526 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:13.526 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:13.526 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:13.526Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:13.526Z [INFO] agent.router.manager: shutting down -17:13.526 [INFO] TestAgent: consul server down -17:13.526 [INFO] TestAgent: shutdown complete -17:13.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31463 network=tcp -17:13.526 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31463 network=udp -17:13.526 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31464 network=tcp -17:14.027 [INFO] TestAgent: Waiting for endpoints to shut down -17:14.027 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_File (0.94s) -2021-10-31T23:17:13.256Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:13.413Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:13.426 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d719a4a5-c953-c276-a012-e027e0f5811c Address:127.0.0.1:31461}]" -17:13.427 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d719a4a5-c953-c276-a012-e027e0f5811c.dc1 127.0.0.1 -17:13.427 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d719a4a5-c953-c276-a012-e027e0f5811c 127.0.0.1 -2021-10-31T23:17:13.427Z [INFO] agent.router: Initializing LAN area manager -17:13.428 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31456 network=udp -17:13.428 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31461 [Follower]" leader= -17:13.428 [INFO] TestAgent.server: Adding LAN server: server="Node-d719a4a5-c953-c276-a012-e027e0f5811c (Addr: tcp/127.0.0.1:31461) (DC: dc1)" -17:13.428 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d719a4a5-c953-c276-a012-e027e0f5811c.dc1 area=wan -17:13.429 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31456 network=tcp -17:13.429 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31457 network=tcp -17:13.429 [INFO] TestAgent: started state syncer -17:13.429 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31462 network=tcp -17:13.501 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:13.501 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31461 [Candidate]" term=2 -17:13.502 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:13.502 [DEBUG] TestAgent.server.raft: vote granted: from=d719a4a5-c953-c276-a012-e027e0f5811c term=2 tally=1 -17:13.502 [INFO] TestAgent.server.raft: election won: tally=1 -17:13.502 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31461 [Leader]" -17:13.502 [INFO] TestAgent.server: cluster leadership acquired -17:13.502 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31461 -17:13.504 [INFO] TestAgent.server: New leader elected: payload=Node-d719a4a5-c953-c276-a012-e027e0f5811c -17:13.510 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:13.517 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:13.517 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:13.517 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:13.517 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:13.517 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:13.517 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d719a4a5-c953-c276-a012-e027e0f5811c -17:13.517 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d719a4a5-c953-c276-a012-e027e0f5811c -17:13.518 [INFO] TestAgent.server: federation state anti-entropy synced -17:13.639 [INFO] TestAgent: Requesting shutdown -17:13.639 [INFO] TestAgent.server: shutting down server -17:13.639 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:13.639 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:13.639 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:13.639 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:13.639 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:13.639 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:13.639 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:13.639 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:13.639 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:13.639 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:13.639 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:13.639 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:13.639Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:13.639Z [INFO] agent.router.manager: shutting down -17:13.639 [INFO] TestAgent: consul server down -17:13.639 [INFO] TestAgent: shutdown complete -17:13.639 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31456 network=tcp -17:13.639 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31456 network=udp -17:13.639 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31457 network=tcp -17:14.140 [INFO] TestAgent: Waiting for endpoints to shut down -17:14.140 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_FileNoExist (1.05s) -2021-10-31T23:17:13.282Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:13.323Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:13.372 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:219ba6b1-a882-ef2e-311f-465eadd7e162 Address:127.0.0.1:31482}]" -17:13.373 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-219ba6b1-a882-ef2e-311f-465eadd7e162.dc1 127.0.0.1 -17:13.374 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-219ba6b1-a882-ef2e-311f-465eadd7e162 127.0.0.1 -2021-10-31T23:17:13.374Z [INFO] agent.router: Initializing LAN area manager -17:13.374 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31477 network=udp -17:13.375 [INFO] TestAgent.server: Adding LAN server: server="Node-219ba6b1-a882-ef2e-311f-465eadd7e162 (Addr: tcp/127.0.0.1:31482) (DC: dc1)" -17:13.375 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31477 network=tcp -17:13.375 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31478 network=tcp -17:13.375 [INFO] TestAgent: started state syncer -17:13.375 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31483 network=tcp -17:13.376 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31482 [Follower]" leader= -17:13.376 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-219ba6b1-a882-ef2e-311f-465eadd7e162.dc1 area=wan -17:13.428 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:13.428 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31482 [Candidate]" term=2 -17:13.428 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:13.428 [DEBUG] TestAgent.server.raft: vote granted: from=219ba6b1-a882-ef2e-311f-465eadd7e162 term=2 tally=1 -17:13.428 [INFO] TestAgent.server.raft: election won: tally=1 -17:13.428 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31482 [Leader]" -17:13.429 [INFO] TestAgent.server: cluster leadership acquired -17:13.430 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31482 -17:13.431 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:13.431 [INFO] TestAgent.server: New leader elected: payload=Node-219ba6b1-a882-ef2e-311f-465eadd7e162 -17:13.432 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:13.432 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:13.432 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:13.432 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:13.432 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:13.432 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-219ba6b1-a882-ef2e-311f-465eadd7e162 -17:13.432 [INFO] TestAgent.server: member joined, marking health alive: member=Node-219ba6b1-a882-ef2e-311f-465eadd7e162 -17:13.433 [INFO] TestAgent.server: federation state anti-entropy synced -17:13.637 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:36858 latency=1.884991ms -17:13.644 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:36860 latency=286.225µs -17:13.645 [INFO] TestAgent: Requesting shutdown -17:13.645 [INFO] TestAgent.server: shutting down server -17:13.645 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:13.645 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:13.645 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:13.645 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:13.645 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:13.645 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:13.645 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:13.645 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:13.645 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:13.645 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:13.645 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:13.645 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:13.645Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:13.645Z [INFO] agent.router.manager: shutting down -17:13.645 [INFO] TestAgent: consul server down -17:13.645 [INFO] TestAgent: shutdown complete -17:13.645 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31477 network=tcp -17:13.646 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31477 network=udp -17:13.646 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31478 network=tcp -17:14.146 [INFO] TestAgent: Waiting for endpoints to shut down -17:14.146 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_deny (1.06s) -2021-10-31T23:17:13.160Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:13.235Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:13.256 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dd9b7ed6-c3f9-55ba-3098-15338d7f9339 Address:127.0.0.1:31447}]" -17:13.257 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dd9b7ed6-c3f9-55ba-3098-15338d7f9339.dc1 127.0.0.1 -17:13.258 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= -17:13.262 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dd9b7ed6-c3f9-55ba-3098-15338d7f9339 127.0.0.1 -2021-10-31T23:17:13.262Z [INFO] agent.router: Initializing LAN area manager -17:13.264 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=udp -17:13.264 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dd9b7ed6-c3f9-55ba-3098-15338d7f9339.dc1 area=wan -17:13.264 [INFO] TestAgent.server: Adding LAN server: server="Node-dd9b7ed6-c3f9-55ba-3098-15338d7f9339 (Addr: tcp/127.0.0.1:31447) (DC: dc1)" -17:13.264 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=tcp -17:13.314 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:13.314 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 -17:13.337 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31443 network=tcp -17:13.362 [INFO] TestAgent: started state syncer -17:13.359 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:13.362 [DEBUG] TestAgent.server.raft: vote granted: from=dd9b7ed6-c3f9-55ba-3098-15338d7f9339 term=2 tally=1 -17:13.362 [INFO] TestAgent.server.raft: election won: tally=1 -17:13.362 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" -17:13.376 [INFO] TestAgent: Synced node info -17:13.376 [DEBUG] TestAgent: Node info in sync -17:13.376 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31448 network=tcp -17:13.376 [INFO] TestAgent.server: cluster leadership acquired -17:13.377 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31447 -17:13.377 [INFO] TestAgent.server: New leader elected: payload=Node-dd9b7ed6-c3f9-55ba-3098-15338d7f9339 -17:13.399 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:13.439 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:13.439 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:13.439 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:13.439 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:13.439 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:13.439 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dd9b7ed6-c3f9-55ba-3098-15338d7f9339 -17:13.439 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dd9b7ed6-c3f9-55ba-3098-15338d7f9339 -17:13.440 [INFO] TestAgent.server: federation state anti-entropy synced -17:13.629 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:54934 latency=1.398893ms -17:13.630 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:54936 latency=293.764µs -17:13.632 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions from=127.0.0.1:54938 error="duplicate intention found: ALLOW default/foo => default/bar (ID: 5fe2b84c-3372-94c8-8d8b-1b7ed22a8d00, Precedence: 9)" -17:13.632 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:54938 latency=518.669µs -17:13.633 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:54940 latency=249.271µs -17:13.643 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions/5fe2b84c-3372-94c8-8d8b-1b7ed22a8d00 from=127.0.0.1:54940 latency=886.097µs -17:13.646 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:54936 latency=252.137µs -17:13.646 [INFO] TestAgent: Requesting shutdown -17:13.646 [INFO] TestAgent.server: shutting down server -17:13.646 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:13.646 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:13.646 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:13.646 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:13.646 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:13.646 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:13.646 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:13.646 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:13.646 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:13.646 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:13.646 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:13.647Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:13.647Z [INFO] agent.router.manager: shutting down -17:13.647 [INFO] TestAgent: consul server down -17:13.647 [INFO] TestAgent: shutdown complete -17:13.647 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp -17:13.647 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp -17:13.647 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp -17:14.147 [INFO] TestAgent: Waiting for endpoints to shut down -17:14.147 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_replace (1.06s) -2021-10-31T23:17:13.314Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:13.501Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:13.515 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:93bcde68-ec16-570d-c454-114b0626632b Address:127.0.0.1:31475}]" -17:13.517 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-93bcde68-ec16-570d-c454-114b0626632b.dc1 127.0.0.1 -17:13.517 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31475 [Follower]" leader= -17:13.519 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-93bcde68-ec16-570d-c454-114b0626632b 127.0.0.1 -2021-10-31T23:17:13.519Z [INFO] agent.router: Initializing LAN area manager -17:13.520 [INFO] TestAgent.server: Adding LAN server: server="Node-93bcde68-ec16-570d-c454-114b0626632b (Addr: tcp/127.0.0.1:31475) (DC: dc1)" -17:13.520 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31470 network=udp -17:13.520 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-93bcde68-ec16-570d-c454-114b0626632b.dc1 area=wan -17:13.521 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31470 network=tcp -17:13.525 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31471 network=tcp -17:13.525 [INFO] TestAgent: started state syncer -17:13.525 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31476 network=tcp -17:13.586 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:13.586 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31475 [Candidate]" term=2 -17:13.586 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:13.586 [DEBUG] TestAgent.server.raft: vote granted: from=93bcde68-ec16-570d-c454-114b0626632b term=2 tally=1 -17:13.586 [INFO] TestAgent.server.raft: election won: tally=1 -17:13.586 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31475 [Leader]" -17:13.586 [INFO] TestAgent.server: cluster leadership acquired -17:13.586 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31475 -17:13.587 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:13.589 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:13.589 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:13.589 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:13.589 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:13.589 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:13.589 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-93bcde68-ec16-570d-c454-114b0626632b -17:13.589 [INFO] TestAgent.server: member joined, marking health alive: member=Node-93bcde68-ec16-570d-c454-114b0626632b -17:13.589 [INFO] TestAgent.server: New leader elected: payload=Node-93bcde68-ec16-570d-c454-114b0626632b -17:13.590 [INFO] TestAgent.server: federation state anti-entropy synced -17:13.767 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:42954 latency=4.957236ms -17:13.776 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:42956 latency=1.661755ms -17:13.782 [INFO] TestAgent: Requesting shutdown -17:13.782 [INFO] TestAgent.server: shutting down server -17:13.782 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:13.782 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:13.782 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:13.782 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:13.782 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:13.782 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:13.783 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:13.783 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:13.783 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:13.783 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:13.783 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:13.783 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:13.783Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:13.783Z [INFO] agent.router.manager: shutting down -17:13.783 [INFO] TestAgent: consul server down -17:13.783 [INFO] TestAgent: shutdown complete -17:13.783 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31470 network=tcp -17:13.783 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31470 network=udp -17:13.783 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31471 network=tcp -17:14.284 [INFO] TestAgent: Waiting for endpoints to shut down -17:14.284 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand (1.20s) -2021-10-31T23:17:13.179Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:13.328Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:13.374 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aa93b9c7-ab19-93db-cd8d-c53bb8fb7182 Address:127.0.0.1:31454}]" -17:13.376 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31454 [Follower]" leader= -17:13.376 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aa93b9c7-ab19-93db-cd8d-c53bb8fb7182.dc1 127.0.0.1 -17:13.398 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aa93b9c7-ab19-93db-cd8d-c53bb8fb7182 127.0.0.1 -2021-10-31T23:17:13.398Z [INFO] agent.router: Initializing LAN area manager -17:13.399 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31449 network=udp -17:13.399 [INFO] TestAgent.server: Adding LAN server: server="Node-aa93b9c7-ab19-93db-cd8d-c53bb8fb7182 (Addr: tcp/127.0.0.1:31454) (DC: dc1)" -17:13.399 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aa93b9c7-ab19-93db-cd8d-c53bb8fb7182.dc1 area=wan -17:13.399 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31449 network=tcp -17:13.400 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31450 network=tcp -17:13.401 [INFO] TestAgent: started state syncer -17:13.401 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31455 network=tcp -17:13.431 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:13.431 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31454 [Candidate]" term=2 -17:13.431 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:13.431 [DEBUG] TestAgent.server.raft: vote granted: from=aa93b9c7-ab19-93db-cd8d-c53bb8fb7182 term=2 tally=1 -17:13.431 [INFO] TestAgent.server.raft: election won: tally=1 -17:13.431 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31454 [Leader]" -17:13.431 [INFO] TestAgent.server: cluster leadership acquired -17:13.431 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31454 -17:13.432 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:13.434 [INFO] TestAgent.server: New leader elected: payload=Node-aa93b9c7-ab19-93db-cd8d-c53bb8fb7182 -17:13.442 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:13.442 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:13.442 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:13.442 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:13.442 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:13.443 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aa93b9c7-ab19-93db-cd8d-c53bb8fb7182 -17:13.443 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aa93b9c7-ab19-93db-cd8d-c53bb8fb7182 -17:13.443 [INFO] TestAgent.server: federation state anti-entropy synced -17:13.598 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:13.598 [INFO] TestAgent: Synced node info -17:13.598 [DEBUG] TestAgent: Node info in sync -17:13.842 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:45732 latency=754.9µs -17:13.843 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:45734 latency=227.88µs -17:13.845 [INFO] TestAgent: Requesting shutdown -17:13.845 [INFO] TestAgent.server: shutting down server -17:13.845 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:13.845 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:13.845 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:13.845 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:13.845 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:13.845 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:13.845 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:13.845 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:13.845 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:13.845 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:13.845 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:13.845Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:13.845Z [INFO] agent.router.manager: shutting down -17:13.845 [INFO] TestAgent: consul server down -17:13.845 [INFO] TestAgent: shutdown complete -17:13.845 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31449 network=tcp -17:13.845 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31449 network=udp -17:13.845 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31450 network=tcp -17:14.346 [INFO] TestAgent: Waiting for endpoints to shut down -17:14.346 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_meta (1.26s) +2022-12-04T05:50:06.854Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:06.873Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:06.879 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:548e1896-a2fa-b007-db9a-f585382fe979 Address:127.0.0.1:29433}]" +50:06.879 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29433 [Follower]" leader= +50:06.880 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-548e1896-a2fa-b007-db9a-f585382fe979.dc1 127.0.0.1 +50:06.881 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-548e1896-a2fa-b007-db9a-f585382fe979 127.0.0.1 +2022-12-04T05:50:06.882Z [INFO] agent.router: Initializing LAN area manager +50:06.882 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29428 network=udp +50:06.882 [INFO] TestAgent.server: Adding LAN server: server="Node-548e1896-a2fa-b007-db9a-f585382fe979 (Addr: tcp/127.0.0.1:29433) (DC: dc1)" +50:06.882 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-548e1896-a2fa-b007-db9a-f585382fe979.dc1 area=wan +50:06.883 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29428 network=tcp +50:06.884 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29429 network=tcp +50:06.884 [INFO] TestAgent: started state syncer +50:06.884 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29434 network=tcp +50:06.940 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:06.940 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29433 [Candidate]" term=2 +50:06.940 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:06.940 [DEBUG] TestAgent.server.raft: vote granted: from=548e1896-a2fa-b007-db9a-f585382fe979 term=2 tally=1 +50:06.940 [INFO] TestAgent.server.raft: election won: tally=1 +50:06.940 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29433 [Leader]" +50:06.941 [INFO] TestAgent.server: cluster leadership acquired +50:06.942 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29433 +50:06.942 [INFO] TestAgent.server: New leader elected: payload=Node-548e1896-a2fa-b007-db9a-f585382fe979 +50:06.947 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:06.952 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:43796 latency=813.845µs +50:06.963 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:43798 latency=9.667847ms +50:06.965 [INFO] TestAgent: Requesting shutdown +50:06.965 [INFO] TestAgent.server: shutting down server +50:06.965 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:06.965 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:06.965 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:06.965Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:06.965Z [INFO] agent.router.manager: shutting down +50:06.965 [INFO] TestAgent: consul server down +50:06.965 [INFO] TestAgent: shutdown complete +50:06.965 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29428 network=tcp +50:06.965 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29428 network=udp +50:06.965 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29429 network=tcp +50:06.972 [ERROR] TestAgent.server: failed to establish leadership: error="error generating CA root certificate: error computing next serial number: raft is already shutdown" +50:07.466 [INFO] TestAgent: Waiting for endpoints to shut down +50:07.466 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_meta (0.66s) +2022-12-04T05:50:06.873Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:06.893Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:06.894 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:61b23eb2-662f-3108-8237-a7ecba7d7266 Address:127.0.0.1:29419}]" +50:06.895 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29419 [Follower]" leader= +50:06.895 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-61b23eb2-662f-3108-8237-a7ecba7d7266.dc1 127.0.0.1 +50:06.895 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-61b23eb2-662f-3108-8237-a7ecba7d7266 127.0.0.1 +2022-12-04T05:50:06.895Z [INFO] agent.router: Initializing LAN area manager +50:06.896 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29414 network=udp +50:06.896 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-61b23eb2-662f-3108-8237-a7ecba7d7266.dc1 area=wan +50:06.896 [INFO] TestAgent.server: Adding LAN server: server="Node-61b23eb2-662f-3108-8237-a7ecba7d7266 (Addr: tcp/127.0.0.1:29419) (DC: dc1)" +50:06.940 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:06.940 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29419 [Candidate]" term=2 +50:06.941 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:06.941 [DEBUG] TestAgent.server.raft: vote granted: from=61b23eb2-662f-3108-8237-a7ecba7d7266 term=2 tally=1 +50:06.941 [INFO] TestAgent.server.raft: election won: tally=1 +50:06.941 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29419 [Leader]" +50:06.940 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29414 network=tcp +50:06.941 [INFO] TestAgent.server: cluster leadership acquired +50:06.941 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29419 +50:06.942 [INFO] TestAgent.server: New leader elected: payload=Node-61b23eb2-662f-3108-8237-a7ecba7d7266 +50:06.944 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29415 network=tcp +50:06.944 [INFO] TestAgent: started state syncer +50:06.945 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:06.947 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29420 network=tcp +50:06.949 [INFO] TestAgent: Synced node info +50:06.949 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:43892 latency=1.786334ms +50:06.953 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:43896 latency=1.630461ms +50:06.965 [ERROR] TestAgent.http: Request error: method=POST url=/v1/connect/intentions from=127.0.0.1:43898 error="duplicate intention found: ALLOW default/foo => default/bar (ID: a03d462c-001a-ef2e-723f-8c8c065258d8, Precedence: 9)" +50:06.966 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:43898 latency=1.249294ms +50:06.967 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:43900 latency=232.201µs +50:06.968 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/connect/intentions/a03d462c-001a-ef2e-723f-8c8c065258d8 from=127.0.0.1:43900 latency=446.649µs +50:06.969 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:43896 latency=831.589µs +50:06.970 [INFO] TestAgent: Requesting shutdown +50:06.970 [INFO] TestAgent.server: shutting down server +50:06.970 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:06.970 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:06.970Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:06.970Z [INFO] agent.router.manager: shutting down +50:06.970 [INFO] TestAgent: consul server down +50:06.970 [INFO] TestAgent: shutdown complete +50:06.970 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29414 network=tcp +50:06.970 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29414 network=udp +50:06.970 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29415 network=tcp +50:06.971 [ERROR] TestAgent.server: failed to establish leadership: error="error generating CA root certificate: error computing next serial number: raft is already shutdown" +50:07.470 [INFO] TestAgent: Waiting for endpoints to shut down +50:07.470 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_replace (0.67s) +2022-12-04T05:50:06.873Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:06.947Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:06.949 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6063c50a-3c99-9af2-2106-6bd65dd97c58 Address:127.0.0.1:29440}]" +50:06.949 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29440 [Follower]" leader= +50:06.951 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6063c50a-3c99-9af2-2106-6bd65dd97c58.dc1 127.0.0.1 +50:06.952 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6063c50a-3c99-9af2-2106-6bd65dd97c58 127.0.0.1 +2022-12-04T05:50:06.952Z [INFO] agent.router: Initializing LAN area manager +50:06.953 [INFO] TestAgent.server: Adding LAN server: server="Node-6063c50a-3c99-9af2-2106-6bd65dd97c58 (Addr: tcp/127.0.0.1:29440) (DC: dc1)" +50:06.953 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6063c50a-3c99-9af2-2106-6bd65dd97c58.dc1 area=wan +50:06.953 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29435 network=udp +50:06.953 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29435 network=tcp +50:06.954 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29436 network=tcp +50:06.954 [INFO] TestAgent: started state syncer +50:06.955 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29441 network=tcp +50:07.006 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:07.006 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29440 [Candidate]" term=2 +50:07.006 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:07.006 [DEBUG] TestAgent.server.raft: vote granted: from=6063c50a-3c99-9af2-2106-6bd65dd97c58 term=2 tally=1 +50:07.006 [INFO] TestAgent.server.raft: election won: tally=1 +50:07.006 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29440 [Leader]" +50:07.006 [INFO] TestAgent.server: cluster leadership acquired +50:07.007 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29440 +50:07.007 [INFO] TestAgent.server: New leader elected: payload=Node-6063c50a-3c99-9af2-2106-6bd65dd97c58 +50:07.007 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:07.008 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:07.008 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:07.008 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:07.008 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:07.008 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:07.008 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6063c50a-3c99-9af2-2106-6bd65dd97c58 +50:07.009 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6063c50a-3c99-9af2-2106-6bd65dd97c58 +50:07.009 [INFO] TestAgent.server: federation state anti-entropy synced +50:07.012 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:58894 latency=673.749µs +50:07.013 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:58896 latency=190.753µs +50:07.014 [INFO] TestAgent: Requesting shutdown +50:07.014 [INFO] TestAgent.server: shutting down server +50:07.014 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:07.014 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:07.014 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:07.014 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:07.014 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:07.014 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:07.014 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:07.014 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:07.014 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +50:07.014 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:07.014 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:07.014 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:50:07.014Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:07.014Z [INFO] agent.router.manager: shutting down +50:07.014 [INFO] TestAgent: consul server down +50:07.014 [INFO] TestAgent: shutdown complete +50:07.014 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29435 network=tcp +50:07.014 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29435 network=udp +50:07.014 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29436 network=tcp +50:07.516 [INFO] TestAgent: Waiting for endpoints to shut down +50:07.516 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand (0.71s) +2022-12-04T05:50:06.857Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:06.878Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:06.881 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:098ac438-fe16-3d11-3191-154c783eb6c3 Address:127.0.0.1:29412}]" +50:06.884 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29412 [Follower]" leader= +50:06.888 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-098ac438-fe16-3d11-3191-154c783eb6c3.dc1 127.0.0.1 +50:06.888 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-098ac438-fe16-3d11-3191-154c783eb6c3 127.0.0.1 +2022-12-04T05:50:06.888Z [INFO] agent.router: Initializing LAN area manager +50:06.888 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29407 network=udp +50:06.888 [INFO] TestAgent.server: Adding LAN server: server="Node-098ac438-fe16-3d11-3191-154c783eb6c3 (Addr: tcp/127.0.0.1:29412) (DC: dc1)" +50:06.889 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-098ac438-fe16-3d11-3191-154c783eb6c3.dc1 area=wan +50:06.889 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29407 network=tcp +50:06.889 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29408 network=tcp +50:06.889 [INFO] TestAgent: started state syncer +50:06.889 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29413 network=tcp +50:06.943 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:06.943 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29412 [Candidate]" term=2 +50:06.947 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:06.947 [DEBUG] TestAgent.server.raft: vote granted: from=098ac438-fe16-3d11-3191-154c783eb6c3 term=2 tally=1 +50:06.947 [INFO] TestAgent.server.raft: election won: tally=1 +50:06.947 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29412 [Leader]" +50:06.947 [INFO] TestAgent.server: cluster leadership acquired +50:06.948 [INFO] TestAgent.server: New leader elected: payload=Node-098ac438-fe16-3d11-3191-154c783eb6c3 +50:06.948 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29412 +50:06.950 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:06.973 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:06.973 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:06.973 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:06.973 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:06.973 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:06.973 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-098ac438-fe16-3d11-3191-154c783eb6c3 +50:06.973 [INFO] TestAgent.server: member joined, marking health alive: member=Node-098ac438-fe16-3d11-3191-154c783eb6c3 +50:06.974 [INFO] TestAgent.server: federation state anti-entropy synced +50:07.024 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:34424 latency=609.539µs +50:07.025 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:34426 latency=259.785µs +50:07.025 [INFO] TestAgent: Requesting shutdown +50:07.025 [INFO] TestAgent.server: shutting down server +50:07.025 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:07.026 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:07.026 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:07.026 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:07.026 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:07.026 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:07.026 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:07.026 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:07.026 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:07.026 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:07.026 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:07.026 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:07.026Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:07.026Z [INFO] agent.router.manager: shutting down +50:07.026 [INFO] TestAgent: consul server down +50:07.026 [INFO] TestAgent: shutdown complete +50:07.026 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29407 network=tcp +50:07.026 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29407 network=udp +50:07.026 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29408 network=tcp +50:07.528 [INFO] TestAgent: Waiting for endpoints to shut down +50:07.528 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_File (0.72s) +2022-12-04T05:50:06.852Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:06.873Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:06.879 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d3be08ba-64fb-1aeb-0db6-830be6723e6f Address:127.0.0.1:29426}]" +50:06.879 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29426 [Follower]" leader= +50:06.880 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d3be08ba-64fb-1aeb-0db6-830be6723e6f.dc1 127.0.0.1 +50:06.881 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d3be08ba-64fb-1aeb-0db6-830be6723e6f 127.0.0.1 +2022-12-04T05:50:06.881Z [INFO] agent.router: Initializing LAN area manager +50:06.884 [INFO] TestAgent.server: Adding LAN server: server="Node-d3be08ba-64fb-1aeb-0db6-830be6723e6f (Addr: tcp/127.0.0.1:29426) (DC: dc1)" +50:06.884 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d3be08ba-64fb-1aeb-0db6-830be6723e6f.dc1 area=wan +50:06.885 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29421 network=udp +50:06.885 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29421 network=tcp +50:06.890 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29422 network=tcp +50:06.890 [INFO] TestAgent: started state syncer +50:06.890 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29427 network=tcp +50:06.941 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:06.941 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29426 [Candidate]" term=2 +50:06.941 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:06.941 [DEBUG] TestAgent.server.raft: vote granted: from=d3be08ba-64fb-1aeb-0db6-830be6723e6f term=2 tally=1 +50:06.941 [INFO] TestAgent.server.raft: election won: tally=1 +50:06.941 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29426 [Leader]" +50:06.941 [INFO] TestAgent.server: cluster leadership acquired +50:06.942 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29426 +50:06.942 [INFO] TestAgent.server: New leader elected: payload=Node-d3be08ba-64fb-1aeb-0db6-830be6723e6f +50:06.946 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:06.973 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:06.973 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:06.973 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:06.973 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:06.973 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:06.973 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d3be08ba-64fb-1aeb-0db6-830be6723e6f +50:06.973 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d3be08ba-64fb-1aeb-0db6-830be6723e6f +50:06.973 [INFO] TestAgent.server: federation state anti-entropy synced +50:07.183 [INFO] TestAgent: Requesting shutdown +50:07.183 [INFO] TestAgent.server: shutting down server +50:07.183 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:07.184 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:07.184 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:07.184 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:07.184 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:07.184 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:07.184 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:07.184 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:07.184 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:07.184 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:07.184 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:07.184 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:07.184Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:07.184Z [INFO] agent.router.manager: shutting down +50:07.184 [INFO] TestAgent: consul server down +50:07.184 [INFO] TestAgent: shutdown complete +50:07.184 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29421 network=tcp +50:07.184 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29421 network=udp +50:07.184 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29422 network=tcp +50:07.688 [INFO] TestAgent: Waiting for endpoints to shut down +50:07.688 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_FileNoExist (0.88s) +2022-12-04T05:50:06.864Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:06.889Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:06.890 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:facc67d1-ecb9-95ce-06ed-9bcbfc4e8e3d Address:127.0.0.1:29405}]" +50:06.890 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29405 [Follower]" leader= +50:06.891 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-facc67d1-ecb9-95ce-06ed-9bcbfc4e8e3d.dc1 127.0.0.1 +50:06.891 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-facc67d1-ecb9-95ce-06ed-9bcbfc4e8e3d 127.0.0.1 +2022-12-04T05:50:06.891Z [INFO] agent.router: Initializing LAN area manager +50:06.891 [INFO] TestAgent.server: Adding LAN server: server="Node-facc67d1-ecb9-95ce-06ed-9bcbfc4e8e3d (Addr: tcp/127.0.0.1:29405) (DC: dc1)" +50:06.891 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=udp +50:06.891 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-facc67d1-ecb9-95ce-06ed-9bcbfc4e8e3d.dc1 area=wan +50:06.891 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=tcp +50:06.892 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29401 network=tcp +50:06.892 [INFO] TestAgent: started state syncer +50:06.892 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29406 network=tcp +50:06.940 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:06.940 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29405 [Candidate]" term=2 +50:06.940 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:06.940 [DEBUG] TestAgent.server.raft: vote granted: from=facc67d1-ecb9-95ce-06ed-9bcbfc4e8e3d term=2 tally=1 +50:06.940 [INFO] TestAgent.server.raft: election won: tally=1 +50:06.940 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29405 [Leader]" +50:06.941 [INFO] TestAgent.server: cluster leadership acquired +50:06.941 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29405 +50:06.942 [INFO] TestAgent.server: New leader elected: payload=Node-facc67d1-ecb9-95ce-06ed-9bcbfc4e8e3d +50:06.948 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:06.974 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:06.974 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:06.974 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:06.974 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:06.974 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:06.974 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-facc67d1-ecb9-95ce-06ed-9bcbfc4e8e3d +50:06.974 [INFO] TestAgent.server: member joined, marking health alive: member=Node-facc67d1-ecb9-95ce-06ed-9bcbfc4e8e3d +50:06.974 [INFO] TestAgent.server: federation state anti-entropy synced +50:07.073 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:07.074 [INFO] TestAgent: Synced node info +50:07.074 [DEBUG] TestAgent: Node info in sync +50:07.317 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:56164 latency=500.702µs +50:07.317 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:56166 latency=125.483µs +50:07.318 [INFO] TestAgent: Requesting shutdown +50:07.318 [INFO] TestAgent.server: shutting down server +50:07.318 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:07.318 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:07.318 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:07.318 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:07.318 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:07.318 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:07.318 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:07.318 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:07.318 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:07.318 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:07.318 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:07.318Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:07.318Z [INFO] agent.router.manager: shutting down +50:07.318 [INFO] TestAgent: consul server down +50:07.318 [INFO] TestAgent: shutdown complete +50:07.318 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=tcp +50:07.318 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=udp +50:07.318 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29401 network=tcp +50:07.818 [INFO] TestAgent: Waiting for endpoints to shut down +50:07.818 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_deny (1.02s) PASS -ok github.com/hashicorp/consul/command/intention/create 1.365s +ok github.com/hashicorp/consul/command/intention/create 1.073s === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === RUN TestCommand_Validation @@ -85443,143 +85582,144 @@ === CONT TestCommand_noTabs === CONT TestCommand === CONT TestCommand_Validation ---- PASS: TestCommand_noTabs (0.00s) [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestCommand_noTabs (0.00s) === RUN TestCommand_Validation/0_args +[INFO] freeport: detected ephemeral port range of [32768, 60999] +[INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestCommand_Validation/3_args --- PASS: TestCommand_Validation (0.00s) --- PASS: TestCommand_Validation/0_args (0.00s) --- PASS: TestCommand_Validation/3_args (0.00s) -[INFO] freeport: detected ephemeral port range of [32768, 60999] -[INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:17:21.767Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:21.822Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:21.836 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d84e9e25-5d54-3c28-2cd4-cccc289988ac Address:127.0.0.1:18174}]" -17:21.838 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18174 [Follower]" leader= -17:21.838 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d84e9e25-5d54-3c28-2cd4-cccc289988ac.dc1 127.0.0.1 -17:21.839 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d84e9e25-5d54-3c28-2cd4-cccc289988ac 127.0.0.1 -2021-10-31T23:17:21.839Z [INFO] agent.router: Initializing LAN area manager -17:21.840 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=udp -17:21.840 [INFO] TestAgent.server: Adding LAN server: server="Node-d84e9e25-5d54-3c28-2cd4-cccc289988ac (Addr: tcp/127.0.0.1:18174) (DC: dc1)" -17:21.840 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d84e9e25-5d54-3c28-2cd4-cccc289988ac.dc1 area=wan -17:21.866 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=tcp -17:21.866 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18170 network=tcp -17:21.866 [INFO] TestAgent: started state syncer -17:21.866 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18175 network=tcp -17:21.877 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:21.877 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18174 [Candidate]" term=2 -17:21.878 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:21.878 [DEBUG] TestAgent.server.raft: vote granted: from=d84e9e25-5d54-3c28-2cd4-cccc289988ac term=2 tally=1 -17:21.878 [INFO] TestAgent.server.raft: election won: tally=1 -17:21.878 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18174 [Leader]" -17:21.878 [INFO] TestAgent.server: cluster leadership acquired -17:21.878 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18174 -17:21.879 [INFO] TestAgent.server: New leader elected: payload=Node-d84e9e25-5d54-3c28-2cd4-cccc289988ac -17:21.879 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:21.945 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:21.945 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:21.945 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:21.945 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:21.945 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:21.946 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d84e9e25-5d54-3c28-2cd4-cccc289988ac -17:21.946 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d84e9e25-5d54-3c28-2cd4-cccc289988ac -17:21.947 [INFO] TestAgent.server: federation state anti-entropy synced -17:22.036 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:46518 latency=1.022578ms -17:22.042 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:46520 latency=706.151µs -17:22.047 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions/353a8284-67b7-a44f-51b5-ed0eec26f1e3 from=127.0.0.1:46520 latency=742.845µs -17:22.048 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:46518 latency=773.616µs -17:22.048 [INFO] TestAgent: Requesting shutdown -17:22.048 [INFO] TestAgent.server: shutting down server -17:22.048 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:22.048 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:22.048 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:22.048 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:22.048 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:22.048 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:22.048 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:22.049 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:22.049 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:22.049 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:22.049 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:22.049 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:22.049Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:22.049Z [INFO] agent.router.manager: shutting down -17:22.049 [INFO] TestAgent: consul server down -17:22.049 [INFO] TestAgent: shutdown complete -17:22.049 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=tcp -17:22.049 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=udp -17:22.049 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18170 network=tcp -17:22.549 [INFO] TestAgent: Waiting for endpoints to shut down -17:22.549 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand (0.93s) +2022-12-04T05:50:08.446Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:08.466Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:08.468 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e26d7ba1-3c8d-f958-bbc9-54fdc7a66764 Address:127.0.0.1:29405}]" +50:08.468 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29405 [Follower]" leader= +50:08.471 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e26d7ba1-3c8d-f958-bbc9-54fdc7a66764.dc1 127.0.0.1 +50:08.472 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e26d7ba1-3c8d-f958-bbc9-54fdc7a66764 127.0.0.1 +2022-12-04T05:50:08.472Z [INFO] agent.router: Initializing LAN area manager +50:08.472 [INFO] TestAgent.server: Adding LAN server: server="Node-e26d7ba1-3c8d-f958-bbc9-54fdc7a66764 (Addr: tcp/127.0.0.1:29405) (DC: dc1)" +50:08.472 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e26d7ba1-3c8d-f958-bbc9-54fdc7a66764.dc1 area=wan +50:08.472 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=udp +50:08.472 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=tcp +50:08.473 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29401 network=tcp +50:08.473 [INFO] TestAgent: started state syncer +50:08.473 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29406 network=tcp +50:08.511 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:08.511 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29405 [Candidate]" term=2 +50:08.511 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:08.511 [DEBUG] TestAgent.server.raft: vote granted: from=e26d7ba1-3c8d-f958-bbc9-54fdc7a66764 term=2 tally=1 +50:08.511 [INFO] TestAgent.server.raft: election won: tally=1 +50:08.511 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29405 [Leader]" +50:08.511 [INFO] TestAgent.server: cluster leadership acquired +50:08.511 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29405 +50:08.512 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:08.518 [INFO] TestAgent.server: New leader elected: payload=Node-e26d7ba1-3c8d-f958-bbc9-54fdc7a66764 +50:08.534 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:08.534 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:08.534 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:08.534 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:08.534 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:08.534 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e26d7ba1-3c8d-f958-bbc9-54fdc7a66764 +50:08.534 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e26d7ba1-3c8d-f958-bbc9-54fdc7a66764 +50:08.535 [INFO] TestAgent.server: federation state anti-entropy synced +50:08.665 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:56168 latency=1.048245ms +50:08.668 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:56170 latency=400.916µs +50:08.671 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/connect/intentions/57c4cd29-cb82-92bd-1897-81c5186a1c4d from=127.0.0.1:56170 latency=2.198751ms +50:08.676 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:56168 latency=1.046972ms +50:08.677 [INFO] TestAgent: Requesting shutdown +50:08.677 [INFO] TestAgent.server: shutting down server +50:08.677 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:08.677 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:08.677 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:08.677 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:08.677 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:08.677 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:08.677 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:08.677 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:08.677 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:08.677 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:08.677 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:08.677 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:08.677Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:08.677Z [INFO] agent.router.manager: shutting down +50:08.677 [INFO] TestAgent: consul server down +50:08.677 [INFO] TestAgent: shutdown complete +50:08.677 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=tcp +50:08.677 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=udp +50:08.677 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29401 network=tcp +50:09.178 [INFO] TestAgent: Waiting for endpoints to shut down +50:09.178 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand (0.78s) PASS -ok github.com/hashicorp/consul/command/intention/delete 1.812s +ok github.com/hashicorp/consul/command/intention/delete 0.829s === RUN TestFinder === PAUSE TestFinder === CONT TestFinder [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:17:26.149Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:26.211Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:26.215 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d8a6a87d-9dbb-8d34-0218-5d1c471e1994 Address:127.0.0.1:16132}]" -17:26.215 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d8a6a87d-9dbb-8d34-0218-5d1c471e1994.dc1 127.0.0.1 -17:26.216 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d8a6a87d-9dbb-8d34-0218-5d1c471e1994 127.0.0.1 -2021-10-31T23:17:26.216Z [INFO] agent.router: Initializing LAN area manager -17:26.216 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=udp -17:26.216 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16132 [Follower]" leader= -17:26.217 [INFO] TestAgent.server: Adding LAN server: server="Node-d8a6a87d-9dbb-8d34-0218-5d1c471e1994 (Addr: tcp/127.0.0.1:16132) (DC: dc1)" -17:26.217 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d8a6a87d-9dbb-8d34-0218-5d1c471e1994.dc1 area=wan -17:26.217 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=tcp -17:26.218 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16128 network=tcp -17:26.218 [INFO] TestAgent: started state syncer -17:26.218 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16133 network=tcp -17:26.270 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:26.270 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16132 [Candidate]" term=2 -17:26.270 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:26.270 [DEBUG] TestAgent.server.raft: vote granted: from=d8a6a87d-9dbb-8d34-0218-5d1c471e1994 term=2 tally=1 -17:26.270 [INFO] TestAgent.server.raft: election won: tally=1 -17:26.270 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16132 [Leader]" -17:26.270 [INFO] TestAgent.server: cluster leadership acquired -17:26.270 [INFO] TestAgent.server: New leader elected: payload=Node-d8a6a87d-9dbb-8d34-0218-5d1c471e1994 -17:26.270 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16132 -17:26.271 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:26.305 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:26.305 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:26.305 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:26.305 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:26.307 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:26.307 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d8a6a87d-9dbb-8d34-0218-5d1c471e1994 -17:26.307 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d8a6a87d-9dbb-8d34-0218-5d1c471e1994 -17:26.356 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:26.357 [INFO] TestAgent: Synced node info -17:26.490 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:43618 latency=11.100023ms -17:26.492 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:43618 latency=1.227111ms -17:26.493 [INFO] TestAgent: Requesting shutdown -17:26.493 [INFO] TestAgent.server: shutting down server -17:26.493 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:26.493 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:26.493 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:26.493 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:26.493 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:26.494 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" -17:26.494 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:26.494 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:26.494 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:26.494 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:26.494 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:26.494Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:26.494Z [INFO] agent.router.manager: shutting down -17:26.494 [INFO] TestAgent: consul server down -17:26.494 [INFO] TestAgent: shutdown complete -17:26.494 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=tcp -17:26.494 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=udp -17:26.494 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16128 network=tcp -17:26.995 [INFO] TestAgent: Waiting for endpoints to shut down -17:26.995 [INFO] TestAgent: Endpoints down ---- PASS: TestFinder (0.94s) +2022-12-04T05:50:12.281Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:12.291Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:12.296 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d2c0acfc-2806-1c34-9911-1d5611bfffc8 Address:127.0.0.1:27363}]" +50:12.296 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= +50:12.297 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d2c0acfc-2806-1c34-9911-1d5611bfffc8.dc1 127.0.0.1 +50:12.297 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d2c0acfc-2806-1c34-9911-1d5611bfffc8 127.0.0.1 +2022-12-04T05:50:12.297Z [INFO] agent.router: Initializing LAN area manager +50:12.298 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d2c0acfc-2806-1c34-9911-1d5611bfffc8.dc1 area=wan +50:12.298 [INFO] TestAgent.server: Adding LAN server: server="Node-d2c0acfc-2806-1c34-9911-1d5611bfffc8 (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +50:12.298 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp +50:12.298 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp +50:12.298 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp +50:12.298 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp +50:12.298 [INFO] TestAgent: started state syncer +50:12.346 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:12.346 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 +50:12.346 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:12.346 [DEBUG] TestAgent.server.raft: vote granted: from=d2c0acfc-2806-1c34-9911-1d5611bfffc8 term=2 tally=1 +50:12.346 [INFO] TestAgent.server.raft: election won: tally=1 +50:12.346 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" +50:12.346 [INFO] TestAgent.server: cluster leadership acquired +50:12.346 [INFO] TestAgent.server: New leader elected: payload=Node-d2c0acfc-2806-1c34-9911-1d5611bfffc8 +50:12.346 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27363 +50:12.347 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:12.363 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:12.363 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:12.363 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:12.363 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:12.363 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:12.364 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d2c0acfc-2806-1c34-9911-1d5611bfffc8 +50:12.364 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d2c0acfc-2806-1c34-9911-1d5611bfffc8 +50:12.364 [INFO] TestAgent.server: federation state anti-entropy synced +50:12.453 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:12.454 [INFO] TestAgent: Synced node info +50:12.498 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:59694 latency=4.00193ms +50:12.500 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:59694 latency=1.062975ms +50:12.501 [INFO] TestAgent: Requesting shutdown +50:12.501 [INFO] TestAgent.server: shutting down server +50:12.501 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:12.501 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:12.501 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:12.501 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:12.501 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:12.501 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:12.501 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:12.501 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:12.501 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:12.501 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:12.501 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:12.501Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:12.501Z [INFO] agent.router.manager: shutting down +50:12.501 [INFO] TestAgent: consul server down +50:12.501 [INFO] TestAgent: shutdown complete +50:12.501 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp +50:12.501 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp +50:12.501 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp +50:13.001 [INFO] TestAgent: Waiting for endpoints to shut down +50:13.001 [INFO] TestAgent: Endpoints down +--- PASS: TestFinder (0.75s) PASS -ok github.com/hashicorp/consul/command/intention/finder 1.484s +ok github.com/hashicorp/consul/command/intention/finder 0.803s === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === RUN TestCommand_Validation @@ -85591,197 +85731,199 @@ === RUN TestCommand_verticalBar === PAUSE TestCommand_verticalBar === CONT TestCommand_noTabs ---- PASS: TestCommand_noTabs (0.02s) === CONT TestCommand_verticalBar +--- PASS: TestCommand_noTabs (0.00s) +=== CONT TestCommand_srcDst [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestCommand_id -=== CONT TestCommand_srcDst === CONT TestCommand_Validation === RUN TestCommand_Validation/0_args === RUN TestCommand_Validation/3_args +[INFO] freeport: detected ephemeral port range of [32768, 60999] +[INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range --- PASS: TestCommand_Validation (0.00s) --- PASS: TestCommand_Validation/0_args (0.00s) --- PASS: TestCommand_Validation/3_args (0.00s) -[INFO] freeport: detected ephemeral port range of [32768, 60999] -[INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:17:32.124Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:32.174Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:32.177 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fe1eb0dd-87ce-d622-d558-c70bce60996b Address:127.0.0.1:23286}]" -17:32.182 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23286 [Follower]" leader= -17:32.183 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fe1eb0dd-87ce-d622-d558-c70bce60996b.dc1 127.0.0.1 -17:32.183 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fe1eb0dd-87ce-d622-d558-c70bce60996b 127.0.0.1 -2021-10-31T23:17:32.183Z [INFO] agent.router: Initializing LAN area manager -17:32.184 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23281 network=udp -17:32.184 [INFO] TestAgent.server: Adding LAN server: server="Node-fe1eb0dd-87ce-d622-d558-c70bce60996b (Addr: tcp/127.0.0.1:23286) (DC: dc1)" -17:32.184 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fe1eb0dd-87ce-d622-d558-c70bce60996b.dc1 area=wan -17:32.184 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23281 network=tcp -17:32.185 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23282 network=tcp -17:32.185 [INFO] TestAgent: started state syncer -17:32.185 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23287 network=tcp -17:32.233 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:32.234 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23286 [Candidate]" term=2 -17:32.234 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:32.234 [DEBUG] TestAgent.server.raft: vote granted: from=fe1eb0dd-87ce-d622-d558-c70bce60996b term=2 tally=1 -17:32.234 [INFO] TestAgent.server.raft: election won: tally=1 -17:32.234 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23286 [Leader]" -17:32.234 [INFO] TestAgent.server: cluster leadership acquired -17:32.234 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23286 -17:32.234 [INFO] TestAgent.server: New leader elected: payload=Node-fe1eb0dd-87ce-d622-d558-c70bce60996b -17:32.235 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:32.277 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:32.277 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:32.277 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:32.277 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:32.277 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:32.278 [INFO] TestAgent.server: federation state anti-entropy synced -17:32.289 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fe1eb0dd-87ce-d622-d558-c70bce60996b -17:32.310 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fe1eb0dd-87ce-d622-d558-c70bce60996b -17:32.311 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:60744 latency=11.879239ms -17:32.344 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/c30c75dd-9985-97e6-a7ce-0b6aca38f866 from=127.0.0.1:60746 latency=1.8455ms -17:32.345 [INFO] TestAgent: Requesting shutdown -17:32.345 [INFO] TestAgent.server: shutting down server -17:32.345 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:32.345 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:32.345 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:32.345 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:32.345 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:32.345 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:32.345 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:32.345 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:32.345 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:32.345 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:32.345 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:32.345 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:32.345Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:32.345Z [INFO] agent.router.manager: shutting down -17:32.353 [INFO] TestAgent: consul server down -17:32.354 [INFO] TestAgent: shutdown complete -17:32.354 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23281 network=tcp -17:32.354 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23281 network=udp -17:32.354 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23282 network=tcp -17:32.856 [INFO] TestAgent: Waiting for endpoints to shut down -17:32.856 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_id (0.91s) -2021-10-31T23:17:32.124Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:32.216Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:32.218 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3a6cfe60-f083-06bf-43e3-7691e97d714d Address:127.0.0.1:23293}]" -17:32.226 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3a6cfe60-f083-06bf-43e3-7691e97d714d.dc1 127.0.0.1 -17:32.227 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3a6cfe60-f083-06bf-43e3-7691e97d714d 127.0.0.1 -2021-10-31T23:17:32.227Z [INFO] agent.router: Initializing LAN area manager -17:32.227 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23288 network=udp -17:32.227 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23293 [Follower]" leader= -17:32.228 [INFO] TestAgent.server: Adding LAN server: server="Node-3a6cfe60-f083-06bf-43e3-7691e97d714d (Addr: tcp/127.0.0.1:23293) (DC: dc1)" -17:32.228 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3a6cfe60-f083-06bf-43e3-7691e97d714d.dc1 area=wan -17:32.229 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23288 network=tcp -17:32.229 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23289 network=tcp -17:32.229 [INFO] TestAgent: started state syncer -17:32.229 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23294 network=tcp -17:32.273 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:32.273 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23293 [Candidate]" term=2 -17:32.274 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:32.274 [DEBUG] TestAgent.server.raft: vote granted: from=3a6cfe60-f083-06bf-43e3-7691e97d714d term=2 tally=1 -17:32.274 [INFO] TestAgent.server.raft: election won: tally=1 -17:32.274 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23293 [Leader]" -17:32.274 [INFO] TestAgent.server: cluster leadership acquired -17:32.274 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23293 -17:32.275 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:32.275 [INFO] TestAgent.server: New leader elected: payload=Node-3a6cfe60-f083-06bf-43e3-7691e97d714d -17:32.280 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:32.280 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:32.280 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:32.280 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:32.280 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:32.280 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3a6cfe60-f083-06bf-43e3-7691e97d714d -17:32.280 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3a6cfe60-f083-06bf-43e3-7691e97d714d -17:32.282 [INFO] TestAgent.server: federation state anti-entropy synced -17:32.566 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:39704 latency=990.266µs -17:32.571 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:39706 latency=366.439µs -17:32.572 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/2327bada-2d28-e6a5-dd28-4e445a921ce4 from=127.0.0.1:39706 latency=142.906µs -17:32.572 [INFO] TestAgent: Requesting shutdown -17:32.572 [INFO] TestAgent.server: shutting down server -17:32.572 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:32.572 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:32.572 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:32.572 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:32.572 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:32.573 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:32.573 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:32.573 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:32.573 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:32.573 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:32.573 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:32.573 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:32.573Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:32.573Z [INFO] agent.router.manager: shutting down -17:32.573 [INFO] TestAgent: consul server down -17:32.573 [INFO] TestAgent: shutdown complete -17:32.573 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23288 network=tcp -17:32.573 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23288 network=udp -17:32.573 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23289 network=tcp -17:33.073 [INFO] TestAgent: Waiting for endpoints to shut down -17:33.073 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_srcDst (1.13s) -2021-10-31T23:17:32.159Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:32.228Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:32.232 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8271d206-b3e6-d6a4-7dd4-033e19051ce6 Address:127.0.0.1:23279}]" -17:32.232 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8271d206-b3e6-d6a4-7dd4-033e19051ce6.dc1 127.0.0.1 -17:32.233 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8271d206-b3e6-d6a4-7dd4-033e19051ce6 127.0.0.1 -2021-10-31T23:17:32.233Z [INFO] agent.router: Initializing LAN area manager -17:32.233 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=udp -17:32.233 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:23279 [Follower]" leader= -17:32.233 [INFO] TestAgent.server: Adding LAN server: server="Node-8271d206-b3e6-d6a4-7dd4-033e19051ce6 (Addr: tcp/127.0.0.1:23279) (DC: dc1)" -17:32.233 [INFO] TestAgent: Started DNS server: address=127.0.0.1:23274 network=tcp -17:32.234 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8271d206-b3e6-d6a4-7dd4-033e19051ce6.dc1 area=wan -17:32.234 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:23275 network=tcp -17:32.234 [INFO] TestAgent: started state syncer -17:32.234 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:23280 network=tcp -17:32.278 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:32.278 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:23279 [Candidate]" term=2 -17:32.278 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:32.278 [DEBUG] TestAgent.server.raft: vote granted: from=8271d206-b3e6-d6a4-7dd4-033e19051ce6 term=2 tally=1 -17:32.278 [INFO] TestAgent.server.raft: election won: tally=1 -17:32.278 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:23279 [Leader]" -17:32.280 [INFO] TestAgent.server: cluster leadership acquired -17:32.281 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:23279 -17:32.282 [INFO] TestAgent.server: New leader elected: payload=Node-8271d206-b3e6-d6a4-7dd4-033e19051ce6 -17:32.286 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:32.289 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:32.289 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:32.289 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:32.289 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:32.289 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:32.309 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8271d206-b3e6-d6a4-7dd4-033e19051ce6 -17:32.310 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8271d206-b3e6-d6a4-7dd4-033e19051ce6 -17:32.310 [INFO] TestAgent.server: federation state anti-entropy synced -17:32.414 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:32.414 [INFO] TestAgent: Synced node info -17:32.635 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:43006 latency=822.85µs -17:32.637 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/16d8c093-9c18-498d-05a5-8caae2567167 from=127.0.0.1:43008 latency=305.392µs -17:32.637 [INFO] TestAgent: Requesting shutdown -17:32.637 [INFO] TestAgent.server: shutting down server -17:32.637 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:32.637 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:32.637 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:32.637 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:32.637 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:32.637 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:32.637 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:32.637 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:32.637 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:32.638 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:32.638Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:32.638Z [INFO] agent.router.manager: shutting down -17:32.638 [INFO] TestAgent: consul server down -17:32.638 [INFO] TestAgent: shutdown complete -17:32.638 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=tcp -17:32.638 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:23274 network=udp -17:32.638 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:23275 network=tcp -17:32.642 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:33.141 [INFO] TestAgent: Waiting for endpoints to shut down -17:33.142 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_verticalBar (1.20s) +2022-12-04T05:50:13.592Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:13.631Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:13.633 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6b0250fe-6362-4893-0d27-5df3b75bc574 Address:127.0.0.1:10013}]" +50:13.640 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10013 [Follower]" leader= +50:13.655 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6b0250fe-6362-4893-0d27-5df3b75bc574.dc1 127.0.0.1 +50:13.657 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6b0250fe-6362-4893-0d27-5df3b75bc574 127.0.0.1 +2022-12-04T05:50:13.657Z [INFO] agent.router: Initializing LAN area manager +50:13.658 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10008 network=udp +50:13.658 [INFO] TestAgent.server: Adding LAN server: server="Node-6b0250fe-6362-4893-0d27-5df3b75bc574 (Addr: tcp/127.0.0.1:10013) (DC: dc1)" +50:13.658 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6b0250fe-6362-4893-0d27-5df3b75bc574.dc1 area=wan +50:13.658 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10008 network=tcp +50:13.659 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10009 network=tcp +50:13.659 [INFO] TestAgent: started state syncer +50:13.659 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10014 network=tcp +50:13.701 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:13.701 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10013 [Candidate]" term=2 +50:13.702 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:13.702 [DEBUG] TestAgent.server.raft: vote granted: from=6b0250fe-6362-4893-0d27-5df3b75bc574 term=2 tally=1 +50:13.702 [INFO] TestAgent.server.raft: election won: tally=1 +50:13.702 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10013 [Leader]" +50:13.702 [INFO] TestAgent.server: cluster leadership acquired +50:13.702 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10013 +50:13.703 [INFO] TestAgent.server: New leader elected: payload=Node-6b0250fe-6362-4893-0d27-5df3b75bc574 +50:13.703 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:13.705 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:13.705 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:13.705 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:13.705 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:13.705 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:13.705 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6b0250fe-6362-4893-0d27-5df3b75bc574 +50:13.705 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6b0250fe-6362-4893-0d27-5df3b75bc574 +50:13.706 [INFO] TestAgent.server: federation state anti-entropy synced +50:13.773 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:13.774 [INFO] TestAgent: Synced node info +50:13.782 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:36474 latency=820.529µs +50:13.783 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/c8c7e7bf-f013-24d3-3d0a-aeaaa7a0717a from=127.0.0.1:36476 latency=277.339µs +50:13.784 [INFO] TestAgent: Requesting shutdown +50:13.784 [INFO] TestAgent.server: shutting down server +50:13.784 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:13.784 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:13.784 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:13.784 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:13.784 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:13.784 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:13.784 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:13.784 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:13.784 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:13.784 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:13.784 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:13.784Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:13.785Z [INFO] agent.router.manager: shutting down +50:13.785 [INFO] TestAgent: consul server down +50:13.785 [INFO] TestAgent: shutdown complete +50:13.785 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10008 network=tcp +50:13.785 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10008 network=udp +50:13.785 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10009 network=tcp +50:14.286 [INFO] TestAgent: Waiting for endpoints to shut down +50:14.286 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_verticalBar (0.74s) +2022-12-04T05:50:13.593Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:13.630Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:13.633 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0f05fa3b-afc1-4001-38d1-73dfa9e5f832 Address:127.0.0.1:10020}]" +50:13.640 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10020 [Follower]" leader= +50:13.652 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0f05fa3b-afc1-4001-38d1-73dfa9e5f832.dc1 127.0.0.1 +50:13.657 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0f05fa3b-afc1-4001-38d1-73dfa9e5f832 127.0.0.1 +2022-12-04T05:50:13.657Z [INFO] agent.router: Initializing LAN area manager +50:13.658 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10015 network=udp +50:13.658 [INFO] TestAgent.server: Adding LAN server: server="Node-0f05fa3b-afc1-4001-38d1-73dfa9e5f832 (Addr: tcp/127.0.0.1:10020) (DC: dc1)" +50:13.658 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10015 network=tcp +50:13.658 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0f05fa3b-afc1-4001-38d1-73dfa9e5f832.dc1 area=wan +50:13.659 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10016 network=tcp +50:13.659 [INFO] TestAgent: started state syncer +50:13.659 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10021 network=tcp +50:13.682 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:13.682 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10020 [Candidate]" term=2 +50:13.682 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:13.682 [DEBUG] TestAgent.server.raft: vote granted: from=0f05fa3b-afc1-4001-38d1-73dfa9e5f832 term=2 tally=1 +50:13.682 [INFO] TestAgent.server.raft: election won: tally=1 +50:13.682 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10020 [Leader]" +50:13.682 [INFO] TestAgent.server: cluster leadership acquired +50:13.683 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10020 +50:13.683 [INFO] TestAgent.server: New leader elected: payload=Node-0f05fa3b-afc1-4001-38d1-73dfa9e5f832 +50:13.683 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:13.703 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:13.703 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:13.703 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:13.703 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:13.703 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:13.703 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0f05fa3b-afc1-4001-38d1-73dfa9e5f832 +50:13.703 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0f05fa3b-afc1-4001-38d1-73dfa9e5f832 +50:13.705 [INFO] TestAgent.server: federation state anti-entropy synced +50:13.882 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:13.882 [INFO] TestAgent: Synced node info +50:13.913 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:56704 latency=1.185158ms +50:13.915 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/1ce1bb9a-eb0f-07db-6bb6-c2e91ee027f4 from=127.0.0.1:56706 latency=298.56µs +50:13.915 [INFO] TestAgent: Requesting shutdown +50:13.915 [INFO] TestAgent.server: shutting down server +50:13.915 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:13.915 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:13.915 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:13.915 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:13.915 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:13.915 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:13.915 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:13.915 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:13.915 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:13.915 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:13.915 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:13.915Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:13.915Z [INFO] agent.router.manager: shutting down +50:13.915 [INFO] TestAgent: consul server down +50:13.915 [INFO] TestAgent: shutdown complete +50:13.915 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10015 network=tcp +50:13.915 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10015 network=udp +50:13.916 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10016 network=tcp +50:14.416 [INFO] TestAgent: Waiting for endpoints to shut down +50:14.416 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_id (0.87s) +2022-12-04T05:50:13.593Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:13.632Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:13.657 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cbb9fcd1-683b-fd2c-86bd-33661620491d Address:127.0.0.1:10006}]" +50:13.657 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:10006 [Follower]" leader= +50:13.658 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cbb9fcd1-683b-fd2c-86bd-33661620491d.dc1 127.0.0.1 +50:13.659 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cbb9fcd1-683b-fd2c-86bd-33661620491d 127.0.0.1 +2022-12-04T05:50:13.659Z [INFO] agent.router: Initializing LAN area manager +50:13.660 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cbb9fcd1-683b-fd2c-86bd-33661620491d.dc1 area=wan +50:13.660 [INFO] TestAgent.server: Adding LAN server: server="Node-cbb9fcd1-683b-fd2c-86bd-33661620491d (Addr: tcp/127.0.0.1:10006) (DC: dc1)" +50:13.660 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=udp +50:13.660 [INFO] TestAgent: Started DNS server: address=127.0.0.1:10001 network=tcp +50:13.661 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:10002 network=tcp +50:13.661 [INFO] TestAgent: started state syncer +50:13.661 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:10007 network=tcp +50:13.702 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:13.702 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:10006 [Candidate]" term=2 +50:13.702 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:13.702 [DEBUG] TestAgent.server.raft: vote granted: from=cbb9fcd1-683b-fd2c-86bd-33661620491d term=2 tally=1 +50:13.702 [INFO] TestAgent.server.raft: election won: tally=1 +50:13.702 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:10006 [Leader]" +50:13.702 [INFO] TestAgent.server: cluster leadership acquired +50:13.703 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10006 +50:13.703 [INFO] TestAgent.server: New leader elected: payload=Node-cbb9fcd1-683b-fd2c-86bd-33661620491d +50:13.706 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:13.709 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:13.710 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:13.710 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:13.710 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:13.710 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:13.710 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cbb9fcd1-683b-fd2c-86bd-33661620491d +50:13.710 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cbb9fcd1-683b-fd2c-86bd-33661620491d +50:13.711 [INFO] TestAgent.server: federation state anti-entropy synced +50:13.796 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:13.796 [INFO] TestAgent: Synced node info +50:13.997 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:56262 latency=743.441µs +50:14.002 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions from=127.0.0.1:56264 latency=911.566µs +50:14.009 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/a9d3468f-5491-1a96-9dde-8e325c12cb4e from=127.0.0.1:56264 latency=318.221µs +50:14.009 [INFO] TestAgent: Requesting shutdown +50:14.009 [INFO] TestAgent.server: shutting down server +50:14.009 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:14.009 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:14.009 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:14.009 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:14.009 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:14.009 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:14.009 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:14.009 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:14.009 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:14.009 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:14.009 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:14.010Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:14.010Z [INFO] agent.router.manager: shutting down +50:14.010 [INFO] TestAgent: consul server down +50:14.010 [INFO] TestAgent: shutdown complete +50:14.010 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=tcp +50:14.010 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:10001 network=udp +50:14.010 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:10002 network=tcp +50:14.510 [INFO] TestAgent: Waiting for endpoints to shut down +50:14.510 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_srcDst (0.96s) PASS -ok github.com/hashicorp/consul/command/intention/get 1.534s +ok github.com/hashicorp/consul/command/intention/get 1.019s === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === RUN TestCommand_Validation @@ -85792,142 +85934,143 @@ === PAUSE TestCommand_matchSource === CONT TestCommand_noTabs === CONT TestCommand_matchSource -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestCommand_noTabs (0.00s) === CONT TestCommand_matchDst === CONT TestCommand_Validation -=== RUN TestCommand_Validation/0_args +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestCommand_noTabs (0.00s) +=== RUN TestCommand_Validation/3_args [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -=== RUN TestCommand_Validation/3_args === RUN TestCommand_Validation/both_source_and_dest +=== RUN TestCommand_Validation/0_args --- PASS: TestCommand_Validation (0.01s) - --- PASS: TestCommand_Validation/0_args (0.00s) --- PASS: TestCommand_Validation/3_args (0.00s) --- PASS: TestCommand_Validation/both_source_and_dest (0.00s) -2021-10-31T23:17:38.665Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:38.696Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:38.700 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:083153d1-bb2a-6268-6e62-44d6630b69c6 Address:127.0.0.1:15118}]" -17:38.700 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15118 [Follower]" leader= -17:38.710 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-083153d1-bb2a-6268-6e62-44d6630b69c6.dc1 127.0.0.1 -17:38.719 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-083153d1-bb2a-6268-6e62-44d6630b69c6 127.0.0.1 -2021-10-31T23:17:38.720Z [INFO] agent.router: Initializing LAN area manager -17:38.720 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=udp -17:38.720 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-083153d1-bb2a-6268-6e62-44d6630b69c6.dc1 area=wan -17:38.732 [INFO] TestAgent.server: Adding LAN server: server="Node-083153d1-bb2a-6268-6e62-44d6630b69c6 (Addr: tcp/127.0.0.1:15118) (DC: dc1)" -17:38.734 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=tcp -17:38.735 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15114 network=tcp -17:38.735 [INFO] TestAgent: started state syncer -17:38.735 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15119 network=tcp -17:38.751 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:38.751 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15118 [Candidate]" term=2 -17:38.752 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:38.752 [DEBUG] TestAgent.server.raft: vote granted: from=083153d1-bb2a-6268-6e62-44d6630b69c6 term=2 tally=1 -17:38.752 [INFO] TestAgent.server.raft: election won: tally=1 -17:38.752 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15118 [Leader]" -17:38.752 [INFO] TestAgent.server: cluster leadership acquired -17:38.752 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15118 -17:38.753 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:38.753 [INFO] TestAgent.server: New leader elected: payload=Node-083153d1-bb2a-6268-6e62-44d6630b69c6 -17:38.774 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:38.774 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:38.774 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:38.774 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:38.774 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:38.774 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-083153d1-bb2a-6268-6e62-44d6630b69c6 -17:38.774 [INFO] TestAgent.server: member joined, marking health alive: member=Node-083153d1-bb2a-6268-6e62-44d6630b69c6 -17:38.775 [INFO] TestAgent.server: federation state anti-entropy synced -17:38.888 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:42318 latency=2.146866ms -17:38.890 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:42318 latency=1.538501ms -17:38.898 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:42318 latency=851.612µs -17:38.906 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/match?by=destination&name=db from=127.0.0.1:42320 latency=590.264µs -17:38.907 [INFO] TestAgent: Requesting shutdown -17:38.907 [INFO] TestAgent.server: shutting down server -17:38.907 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:38.907 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:38.907 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:38.907 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:38.907 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:38.907 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:38.907 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:38.907 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:38.907 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:38.907 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:38.907 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:38.907 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:38.907Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:38.907Z [INFO] agent.router.manager: shutting down -17:38.907 [INFO] TestAgent: consul server down -17:38.907 [INFO] TestAgent: shutdown complete -17:38.907 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=tcp -17:38.907 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=udp -17:38.907 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15114 network=tcp -17:39.409 [INFO] TestAgent: Waiting for endpoints to shut down -17:39.409 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_matchDst (0.81s) -2021-10-31T23:17:38.666Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:38.699Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:38.713 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4dd211b1-709e-9cfd-fa11-97a74b0a3e71 Address:127.0.0.1:15111}]" -17:38.713 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15111 [Follower]" leader= -17:38.731 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4dd211b1-709e-9cfd-fa11-97a74b0a3e71.dc1 127.0.0.1 -17:38.732 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4dd211b1-709e-9cfd-fa11-97a74b0a3e71 127.0.0.1 -2021-10-31T23:17:38.732Z [INFO] agent.router: Initializing LAN area manager -17:38.732 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=udp -17:38.733 [INFO] TestAgent.server: Adding LAN server: server="Node-4dd211b1-709e-9cfd-fa11-97a74b0a3e71 (Addr: tcp/127.0.0.1:15111) (DC: dc1)" -17:38.733 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4dd211b1-709e-9cfd-fa11-97a74b0a3e71.dc1 area=wan -17:38.733 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=tcp -17:38.735 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15107 network=tcp -17:38.736 [INFO] TestAgent: started state syncer -17:38.736 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15112 network=tcp -17:38.785 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:38.785 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15111 [Candidate]" term=2 -17:38.786 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:38.786 [DEBUG] TestAgent.server.raft: vote granted: from=4dd211b1-709e-9cfd-fa11-97a74b0a3e71 term=2 tally=1 -17:38.786 [INFO] TestAgent.server.raft: election won: tally=1 -17:38.786 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15111 [Leader]" -17:38.786 [INFO] TestAgent.server: cluster leadership acquired -17:38.786 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15111 -17:38.786 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:38.787 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:38.787 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:38.787 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:38.787 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:38.787 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:38.787 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4dd211b1-709e-9cfd-fa11-97a74b0a3e71 -17:38.787 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4dd211b1-709e-9cfd-fa11-97a74b0a3e71 -17:38.788 [INFO] TestAgent.server: New leader elected: payload=Node-4dd211b1-709e-9cfd-fa11-97a74b0a3e71 -17:38.788 [INFO] TestAgent.server: federation state anti-entropy synced -17:38.982 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:38.993 [INFO] TestAgent: Synced node info -17:39.059 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:32930 latency=659.15µs -17:39.060 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:32930 latency=486.901µs -17:39.066 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:32930 latency=732.915µs -17:39.068 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/match?by=source&name=foo from=127.0.0.1:32932 latency=327.866µs -17:39.068 [INFO] TestAgent: Requesting shutdown -17:39.068 [INFO] TestAgent.server: shutting down server -17:39.068 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:39.068 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:39.068 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:39.068 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:39.068 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:39.068 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:39.068 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:39.068 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:39.068 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:39.068 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:39.068 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:39.068Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:39.069Z [INFO] agent.router.manager: shutting down -17:39.069 [INFO] TestAgent: consul server down -17:39.069 [INFO] TestAgent: shutdown complete -17:39.069 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=tcp -17:39.069 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=udp -17:39.069 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15107 network=tcp -17:39.569 [INFO] TestAgent: Waiting for endpoints to shut down -17:39.569 [INFO] TestAgent: Endpoints down + --- PASS: TestCommand_Validation/0_args (0.00s) +2022-12-04T05:50:13.777Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:13.791Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:13.799 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:39d76838-f90f-22bf-d177-1f5ad7d7317b Address:127.0.0.1:13076}]" +50:13.799 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13076 [Follower]" leader= +50:13.800 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-39d76838-f90f-22bf-d177-1f5ad7d7317b.dc1 127.0.0.1 +50:13.804 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-39d76838-f90f-22bf-d177-1f5ad7d7317b 127.0.0.1 +2022-12-04T05:50:13.804Z [INFO] agent.router: Initializing LAN area manager +50:13.805 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-39d76838-f90f-22bf-d177-1f5ad7d7317b.dc1 area=wan +50:13.805 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=tcp +50:13.805 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=udp +50:13.805 [INFO] TestAgent.server: Adding LAN server: server="Node-39d76838-f90f-22bf-d177-1f5ad7d7317b (Addr: tcp/127.0.0.1:13076) (DC: dc1)" +50:13.805 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13072 network=tcp +50:13.805 [INFO] TestAgent: started state syncer +50:13.805 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13077 network=tcp +50:13.870 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:13.870 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13076 [Candidate]" term=2 +50:13.870 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:13.870 [DEBUG] TestAgent.server.raft: vote granted: from=39d76838-f90f-22bf-d177-1f5ad7d7317b term=2 tally=1 +50:13.870 [INFO] TestAgent.server.raft: election won: tally=1 +50:13.870 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13076 [Leader]" +50:13.870 [INFO] TestAgent.server: cluster leadership acquired +50:13.870 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13076 +50:13.871 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:13.871 [INFO] TestAgent.server: New leader elected: payload=Node-39d76838-f90f-22bf-d177-1f5ad7d7317b +50:13.890 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:13.890 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:13.890 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:13.890 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:13.890 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:13.890 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-39d76838-f90f-22bf-d177-1f5ad7d7317b +50:13.890 [INFO] TestAgent.server: member joined, marking health alive: member=Node-39d76838-f90f-22bf-d177-1f5ad7d7317b +50:13.891 [INFO] TestAgent.server: federation state anti-entropy synced +50:14.135 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:59718 latency=1.066916ms +50:14.137 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:14.138 [INFO] TestAgent: Synced node info +50:14.140 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:59718 latency=1.516597ms +50:14.141 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:59718 latency=635.565µs +50:14.143 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/match?by=destination&name=db from=127.0.0.1:59720 latency=389.507µs +50:14.143 [INFO] TestAgent: Requesting shutdown +50:14.143 [INFO] TestAgent.server: shutting down server +50:14.143 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:14.143 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:14.143 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:14.143 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:14.143 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:14.143 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:14.143 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:14.143 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:14.143 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:14.143 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:14.143 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:14.143Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:14.143Z [INFO] agent.router.manager: shutting down +50:14.144 [INFO] TestAgent: consul server down +50:14.144 [INFO] TestAgent: shutdown complete +50:14.144 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=tcp +50:14.144 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=udp +50:14.144 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13072 network=tcp +50:14.644 [INFO] TestAgent: Waiting for endpoints to shut down +50:14.644 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_matchDst (0.91s) +2022-12-04T05:50:13.777Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:13.798Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:13.800 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4b0b9e62-4edb-bb41-5c8b-c28d4aa0e980 Address:127.0.0.1:13069}]" +50:13.800 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= +50:13.801 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4b0b9e62-4edb-bb41-5c8b-c28d4aa0e980.dc1 127.0.0.1 +50:13.801 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4b0b9e62-4edb-bb41-5c8b-c28d4aa0e980 127.0.0.1 +2022-12-04T05:50:13.801Z [INFO] agent.router: Initializing LAN area manager +50:13.802 [INFO] TestAgent.server: Adding LAN server: server="Node-4b0b9e62-4edb-bb41-5c8b-c28d4aa0e980 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" +50:13.802 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4b0b9e62-4edb-bb41-5c8b-c28d4aa0e980.dc1 area=wan +50:13.802 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp +50:13.802 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp +50:13.802 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp +50:13.802 [INFO] TestAgent: started state syncer +50:13.802 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp +50:13.865 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:13.865 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 +50:13.865 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:13.865 [DEBUG] TestAgent.server.raft: vote granted: from=4b0b9e62-4edb-bb41-5c8b-c28d4aa0e980 term=2 tally=1 +50:13.865 [INFO] TestAgent.server.raft: election won: tally=1 +50:13.865 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" +50:13.865 [INFO] TestAgent.server: cluster leadership acquired +50:13.865 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13069 +50:13.865 [INFO] TestAgent.server: New leader elected: payload=Node-4b0b9e62-4edb-bb41-5c8b-c28d4aa0e980 +50:13.866 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:13.888 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:13.888 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:13.888 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:13.888 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:13.888 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:13.888 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4b0b9e62-4edb-bb41-5c8b-c28d4aa0e980 +50:13.888 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4b0b9e62-4edb-bb41-5c8b-c28d4aa0e980 +50:13.890 [INFO] TestAgent.server: federation state anti-entropy synced +50:14.072 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:14.072 [INFO] TestAgent: Synced node info +50:14.201 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:54424 latency=718.053µs +50:14.205 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:54424 latency=621.888µs +50:14.206 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:54424 latency=502.614µs +50:14.207 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/connect/intentions/match?by=source&name=foo from=127.0.0.1:54426 latency=232.402µs +50:14.208 [INFO] TestAgent: Requesting shutdown +50:14.208 [INFO] TestAgent.server: shutting down server +50:14.208 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:14.208 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:14.208 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:14.208 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:14.208 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:14.208 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:14.208 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:14.208 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:14.208 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:14.208 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:14.208 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:14.208Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:14.208Z [INFO] agent.router.manager: shutting down +50:14.208 [INFO] TestAgent: consul server down +50:14.208 [INFO] TestAgent: shutdown complete +50:14.208 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp +50:14.208 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp +50:14.208 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp +50:14.709 [INFO] TestAgent: Waiting for endpoints to shut down +50:14.709 [INFO] TestAgent: Endpoints down --- PASS: TestCommand_matchSource (0.97s) PASS -ok github.com/hashicorp/consul/command/intention/match 1.220s +ok github.com/hashicorp/consul/command/intention/match 1.026s === RUN TestJoinCommand_noTabs === PAUSE TestJoinCommand_noTabs === RUN TestJoinCommandJoin_lan @@ -85937,286 +86080,286 @@ === RUN TestJoinCommand_noAddrs === PAUSE TestJoinCommand_noAddrs === CONT TestJoinCommand_noTabs -=== CONT TestJoinCommand_noAddrs --- PASS: TestJoinCommand_noTabs (0.00s) -=== CONT TestJoinCommand_wan +=== CONT TestJoinCommand_noAddrs --- PASS: TestJoinCommand_noAddrs (0.00s) -=== CONT TestJoinCommandJoin_lan +=== CONT TestJoinCommand_wan [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +=== CONT TestJoinCommandJoin_lan [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:17:38.885Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:38.905Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:38.934 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:976fbd27-0315-5c10-816c-9e087a3c1d2f Address:127.0.0.1:16146}]" -17:38.935 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-976fbd27-0315-5c10-816c-9e087a3c1d2f.dc1 127.0.0.1 -17:38.947 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-976fbd27-0315-5c10-816c-9e087a3c1d2f 127.0.0.1 -2021-10-31T23:17:38.947Z [INFO] agent.router: Initializing LAN area manager -17:38.948 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16146 [Follower]" leader= -17:38.948 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16141 network=tcp -17:38.948 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-976fbd27-0315-5c10-816c-9e087a3c1d2f.dc1 area=wan -17:38.966 [INFO] TestAgent.server: Adding LAN server: server="Node-976fbd27-0315-5c10-816c-9e087a3c1d2f (Addr: tcp/127.0.0.1:16146) (DC: dc1)" -17:38.970 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16141 network=udp -17:39.005 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:39.009 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16146 [Candidate]" term=2 -17:39.010 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:39.010 [DEBUG] TestAgent.server.raft: vote granted: from=976fbd27-0315-5c10-816c-9e087a3c1d2f term=2 tally=1 -17:39.026 [INFO] TestAgent.server.raft: election won: tally=1 -17:39.026 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16146 [Leader]" -17:39.026 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16142 network=tcp -17:39.026 [INFO] TestAgent: started state syncer -17:39.027 [INFO] TestAgent.server: cluster leadership acquired -17:39.027 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16146 -17:39.027 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:39.029 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:39.029 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:39.029 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:39.029 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:39.029 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:39.029 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-976fbd27-0315-5c10-816c-9e087a3c1d2f -17:39.029 [INFO] TestAgent.server: member joined, marking health alive: member=Node-976fbd27-0315-5c10-816c-9e087a3c1d2f -17:39.029 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16147 network=tcp -17:39.029 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:39.029 [INFO] TestAgent.server: New leader elected: payload=Node-976fbd27-0315-5c10-816c-9e087a3c1d2f -17:39.030 [INFO] TestAgent: Synced node info -17:39.030 [DEBUG] TestAgent: Node info in sync -17:39.030 [INFO] TestAgent.server: federation state anti-entropy synced -17:39.031 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:34348 -17:39.031 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fc28b1a3-3cdc-326a-072d-f36762c09d8e 127.0.0.1 -17:39.031 [INFO] TestAgent.server: Adding LAN server: server="Node-fc28b1a3-3cdc-326a-072d-f36762c09d8e (Addr: tcp/127.0.0.1:16139) (DC: dc1)" -17:39.031 [INFO] TestAgent.server: New leader elected: payload=Node-fc28b1a3-3cdc-326a-072d-f36762c09d8e -17:39.032 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-fc28b1a3-3cdc-326a-072d-f36762c09d8e other=Node-976fbd27-0315-5c10-816c-9e087a3c1d2f -17:39.032 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fc28b1a3-3cdc-326a-072d-f36762c09d8e -17:39.033 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-fc28b1a3-3cdc-326a-072d-f36762c09d8e.dc1 127.0.0.1:16138 -17:39.033 [INFO] TestAgent: Requesting shutdown -17:39.033 [INFO] TestAgent.server: shutting down server -17:39.033 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:39.033 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:39.033 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:39.033 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:39.033 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:39.033 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:39.033 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:39.033 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:39.033 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:39.033 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:39.033 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -17:39.034 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:54286 -2021-10-31T23:17:39.034Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:39.034Z [INFO] agent.router.manager: shutting down -17:39.038 [INFO] TestAgent: consul server down -17:39.038 [INFO] TestAgent: shutdown complete -17:39.038 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16141 network=tcp -17:39.038 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16141 network=udp -17:39.038 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16142 network=tcp -17:39.041 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fc28b1a3-3cdc-326a-072d-f36762c09d8e.dc1 127.0.0.1 -17:39.041 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-fc28b1a3-3cdc-326a-072d-f36762c09d8e.dc1 address=127.0.0.1:16138 -17:39.541 [INFO] TestAgent: Waiting for endpoints to shut down -17:39.541 [INFO] TestAgent: Endpoints down -2021-10-31T23:17:38.611Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:38.668Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:38.674 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fc28b1a3-3cdc-326a-072d-f36762c09d8e Address:127.0.0.1:16139}]" -17:38.674 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fc28b1a3-3cdc-326a-072d-f36762c09d8e.dc1 127.0.0.1 -17:38.675 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fc28b1a3-3cdc-326a-072d-f36762c09d8e 127.0.0.1 -2021-10-31T23:17:38.675Z [INFO] agent.router: Initializing LAN area manager -17:38.675 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=udp -17:38.675 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16139 [Follower]" leader= -17:38.676 [INFO] TestAgent.server: Adding LAN server: server="Node-fc28b1a3-3cdc-326a-072d-f36762c09d8e (Addr: tcp/127.0.0.1:16139) (DC: dc1)" -17:38.676 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fc28b1a3-3cdc-326a-072d-f36762c09d8e.dc1 area=wan -17:38.676 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=tcp -17:38.676 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16135 network=tcp -17:38.676 [INFO] TestAgent: started state syncer -17:38.676 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16140 network=tcp -17:38.729 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:38.729 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16139 [Candidate]" term=2 -17:38.729 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:38.729 [DEBUG] TestAgent.server.raft: vote granted: from=fc28b1a3-3cdc-326a-072d-f36762c09d8e term=2 tally=1 -17:38.729 [INFO] TestAgent.server.raft: election won: tally=1 -17:38.729 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16139 [Leader]" -17:38.730 [INFO] TestAgent.server: cluster leadership acquired -17:38.730 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16139 -17:38.730 [INFO] TestAgent.server: New leader elected: payload=Node-fc28b1a3-3cdc-326a-072d-f36762c09d8e -17:38.730 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:38.732 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:38.732 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:38.732 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:38.732 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:38.732 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:38.732 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fc28b1a3-3cdc-326a-072d-f36762c09d8e -17:38.732 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fc28b1a3-3cdc-326a-072d-f36762c09d8e -17:38.733 [INFO] TestAgent.server: federation state anti-entropy synced -17:38.882 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:38.882 [INFO] TestAgent: Synced node info -17:39.030 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:16144] -17:39.030 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:16144 -17:39.033 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-976fbd27-0315-5c10-816c-9e087a3c1d2f 127.0.0.1 -17:39.033 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 -17:39.033 [DEBUG] TestAgent: systemd notify failed: error="No socket" -17:39.033 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/join/127.0.0.1:16144 from=127.0.0.1:50916 latency=3.156627ms -17:39.033 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:54670 -17:39.033 [INFO] TestAgent.server: Adding LAN server: server="Node-976fbd27-0315-5c10-816c-9e087a3c1d2f (Addr: tcp/127.0.0.1:16146) (DC: dc1)" -17:39.033 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-976fbd27-0315-5c10-816c-9e087a3c1d2f other=Node-fc28b1a3-3cdc-326a-072d-f36762c09d8e -17:39.033 [INFO] TestAgent.server: member joined, marking health alive: member=Node-976fbd27-0315-5c10-816c-9e087a3c1d2f -17:39.033 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-976fbd27-0315-5c10-816c-9e087a3c1d2f.dc1 127.0.0.1:16145 -17:39.034 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-976fbd27-0315-5c10-816c-9e087a3c1d2f.dc1 127.0.0.1 -17:39.034 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-976fbd27-0315-5c10-816c-9e087a3c1d2f.dc1 area=wan -17:39.046 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-976fbd27-0315-5c10-816c-9e087a3c1d2f.dc1 address=127.0.0.1:16145 -17:39.541 [INFO] TestAgent: Requesting shutdown -17:39.542 [INFO] TestAgent.server: shutting down server -17:39.542 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:39.542 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:39.542 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:39.542 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:39.542 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:39.542 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:39.542 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:39.542 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:39.542 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:39.542 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:39.542 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:39.542Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:39.542Z [INFO] agent.router.manager: shutting down -17:39.542 [INFO] TestAgent: consul server down -17:39.542 [INFO] TestAgent: shutdown complete -17:39.542 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=tcp -17:39.542 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=udp -17:39.542 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16135 network=tcp -17:40.043 [INFO] TestAgent: Waiting for endpoints to shut down -17:40.043 [INFO] TestAgent: Endpoints down ---- PASS: TestJoinCommandJoin_lan (1.51s) -2021-10-31T23:17:38.995Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:39.048Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:39.050 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0422c023-f13d-9a08-c7de-96be4046aa0c Address:127.0.0.1:16153}]" -17:39.050 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0422c023-f13d-9a08-c7de-96be4046aa0c.dc1 127.0.0.1 -17:39.051 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0422c023-f13d-9a08-c7de-96be4046aa0c 127.0.0.1 -2021-10-31T23:17:39.051Z [INFO] agent.router: Initializing LAN area manager -17:39.051 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16148 network=udp -17:39.051 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16153 [Follower]" leader= -17:39.051 [INFO] TestAgent.server: Adding LAN server: server="Node-0422c023-f13d-9a08-c7de-96be4046aa0c (Addr: tcp/127.0.0.1:16153) (DC: dc1)" -17:39.051 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0422c023-f13d-9a08-c7de-96be4046aa0c.dc1 area=wan -17:39.052 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16148 network=tcp -17:39.064 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16149 network=tcp -17:39.064 [INFO] TestAgent: started state syncer -17:39.064 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16154 network=tcp -17:39.099 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:39.099 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16153 [Candidate]" term=2 -17:39.099 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:39.099 [DEBUG] TestAgent.server.raft: vote granted: from=0422c023-f13d-9a08-c7de-96be4046aa0c term=2 tally=1 -17:39.099 [INFO] TestAgent.server.raft: election won: tally=1 -17:39.099 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16153 [Leader]" -17:39.099 [INFO] TestAgent.server: cluster leadership acquired -17:39.099 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16153 -17:39.100 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:39.101 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:39.101 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:39.101 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:39.101 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:39.101 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:39.101 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0422c023-f13d-9a08-c7de-96be4046aa0c -17:39.101 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0422c023-f13d-9a08-c7de-96be4046aa0c -17:39.102 [INFO] TestAgent.server: New leader elected: payload=Node-0422c023-f13d-9a08-c7de-96be4046aa0c -17:39.102 [INFO] TestAgent.server: federation state anti-entropy synced -17:39.196 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:55942 -17:39.196 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e40a611d-c9dd-958b-e04a-1dedc86a45fa.dc1 127.0.0.1 -17:39.196 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e40a611d-c9dd-958b-e04a-1dedc86a45fa.dc1 area=wan -17:39.197 [INFO] TestAgent: Requesting shutdown -17:39.197 [INFO] TestAgent.server: shutting down server -17:39.197 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:39.197 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:39.197 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:39.197 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:39.197 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:39.197 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:39.197 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:39.197 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:39.197 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:39.197 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:39.197 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:39.197 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:39.197Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:39.197Z [INFO] agent.router.manager: shutting down -17:39.197 [INFO] TestAgent: consul server down -17:39.197 [INFO] TestAgent: shutdown complete -17:39.197 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16148 network=tcp -17:39.197 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16148 network=udp -17:39.197 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16149 network=tcp -17:39.701 [INFO] TestAgent: Waiting for endpoints to shut down -17:39.701 [INFO] TestAgent: Endpoints down -2021-10-31T23:17:38.599Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:38.619Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:38.627 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e40a611d-c9dd-958b-e04a-1dedc86a45fa Address:127.0.0.1:16132}]" -17:38.628 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16132 [Follower]" leader= -17:38.628 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e40a611d-c9dd-958b-e04a-1dedc86a45fa.dc1 127.0.0.1 -17:38.642 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e40a611d-c9dd-958b-e04a-1dedc86a45fa 127.0.0.1 -2021-10-31T23:17:38.642Z [INFO] agent.router: Initializing LAN area manager -17:38.648 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=udp -17:38.648 [INFO] TestAgent.server: Adding LAN server: server="Node-e40a611d-c9dd-958b-e04a-1dedc86a45fa (Addr: tcp/127.0.0.1:16132) (DC: dc1)" -17:38.648 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e40a611d-c9dd-958b-e04a-1dedc86a45fa.dc1 area=wan -17:38.648 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=tcp -17:38.649 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16128 network=tcp -17:38.649 [INFO] TestAgent: started state syncer -17:38.649 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16133 network=tcp -17:38.670 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:38.670 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16132 [Candidate]" term=2 -17:38.671 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:38.671 [DEBUG] TestAgent.server.raft: vote granted: from=e40a611d-c9dd-958b-e04a-1dedc86a45fa term=2 tally=1 -17:38.671 [INFO] TestAgent.server.raft: election won: tally=1 -17:38.671 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16132 [Leader]" -17:38.671 [INFO] TestAgent.server: cluster leadership acquired -17:38.671 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16132 -17:38.672 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:38.673 [INFO] TestAgent.server: New leader elected: payload=Node-e40a611d-c9dd-958b-e04a-1dedc86a45fa -17:38.694 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:38.694 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:38.694 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:38.694 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:38.694 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:38.694 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e40a611d-c9dd-958b-e04a-1dedc86a45fa -17:38.694 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e40a611d-c9dd-958b-e04a-1dedc86a45fa -17:38.695 [INFO] TestAgent.server: federation state anti-entropy synced -17:38.970 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:38.994 [INFO] TestAgent: Synced node info -17:38.994 [DEBUG] TestAgent: Node info in sync -17:39.195 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:16152] -17:39.195 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:16152 -17:39.196 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0422c023-f13d-9a08-c7de-96be4046aa0c.dc1 127.0.0.1 -17:39.196 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 -17:39.196 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/join/127.0.0.1:16152?wan=1 from=127.0.0.1:43650 latency=1.318085ms -17:39.196 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0422c023-f13d-9a08-c7de-96be4046aa0c.dc1 area=wan -17:39.701 [INFO] TestAgent: Requesting shutdown -17:39.702 [INFO] TestAgent.server: shutting down server -17:39.702 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:39.702 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:39.702 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:39.702 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:39.702 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:39.702 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:39.702 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:39.702 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:39.702 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:39.702 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:39.702 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:39.702Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:39.702Z [INFO] agent.router.manager: shutting down -17:39.702 [INFO] TestAgent: consul server down -17:39.702 [INFO] TestAgent: shutdown complete -17:39.702 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=tcp -17:39.702 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=udp -17:39.702 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16128 network=tcp -17:40.203 [INFO] TestAgent: Waiting for endpoints to shut down -17:40.203 [INFO] TestAgent: Endpoints down ---- PASS: TestJoinCommand_wan (1.67s) +2022-12-04T05:50:17.039Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:17.092Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:17.094 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bd7ea0a4-6944-72cd-d834-4bc241e6ef61 Address:127.0.0.1:18188}]" +50:17.095 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bd7ea0a4-6944-72cd-d834-4bc241e6ef61.dc1 127.0.0.1 +50:17.096 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bd7ea0a4-6944-72cd-d834-4bc241e6ef61 127.0.0.1 +2022-12-04T05:50:17.096Z [INFO] agent.router: Initializing LAN area manager +50:17.096 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18188 [Follower]" leader= +50:17.096 [INFO] TestAgent.server: Adding LAN server: server="Node-bd7ea0a4-6944-72cd-d834-4bc241e6ef61 (Addr: tcp/127.0.0.1:18188) (DC: dc1)" +50:17.096 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bd7ea0a4-6944-72cd-d834-4bc241e6ef61.dc1 area=wan +50:17.104 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18183 network=udp +50:17.105 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18183 network=tcp +50:17.105 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18184 network=tcp +50:17.105 [INFO] TestAgent: started state syncer +50:17.105 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18189 network=tcp +50:17.136 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:17.136 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18188 [Candidate]" term=2 +50:17.136 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:17.136 [DEBUG] TestAgent.server.raft: vote granted: from=bd7ea0a4-6944-72cd-d834-4bc241e6ef61 term=2 tally=1 +50:17.136 [INFO] TestAgent.server.raft: election won: tally=1 +50:17.136 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18188 [Leader]" +50:17.136 [INFO] TestAgent.server: cluster leadership acquired +50:17.137 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18188 +50:17.137 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:17.139 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:17.139 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:17.139 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:17.139 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:17.139 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:17.139 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bd7ea0a4-6944-72cd-d834-4bc241e6ef61 +50:17.139 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bd7ea0a4-6944-72cd-d834-4bc241e6ef61 +50:17.139 [INFO] TestAgent.server: New leader elected: payload=Node-bd7ea0a4-6944-72cd-d834-4bc241e6ef61 +50:17.140 [INFO] TestAgent.server: federation state anti-entropy synced +50:17.369 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:41476 +50:17.370 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-62338a10-b570-35a4-f9b4-16ceef6fc9f0.dc1 127.0.0.1 +50:17.370 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-62338a10-b570-35a4-f9b4-16ceef6fc9f0.dc1 area=wan +50:17.371 [INFO] TestAgent: Requesting shutdown +50:17.371 [INFO] TestAgent.server: shutting down server +50:17.371 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:17.371 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:17.371 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:17.371 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:17.371 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:17.371 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:17.371 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:17.371 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:17.371 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:17.371 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:17.371 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:17.371 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:17.371Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:17.371Z [INFO] agent.router.manager: shutting down +50:17.371 [INFO] TestAgent: consul server down +50:17.371 [INFO] TestAgent: shutdown complete +50:17.371 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18183 network=tcp +50:17.371 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18183 network=udp +50:17.371 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18184 network=tcp +50:17.872 [INFO] TestAgent: Waiting for endpoints to shut down +50:17.872 [INFO] TestAgent: Endpoints down +2022-12-04T05:50:16.675Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:16.716Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:16.724 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:62338a10-b570-35a4-f9b4-16ceef6fc9f0 Address:127.0.0.1:18174}]" +50:16.724 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18174 [Follower]" leader= +50:16.753 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-62338a10-b570-35a4-f9b4-16ceef6fc9f0.dc1 127.0.0.1 +50:16.754 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-62338a10-b570-35a4-f9b4-16ceef6fc9f0 127.0.0.1 +2022-12-04T05:50:16.754Z [INFO] agent.router: Initializing LAN area manager +50:16.763 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=udp +50:16.763 [INFO] TestAgent.server: Adding LAN server: server="Node-62338a10-b570-35a4-f9b4-16ceef6fc9f0 (Addr: tcp/127.0.0.1:18174) (DC: dc1)" +50:16.763 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-62338a10-b570-35a4-f9b4-16ceef6fc9f0.dc1 area=wan +50:16.763 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=tcp +50:16.764 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18170 network=tcp +50:16.764 [INFO] TestAgent: started state syncer +50:16.764 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18175 network=tcp +50:16.792 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:16.792 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18174 [Candidate]" term=2 +50:16.793 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:16.793 [DEBUG] TestAgent.server.raft: vote granted: from=62338a10-b570-35a4-f9b4-16ceef6fc9f0 term=2 tally=1 +50:16.793 [INFO] TestAgent.server.raft: election won: tally=1 +50:16.793 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18174 [Leader]" +50:16.793 [INFO] TestAgent.server: cluster leadership acquired +50:16.793 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18174 +50:16.794 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:16.796 [INFO] TestAgent.server: New leader elected: payload=Node-62338a10-b570-35a4-f9b4-16ceef6fc9f0 +50:16.830 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:16.830 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:16.830 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:16.830 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:16.830 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:16.830 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-62338a10-b570-35a4-f9b4-16ceef6fc9f0 +50:16.830 [INFO] TestAgent.server: member joined, marking health alive: member=Node-62338a10-b570-35a4-f9b4-16ceef6fc9f0 +50:16.831 [INFO] TestAgent.server: federation state anti-entropy synced +50:17.028 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:17.029 [INFO] TestAgent: Synced node info +50:17.029 [DEBUG] TestAgent: Node info in sync +50:17.368 [INFO] TestAgent: (WAN) joining: wan_addresses=[127.0.0.1:18187] +50:17.369 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: 127.0.0.1:18187 +50:17.370 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bd7ea0a4-6944-72cd-d834-4bc241e6ef61.dc1 127.0.0.1 +50:17.371 [INFO] TestAgent: (WAN) joined: number_of_nodes=1 +50:17.371 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/join/127.0.0.1:18187?wan=1 from=127.0.0.1:50296 latency=2.166547ms +50:17.371 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bd7ea0a4-6944-72cd-d834-4bc241e6ef61.dc1 area=wan +50:17.872 [INFO] TestAgent: Requesting shutdown +50:17.872 [INFO] TestAgent.server: shutting down server +50:17.872 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:17.872 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:17.872 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:17.872 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:17.872 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:17.873 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:17.873 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:17.873 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:17.873 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:17.873 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:17.873 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:17.873Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:17.873Z [INFO] agent.router.manager: shutting down +50:17.873 [INFO] TestAgent: consul server down +50:17.873 [INFO] TestAgent: shutdown complete +50:17.873 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=tcp +50:17.873 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=udp +50:17.873 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18170 network=tcp +50:18.376 [INFO] TestAgent: Waiting for endpoints to shut down +50:18.376 [INFO] TestAgent: Endpoints down +--- PASS: TestJoinCommand_wan (1.85s) +2022-12-04T05:50:17.202Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:17.241Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:17.243 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6d633891-f7eb-b958-6d37-021a4100a0a0 Address:127.0.0.1:18195}]" +50:17.244 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18195 [Follower]" leader= +50:17.245 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6d633891-f7eb-b958-6d37-021a4100a0a0.dc1 127.0.0.1 +50:17.257 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6d633891-f7eb-b958-6d37-021a4100a0a0 127.0.0.1 +2022-12-04T05:50:17.257Z [INFO] agent.router: Initializing LAN area manager +50:17.258 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18190 network=udp +50:17.258 [INFO] TestAgent.server: Adding LAN server: server="Node-6d633891-f7eb-b958-6d37-021a4100a0a0 (Addr: tcp/127.0.0.1:18195) (DC: dc1)" +50:17.258 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6d633891-f7eb-b958-6d37-021a4100a0a0.dc1 area=wan +50:17.258 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18190 network=tcp +50:17.259 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18191 network=tcp +50:17.259 [INFO] TestAgent: started state syncer +50:17.259 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18196 network=tcp +50:17.312 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:17.312 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18195 [Candidate]" term=2 +50:17.312 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:17.312 [DEBUG] TestAgent.server.raft: vote granted: from=6d633891-f7eb-b958-6d37-021a4100a0a0 term=2 tally=1 +50:17.312 [INFO] TestAgent.server.raft: election won: tally=1 +50:17.312 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18195 [Leader]" +50:17.312 [INFO] TestAgent.server: cluster leadership acquired +50:17.313 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18195 +50:17.313 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:17.315 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:17.315 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:17.315 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:17.315 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:17.315 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:17.315 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6d633891-f7eb-b958-6d37-021a4100a0a0 +50:17.315 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6d633891-f7eb-b958-6d37-021a4100a0a0 +50:17.316 [INFO] TestAgent.server: New leader elected: payload=Node-6d633891-f7eb-b958-6d37-021a4100a0a0 +50:17.320 [INFO] TestAgent.server: federation state anti-entropy synced +50:17.498 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Stream connection from=127.0.0.1:50432 +50:17.498 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653 127.0.0.1 +50:17.499 [INFO] TestAgent.server: Adding LAN server: server="Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653 (Addr: tcp/127.0.0.1:18181) (DC: dc1)" +50:17.499 [INFO] TestAgent.server: New leader elected: payload=Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653 +50:17.499 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653 other=Node-6d633891-f7eb-b958-6d37-021a4100a0a0 +50:17.499 [INFO] TestAgent.server: member joined, marking health alive: member=Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653 +50:17.500 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653.dc1 127.0.0.1:18180 +50:17.504 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:41472 +50:17.505 [INFO] TestAgent: Requesting shutdown +50:17.505 [INFO] TestAgent.server: shutting down server +50:17.505 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:17.505 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:17.505 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:17.505 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:17.505 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:17.506 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653.dc1 127.0.0.1 +50:17.506 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653.dc1 area=wan +50:17.506 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:17.506 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:17.506 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:17.506 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:17.506 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:17.506 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653.dc1 address=127.0.0.1:18180 +50:17.506 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:17.506 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:17.506Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:17.506Z [INFO] agent.router.manager: shutting down +50:17.507 [INFO] TestAgent: consul server down +50:17.507 [INFO] TestAgent: shutdown complete +50:17.507 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18190 network=tcp +50:17.507 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18190 network=udp +50:17.507 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18191 network=tcp +50:18.008 [INFO] TestAgent: Waiting for endpoints to shut down +50:18.008 [INFO] TestAgent: Endpoints down +2022-12-04T05:50:16.682Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:16.729Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:16.745 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:89cb1b27-5e99-a0fb-a0cf-26dc9618d653 Address:127.0.0.1:18181}]" +50:16.746 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653.dc1 127.0.0.1 +50:16.746 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653 127.0.0.1 +2022-12-04T05:50:16.746Z [INFO] agent.router: Initializing LAN area manager +50:16.747 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18176 network=udp +50:16.747 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18181 [Follower]" leader= +50:16.761 [INFO] TestAgent.server: Adding LAN server: server="Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653 (Addr: tcp/127.0.0.1:18181) (DC: dc1)" +50:16.761 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653.dc1 area=wan +50:16.761 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18176 network=tcp +50:16.762 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18177 network=tcp +50:16.762 [INFO] TestAgent: started state syncer +50:16.763 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18182 network=tcp +50:16.796 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:16.796 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18181 [Candidate]" term=2 +50:16.796 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:16.796 [DEBUG] TestAgent.server.raft: vote granted: from=89cb1b27-5e99-a0fb-a0cf-26dc9618d653 term=2 tally=1 +50:16.796 [INFO] TestAgent.server.raft: election won: tally=1 +50:16.796 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18181 [Leader]" +50:16.796 [INFO] TestAgent.server: cluster leadership acquired +50:16.797 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18181 +50:16.797 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:16.797 [INFO] TestAgent.server: New leader elected: payload=Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653 +50:16.833 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:16.833 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:16.833 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:16.833 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:16.833 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:16.833 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653 +50:16.833 [INFO] TestAgent.server: member joined, marking health alive: member=Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653 +50:16.834 [INFO] TestAgent.server: federation state anti-entropy synced +50:16.992 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:16.993 [INFO] TestAgent: Synced node info +50:17.497 [INFO] TestAgent: (LAN) joining: lan_addresses=[127.0.0.1:18193] +50:17.498 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Initiating push/pull sync with: 127.0.0.1:18193 +50:17.500 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Stream connection from=127.0.0.1:50938 +50:17.501 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6d633891-f7eb-b958-6d37-021a4100a0a0 127.0.0.1 +50:17.501 [INFO] TestAgent: (LAN) joined: number_of_nodes=1 +50:17.501 [DEBUG] TestAgent: systemd notify failed: error="No socket" +50:17.501 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/join/127.0.0.1:18193 from=127.0.0.1:60970 latency=3.451329ms +50:17.501 [INFO] TestAgent.server: Adding LAN server: server="Node-6d633891-f7eb-b958-6d37-021a4100a0a0 (Addr: tcp/127.0.0.1:18195) (DC: dc1)" +50:17.501 [ERROR] TestAgent.server: Two nodes are in bootstrap mode. Only one node should be in bootstrap mode, not adding Raft peer.: node_to_add=Node-6d633891-f7eb-b958-6d37-021a4100a0a0 other=Node-89cb1b27-5e99-a0fb-a0cf-26dc9618d653 +50:17.501 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6d633891-f7eb-b958-6d37-021a4100a0a0 +50:17.504 [DEBUG] TestAgent.server.memberlist.wan: memberlist: Initiating push/pull sync with: Node-6d633891-f7eb-b958-6d37-021a4100a0a0.dc1 127.0.0.1:18194 +50:17.505 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6d633891-f7eb-b958-6d37-021a4100a0a0.dc1 127.0.0.1 +50:17.506 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6d633891-f7eb-b958-6d37-021a4100a0a0.dc1 area=wan +50:17.506 [DEBUG] TestAgent.server: Successfully performed flood-join for server at address: server=Node-6d633891-f7eb-b958-6d37-021a4100a0a0.dc1 address=127.0.0.1:18194 +50:18.008 [INFO] TestAgent: Requesting shutdown +50:18.008 [INFO] TestAgent.server: shutting down server +50:18.008 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:18.008 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:18.008 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:18.008 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:18.008 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:18.009 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:18.009 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:18.009 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:18.009 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:18.009 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:18.009 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:18.009Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:18.009Z [INFO] agent.router.manager: shutting down +50:18.009 [INFO] TestAgent: consul server down +50:18.009 [INFO] TestAgent: shutdown complete +50:18.009 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18176 network=tcp +50:18.009 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18176 network=udp +50:18.009 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18177 network=tcp +50:18.252 [DEBUG] TestAgent.server.memberlist.lan: memberlist: Failed ping: Node-6d633891-f7eb-b958-6d37-021a4100a0a0 (timeout reached) +50:18.512 [INFO] TestAgent: Waiting for endpoints to shut down +50:18.512 [INFO] TestAgent: Endpoints down +--- PASS: TestJoinCommandJoin_lan (1.98s) PASS -ok github.com/hashicorp/consul/command/join 2.083s +ok github.com/hashicorp/consul/command/join 3.179s === RUN TestKeygenCommand_noTabs === PAUSE TestKeygenCommand_noTabs === RUN TestKeygenCommand === PAUSE TestKeygenCommand === CONT TestKeygenCommand_noTabs ---- PASS: TestKeygenCommand_noTabs (0.00s) === CONT TestKeygenCommand +--- PASS: TestKeygenCommand_noTabs (0.00s) --- PASS: TestKeygenCommand (0.00s) PASS -ok github.com/hashicorp/consul/command/keygen 0.046s +ok github.com/hashicorp/consul/command/keygen 0.007s === RUN TestKeyringCommand_noTabs === PAUSE TestKeyringCommand_noTabs === RUN TestKeyringCommand @@ -86240,119 +86383,120 @@ === RUN TestKeyringCommand_numActions --- PASS: TestKeyringCommand_numActions (0.00s) === CONT TestKeyringCommand_noTabs ---- PASS: TestKeyringCommand_noTabs (0.00s) -=== CONT TestKeyringCommand_invalidRelayFactor ---- PASS: TestKeyringCommand_invalidRelayFactor (0.00s) +--- PASS: TestKeyringCommand_noTabs (0.01s) === CONT TestKeyringCommand_invalidLocalOnly -=== CONT TestKeyringCommand_help -=== CONT TestKeyringCommand_failedConnection --- PASS: TestKeyringCommand_invalidLocalOnly (0.00s) +=== CONT TestKeyringCommand_failedConnection === CONT TestKeyringCommand +=== CONT TestKeyringCommand_invalidRelayFactor [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestKeyringCommand_invalidRelayFactor (0.00s) +=== CONT TestKeyringCommand_help --- PASS: TestKeyringCommand_help (0.00s) ---- PASS: TestKeyringCommand_failedConnection (0.00s) +--- PASS: TestKeyringCommand_failedConnection (0.01s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:17:42.219Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:42.233Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:42.243 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c Address:127.0.0.1:22258}]" -17:42.244 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c.dc1 127.0.0.1 -17:42.245 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c 127.0.0.1 -2021-10-31T23:17:42.245Z [INFO] agent.router: Initializing LAN area manager -17:42.245 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=udp -17:42.245 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22258 [Follower]" leader= -17:42.246 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=tcp -17:42.246 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c.dc1 area=wan -17:42.246 [INFO] TestAgent.server: Adding LAN server: server="Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -17:42.255 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22254 network=tcp -17:42.255 [INFO] TestAgent: started state syncer -17:42.255 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22259 network=tcp -17:42.294 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:42.294 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22258 [Candidate]" term=2 -17:42.295 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:42.295 [DEBUG] TestAgent.server.raft: vote granted: from=e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c term=2 tally=1 -17:42.295 [INFO] TestAgent.server.raft: election won: tally=1 -17:42.295 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22258 [Leader]" -17:42.295 [INFO] TestAgent.server: cluster leadership acquired -17:42.295 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22258 -17:42.296 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:42.297 [INFO] TestAgent.server: New leader elected: payload=Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c -17:42.325 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:42.326 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:42.326 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:42.326 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:42.326 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:42.326 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c -17:42.326 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c -17:42.327 [INFO] TestAgent.server: federation state anti-entropy synced -17:42.334 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:42.334 [INFO] TestAgent: Synced node info -17:42.538 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -17:42.538 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c.dc1 -17:42.538 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -17:42.538 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c -17:42.539 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:50180 latency=1.542877ms -17:42.540 [INFO] TestAgent.server.serf.wan: serf: Received install-key query -17:42.541 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c.dc1 -17:42.541 [INFO] TestAgent.server.serf.lan: serf: Received install-key query -17:42.541 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c -17:42.541 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/keyring from=127.0.0.1:50182 latency=805.678µs -17:42.554 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -17:42.554 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c.dc1 -17:42.554 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -17:42.554 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c -17:42.556 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:50184 latency=2.157229ms -17:42.575 [INFO] TestAgent.server.serf.wan: serf: Received use-key query -17:42.579 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c.dc1 -17:42.580 [INFO] TestAgent.server.serf.lan: serf: Received use-key query -17:42.581 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c -17:42.581 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/keyring from=127.0.0.1:50186 latency=6.466585ms -17:42.583 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -17:42.583 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c.dc1 -17:42.583 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -17:42.583 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c -17:42.584 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:50188 latency=930.596µs -17:42.585 [INFO] TestAgent.server.serf.wan: serf: Received remove-key query -17:42.585 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c.dc1 -17:42.585 [INFO] TestAgent.server.serf.lan: serf: Received remove-key query -17:42.586 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c -17:42.586 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/keyring from=127.0.0.1:50190 latency=1.329211ms -17:42.587 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query -17:42.588 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c.dc1 -17:42.588 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query -17:42.588 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-e5debb94-3e9e-eebb-fd3c-7c1acb8ed95c -17:42.588 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:50192 latency=1.008997ms -17:42.589 [INFO] TestAgent: Requesting shutdown -17:42.589 [INFO] TestAgent.server: shutting down server -17:42.589 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:42.589 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:42.589 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:42.589 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:42.589 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:42.589 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:42.589 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:42.589 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:42.589 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:42.589 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:42.589 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:42.589Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:42.589Z [INFO] agent.router.manager: shutting down -17:42.589 [INFO] TestAgent: consul server down -17:42.589 [INFO] TestAgent: shutdown complete -17:42.589 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=tcp -17:42.589 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=udp -17:42.589 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22254 network=tcp -17:43.089 [INFO] TestAgent: Waiting for endpoints to shut down -17:43.089 [INFO] TestAgent: Endpoints down ---- PASS: TestKeyringCommand (0.95s) +2022-12-04T05:50:26.943Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:27.002Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:27.005 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9bff6ea4-267c-a766-631c-14d2cc32fd8e Address:127.0.0.1:18174}]" +50:27.006 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e.dc1 127.0.0.1 +50:27.007 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e 127.0.0.1 +2022-12-04T05:50:27.007Z [INFO] agent.router: Initializing LAN area manager +50:27.007 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=udp +50:27.007 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:18174 [Follower]" leader= +50:27.008 [INFO] TestAgent.server: Adding LAN server: server="Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e (Addr: tcp/127.0.0.1:18174) (DC: dc1)" +50:27.008 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e.dc1 area=wan +50:27.008 [INFO] TestAgent: Started DNS server: address=127.0.0.1:18169 network=tcp +50:27.017 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:18170 network=tcp +50:27.017 [INFO] TestAgent: started state syncer +50:27.017 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:18175 network=tcp +50:27.060 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:27.060 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:18174 [Candidate]" term=2 +50:27.060 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:27.060 [DEBUG] TestAgent.server.raft: vote granted: from=9bff6ea4-267c-a766-631c-14d2cc32fd8e term=2 tally=1 +50:27.060 [INFO] TestAgent.server.raft: election won: tally=1 +50:27.060 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:18174 [Leader]" +50:27.060 [INFO] TestAgent.server: cluster leadership acquired +50:27.061 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:18174 +50:27.062 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:27.064 [INFO] TestAgent.server: New leader elected: payload=Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e +50:27.100 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:27.100 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:27.100 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:27.100 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:27.100 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:27.100 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e +50:27.100 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e +50:27.101 [INFO] TestAgent.server: federation state anti-entropy synced +50:27.240 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:27.241 [INFO] TestAgent: Synced node info +50:27.241 [DEBUG] TestAgent: Node info in sync +50:27.245 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +50:27.248 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e.dc1 +50:27.249 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +50:27.249 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e +50:27.250 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:50300 latency=5.699344ms +50:27.273 [INFO] TestAgent.server.serf.wan: serf: Received install-key query +50:27.274 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e.dc1 +50:27.274 [INFO] TestAgent.server.serf.lan: serf: Received install-key query +50:27.274 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e +50:27.274 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/operator/keyring from=127.0.0.1:50302 latency=1.250528ms +50:27.280 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +50:27.288 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e.dc1 +50:27.288 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +50:27.289 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e +50:27.289 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:50304 latency=13.256008ms +50:27.291 [INFO] TestAgent.server.serf.wan: serf: Received use-key query +50:27.291 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e.dc1 +50:27.291 [INFO] TestAgent.server.serf.lan: serf: Received use-key query +50:27.291 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e +50:27.291 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/keyring from=127.0.0.1:50306 latency=910.212µs +50:27.297 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +50:27.297 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e.dc1 +50:27.297 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +50:27.297 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e +50:27.298 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:50308 latency=1.147774ms +50:27.300 [INFO] TestAgent.server.serf.wan: serf: Received remove-key query +50:27.300 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e.dc1 +50:27.300 [INFO] TestAgent.server.serf.lan: serf: Received remove-key query +50:27.304 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e +50:27.304 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/keyring from=127.0.0.1:50312 latency=4.794302ms +50:27.305 [INFO] TestAgent.server.serf.wan: serf: Received list-keys query +50:27.306 [DEBUG] TestAgent.server.serf.wan: serf: messageQueryResponseType: Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e.dc1 +50:27.306 [INFO] TestAgent.server.serf.lan: serf: Received list-keys query +50:27.306 [DEBUG] TestAgent.server.serf.lan: serf: messageQueryResponseType: Node-9bff6ea4-267c-a766-631c-14d2cc32fd8e +50:27.306 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/keyring from=127.0.0.1:50314 latency=710.937µs +50:27.306 [INFO] TestAgent: Requesting shutdown +50:27.306 [INFO] TestAgent.server: shutting down server +50:27.306 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:27.306 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:27.306 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:27.306 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:27.306 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:27.306 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:27.306 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:27.306 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:27.306 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:27.306 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:27.307 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:27.307Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:27.307Z [INFO] agent.router.manager: shutting down +50:27.307 [INFO] TestAgent: consul server down +50:27.307 [INFO] TestAgent: shutdown complete +50:27.307 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=tcp +50:27.307 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:18169 network=udp +50:27.307 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:18170 network=tcp +50:27.808 [INFO] TestAgent: Waiting for endpoints to shut down +50:27.808 [INFO] TestAgent: Endpoints down +--- PASS: TestKeyringCommand (1.01s) PASS -ok github.com/hashicorp/consul/command/keyring 1.061s +ok github.com/hashicorp/consul/command/keyring 2.080s === RUN TestKVCommand_noTabs === PAUSE TestKVCommand_noTabs === CONT TestKVCommand_noTabs --- PASS: TestKVCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/kv 0.033s +ok github.com/hashicorp/consul/command/kv 0.011s === RUN TestKVDeleteCommand_noTabs === PAUSE TestKVDeleteCommand_noTabs === RUN TestKVDeleteCommand_Validation @@ -86364,200 +86508,201 @@ === RUN TestKVDeleteCommand_CAS === PAUSE TestKVDeleteCommand_CAS === CONT TestKVDeleteCommand_noTabs ---- PASS: TestKVDeleteCommand_noTabs (0.01s) -=== CONT TestKVDeleteCommand_Recurse -=== CONT TestKVDeleteCommand +--- PASS: TestKVDeleteCommand_noTabs (0.00s) +=== CONT TestKVDeleteCommand_CAS [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +=== CONT TestKVDeleteCommand +=== CONT TestKVDeleteCommand_Recurse === CONT TestKVDeleteCommand_Validation -=== CONT TestKVDeleteCommand_CAS ---- PASS: TestKVDeleteCommand_Validation (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:17:47.856Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:47.923Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:47.925 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:75a817f3-faf7-8174-28d9-3f672598ae51 Address:127.0.0.1:25321}]" -17:47.937 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25321 [Follower]" leader= -17:47.938 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-75a817f3-faf7-8174-28d9-3f672598ae51.dc1 127.0.0.1 -17:47.951 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-75a817f3-faf7-8174-28d9-3f672598ae51 127.0.0.1 -2021-10-31T23:17:47.951Z [INFO] agent.router: Initializing LAN area manager -17:47.951 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=udp -17:47.951 [INFO] TestAgent.server: Adding LAN server: server="Node-75a817f3-faf7-8174-28d9-3f672598ae51 (Addr: tcp/127.0.0.1:25321) (DC: dc1)" -17:47.951 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-75a817f3-faf7-8174-28d9-3f672598ae51.dc1 area=wan -17:47.952 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=tcp -17:47.953 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25317 network=tcp -17:47.953 [INFO] TestAgent: started state syncer -17:47.953 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25322 network=tcp -17:47.989 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:47.989 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25321 [Candidate]" term=2 -17:47.990 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:47.990 [DEBUG] TestAgent.server.raft: vote granted: from=75a817f3-faf7-8174-28d9-3f672598ae51 term=2 tally=1 -17:47.990 [INFO] TestAgent.server.raft: election won: tally=1 -17:47.990 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25321 [Leader]" -17:47.990 [INFO] TestAgent.server: cluster leadership acquired -17:47.990 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25321 -17:47.991 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:47.991 [INFO] TestAgent.server: New leader elected: payload=Node-75a817f3-faf7-8174-28d9-3f672598ae51 -17:48.006 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:48.006 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:48.006 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:48.006 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:48.006 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:48.006 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-75a817f3-faf7-8174-28d9-3f672598ae51 -17:48.006 [INFO] TestAgent.server: member joined, marking health alive: member=Node-75a817f3-faf7-8174-28d9-3f672598ae51 -17:48.007 [INFO] TestAgent.server: federation state anti-entropy synced -17:48.021 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:38284 latency=641.848µs -17:48.030 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/b from=127.0.0.1:38284 latency=9.421492ms -17:48.038 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/food from=127.0.0.1:38284 latency=748.707µs -17:48.040 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/foo?recurse= from=127.0.0.1:38286 latency=1.2521ms -17:48.041 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo/a from=127.0.0.1:38284 latency=72.043µs -17:48.042 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo/b from=127.0.0.1:38284 latency=95.115µs -17:48.074 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/food from=127.0.0.1:38284 latency=82.098µs -17:48.078 [INFO] TestAgent: Requesting shutdown -17:48.078 [INFO] TestAgent.server: shutting down server -17:48.078 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:48.078 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:48.078 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:48.078 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:48.078 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:48.078 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:48.078 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:48.078 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -17:48.078 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:48.078 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:48.078 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:48.078 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:17:48.078Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:48.078Z [INFO] agent.router.manager: shutting down -17:48.078 [INFO] TestAgent: consul server down -17:48.078 [INFO] TestAgent: shutdown complete -17:48.078 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=tcp -17:48.078 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=udp -17:48.078 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25317 network=tcp -17:48.581 [INFO] TestAgent: Waiting for endpoints to shut down -17:48.581 [INFO] TestAgent: Endpoints down ---- PASS: TestKVDeleteCommand_Recurse (0.88s) -2021-10-31T23:17:47.859Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:47.902Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:47.920 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ced0ac1b-a8a5-e868-b279-edfdcdbe626a Address:127.0.0.1:25335}]" -17:47.921 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ced0ac1b-a8a5-e868-b279-edfdcdbe626a.dc1 127.0.0.1 -17:47.921 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ced0ac1b-a8a5-e868-b279-edfdcdbe626a 127.0.0.1 -2021-10-31T23:17:47.921Z [INFO] agent.router: Initializing LAN area manager -17:47.921 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25335 [Follower]" leader= -17:47.922 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25330 network=udp -17:47.922 [INFO] TestAgent.server: Adding LAN server: server="Node-ced0ac1b-a8a5-e868-b279-edfdcdbe626a (Addr: tcp/127.0.0.1:25335) (DC: dc1)" -17:47.922 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ced0ac1b-a8a5-e868-b279-edfdcdbe626a.dc1 area=wan -17:47.922 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25330 network=tcp -17:47.923 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25331 network=tcp -17:47.923 [INFO] TestAgent: started state syncer -17:47.923 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25336 network=tcp -17:47.965 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:47.965 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25335 [Candidate]" term=2 -17:47.966 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:47.966 [DEBUG] TestAgent.server.raft: vote granted: from=ced0ac1b-a8a5-e868-b279-edfdcdbe626a term=2 tally=1 -17:47.966 [INFO] TestAgent.server.raft: election won: tally=1 -17:47.966 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25335 [Leader]" -17:47.966 [INFO] TestAgent.server: cluster leadership acquired -17:47.966 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25335 -17:47.967 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:47.969 [INFO] TestAgent.server: New leader elected: payload=Node-ced0ac1b-a8a5-e868-b279-edfdcdbe626a -17:48.002 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:48.002 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:48.002 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:48.002 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:48.002 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:48.002 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ced0ac1b-a8a5-e868-b279-edfdcdbe626a -17:48.002 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ced0ac1b-a8a5-e868-b279-edfdcdbe626a -17:48.003 [INFO] TestAgent.server: federation state anti-entropy synced -17:48.062 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:37290 latency=9.23583ms -17:48.073 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/foo?cas=1 from=127.0.0.1:37292 latency=1.254451ms -17:48.083 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:37290 latency=1.025458ms -17:48.095 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/foo?cas=12 from=127.0.0.1:37294 latency=773.441µs -17:48.095 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:37290 latency=39.254µs -17:48.096 [INFO] TestAgent: Requesting shutdown -17:48.096 [INFO] TestAgent.server: shutting down server -17:48.096 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:48.096 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:48.096 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:48.096 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:48.096 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:48.096 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:48.096 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:48.096 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:48.096 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:48.096 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:48.096 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:48.096 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:48.096Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:48.096Z [INFO] agent.router.manager: shutting down -17:48.096 [INFO] TestAgent: consul server down -17:48.096 [INFO] TestAgent: shutdown complete -17:48.096 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25330 network=tcp -17:48.096 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25330 network=udp -17:48.096 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25331 network=tcp -17:48.597 [INFO] TestAgent: Waiting for endpoints to shut down -17:48.597 [INFO] TestAgent: Endpoints down ---- PASS: TestKVDeleteCommand_CAS (0.89s) -2021-10-31T23:17:47.858Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:47.920Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:47.922 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e4ed20a0-3afc-f3ae-c6af-1e7772eff811 Address:127.0.0.1:25328}]" -17:47.923 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e4ed20a0-3afc-f3ae-c6af-1e7772eff811.dc1 127.0.0.1 -17:47.923 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25328 [Follower]" leader= -17:47.924 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e4ed20a0-3afc-f3ae-c6af-1e7772eff811 127.0.0.1 -2021-10-31T23:17:47.924Z [INFO] agent.router: Initializing LAN area manager -17:47.924 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=udp -17:47.924 [INFO] TestAgent.server: Adding LAN server: server="Node-e4ed20a0-3afc-f3ae-c6af-1e7772eff811 (Addr: tcp/127.0.0.1:25328) (DC: dc1)" -17:47.924 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e4ed20a0-3afc-f3ae-c6af-1e7772eff811.dc1 area=wan -17:47.924 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=tcp -17:47.925 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25324 network=tcp -17:47.925 [INFO] TestAgent: started state syncer -17:47.925 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25329 network=tcp -17:47.977 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:47.977 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25328 [Candidate]" term=2 -17:47.978 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:47.978 [DEBUG] TestAgent.server.raft: vote granted: from=e4ed20a0-3afc-f3ae-c6af-1e7772eff811 term=2 tally=1 -17:47.978 [INFO] TestAgent.server.raft: election won: tally=1 -17:47.978 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25328 [Leader]" -17:47.978 [INFO] TestAgent.server: cluster leadership acquired -17:47.978 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25328 -17:47.979 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:47.979 [INFO] TestAgent.server: New leader elected: payload=Node-e4ed20a0-3afc-f3ae-c6af-1e7772eff811 -17:47.999 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:47.999 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:47.999 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:47.999 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:47.999 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:47.999 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e4ed20a0-3afc-f3ae-c6af-1e7772eff811 -17:47.999 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e4ed20a0-3afc-f3ae-c6af-1e7772eff811 -17:48.000 [INFO] TestAgent.server: federation state anti-entropy synced -17:48.119 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:52504 latency=589.624µs -17:48.126 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/foo from=127.0.0.1:52506 latency=532.581µs -17:48.127 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:52504 latency=44.747µs -17:48.127 [INFO] TestAgent: Requesting shutdown -17:48.127 [INFO] TestAgent.server: shutting down server -17:48.127 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:48.127 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:48.127 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:48.127 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:48.127 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:48.127 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:48.127 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:48.127 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:48.127 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:48.127 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:48.127 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:48.127 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:48.127Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:48.127Z [INFO] agent.router.manager: shutting down -17:48.127 [INFO] TestAgent: consul server down -17:48.127 [INFO] TestAgent: shutdown complete -17:48.127 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=tcp -17:48.127 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=udp -17:48.127 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25324 network=tcp -17:48.632 [INFO] TestAgent: Waiting for endpoints to shut down -17:48.632 [INFO] TestAgent: Endpoints down ---- PASS: TestKVDeleteCommand (0.93s) +--- PASS: TestKVDeleteCommand_Validation (0.01s) +2022-12-04T05:50:28.288Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:28.349Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:28.370 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ce069a34-8a78-69fb-7b3e-ab952a55e237 Address:127.0.0.1:16132}]" +50:28.370 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16132 [Follower]" leader= +50:28.371 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ce069a34-8a78-69fb-7b3e-ab952a55e237.dc1 127.0.0.1 +50:28.392 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ce069a34-8a78-69fb-7b3e-ab952a55e237 127.0.0.1 +2022-12-04T05:50:28.393Z [INFO] agent.router: Initializing LAN area manager +50:28.393 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=udp +50:28.393 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ce069a34-8a78-69fb-7b3e-ab952a55e237.dc1 area=wan +50:28.393 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=tcp +50:28.394 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16128 network=tcp +50:28.394 [INFO] TestAgent: started state syncer +50:28.394 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16133 network=tcp +50:28.394 [INFO] TestAgent.server: Adding LAN server: server="Node-ce069a34-8a78-69fb-7b3e-ab952a55e237 (Addr: tcp/127.0.0.1:16132) (DC: dc1)" +50:28.416 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:28.416 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16132 [Candidate]" term=2 +50:28.416 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:28.416 [DEBUG] TestAgent.server.raft: vote granted: from=ce069a34-8a78-69fb-7b3e-ab952a55e237 term=2 tally=1 +50:28.416 [INFO] TestAgent.server.raft: election won: tally=1 +50:28.416 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16132 [Leader]" +50:28.417 [INFO] TestAgent.server: cluster leadership acquired +50:28.417 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16132 +50:28.430 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:28.430 [INFO] TestAgent.server: New leader elected: payload=Node-ce069a34-8a78-69fb-7b3e-ab952a55e237 +50:28.486 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:28.486 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:28.486 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:28.486 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:28.486 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:28.486 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ce069a34-8a78-69fb-7b3e-ab952a55e237 +50:28.486 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ce069a34-8a78-69fb-7b3e-ab952a55e237 +50:28.487 [INFO] TestAgent.server: federation state anti-entropy synced +50:28.514 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:40526 latency=2.856915ms +50:28.522 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/foo?cas=1 from=127.0.0.1:40528 latency=632.2µs +50:28.523 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:40526 latency=383.574µs +50:28.537 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/foo?cas=12 from=127.0.0.1:40530 latency=12.96578ms +50:28.541 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:40526 latency=106.472µs +50:28.541 [INFO] TestAgent: Requesting shutdown +50:28.541 [INFO] TestAgent.server: shutting down server +50:28.541 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:28.541 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:28.541 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:28.541 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:28.541 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:28.541 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:28.541 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:28.541 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:28.541 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:28.541 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:28.541 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:28.542 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:28.542Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:28.542Z [INFO] agent.router.manager: shutting down +50:28.542 [INFO] TestAgent: consul server down +50:28.542 [INFO] TestAgent: shutdown complete +50:28.542 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=tcp +50:28.542 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=udp +50:28.542 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16128 network=tcp +50:29.044 [INFO] TestAgent: Waiting for endpoints to shut down +50:29.044 [INFO] TestAgent: Endpoints down +--- PASS: TestKVDeleteCommand_CAS (0.88s) +2022-12-04T05:50:28.304Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:28.391Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:28.393 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f7c5214b-c225-146b-e92a-13ae61fbf643 Address:127.0.0.1:16139}]" +50:28.393 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f7c5214b-c225-146b-e92a-13ae61fbf643.dc1 127.0.0.1 +50:28.394 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f7c5214b-c225-146b-e92a-13ae61fbf643 127.0.0.1 +2022-12-04T05:50:28.394Z [INFO] agent.router: Initializing LAN area manager +50:28.394 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=udp +50:28.394 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16139 [Follower]" leader= +50:28.395 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f7c5214b-c225-146b-e92a-13ae61fbf643.dc1 area=wan +50:28.395 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=tcp +50:28.395 [INFO] TestAgent.server: Adding LAN server: server="Node-f7c5214b-c225-146b-e92a-13ae61fbf643 (Addr: tcp/127.0.0.1:16139) (DC: dc1)" +50:28.395 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16135 network=tcp +50:28.395 [INFO] TestAgent: started state syncer +50:28.395 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16140 network=tcp +50:28.432 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:28.436 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16139 [Candidate]" term=2 +50:28.437 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:28.437 [DEBUG] TestAgent.server.raft: vote granted: from=f7c5214b-c225-146b-e92a-13ae61fbf643 term=2 tally=1 +50:28.437 [INFO] TestAgent.server.raft: election won: tally=1 +50:28.437 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16139 [Leader]" +50:28.437 [INFO] TestAgent.server: cluster leadership acquired +50:28.437 [INFO] TestAgent.server: New leader elected: payload=Node-f7c5214b-c225-146b-e92a-13ae61fbf643 +50:28.437 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16139 +50:28.440 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:28.486 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:28.486 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:28.486 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:28.486 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:28.486 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:28.486 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f7c5214b-c225-146b-e92a-13ae61fbf643 +50:28.486 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f7c5214b-c225-146b-e92a-13ae61fbf643 +50:28.487 [INFO] TestAgent.server: federation state anti-entropy synced +50:28.561 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:51776 latency=772.629µs +50:28.569 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/foo from=127.0.0.1:51778 latency=595.7µs +50:28.570 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:51776 latency=42.065µs +50:28.570 [INFO] TestAgent: Requesting shutdown +50:28.570 [INFO] TestAgent.server: shutting down server +50:28.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:28.570 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:28.570 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:28.570 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:28.570 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:28.570 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:28.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:28.570 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:28.570 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:28.570 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:28.570 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:28.570 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:28.570Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:28.570Z [INFO] agent.router.manager: shutting down +50:28.570 [INFO] TestAgent: consul server down +50:28.570 [INFO] TestAgent: shutdown complete +50:28.570 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=tcp +50:28.570 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=udp +50:28.570 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16135 network=tcp +50:29.072 [INFO] TestAgent: Waiting for endpoints to shut down +50:29.072 [INFO] TestAgent: Endpoints down +--- PASS: TestKVDeleteCommand (0.91s) +2022-12-04T05:50:28.322Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:28.395Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:28.398 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8095de91-b481-fe04-cf79-268337d1f305 Address:127.0.0.1:16146}]" +50:28.399 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8095de91-b481-fe04-cf79-268337d1f305.dc1 127.0.0.1 +50:28.399 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8095de91-b481-fe04-cf79-268337d1f305 127.0.0.1 +2022-12-04T05:50:28.399Z [INFO] agent.router: Initializing LAN area manager +50:28.400 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16141 network=udp +50:28.400 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16146 [Follower]" leader= +50:28.400 [INFO] TestAgent.server: Adding LAN server: server="Node-8095de91-b481-fe04-cf79-268337d1f305 (Addr: tcp/127.0.0.1:16146) (DC: dc1)" +50:28.400 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8095de91-b481-fe04-cf79-268337d1f305.dc1 area=wan +50:28.400 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16141 network=tcp +50:28.401 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16142 network=tcp +50:28.401 [INFO] TestAgent: started state syncer +50:28.401 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16147 network=tcp +50:28.464 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:28.464 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16146 [Candidate]" term=2 +50:28.465 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:28.465 [DEBUG] TestAgent.server.raft: vote granted: from=8095de91-b481-fe04-cf79-268337d1f305 term=2 tally=1 +50:28.465 [INFO] TestAgent.server.raft: election won: tally=1 +50:28.465 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16146 [Leader]" +50:28.465 [INFO] TestAgent.server: cluster leadership acquired +50:28.465 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16146 +50:28.466 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:28.466 [INFO] TestAgent.server: New leader elected: payload=Node-8095de91-b481-fe04-cf79-268337d1f305 +50:28.482 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:28.482 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:28.482 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:28.482 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:28.482 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:28.482 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8095de91-b481-fe04-cf79-268337d1f305 +50:28.482 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8095de91-b481-fe04-cf79-268337d1f305 +50:28.485 [INFO] TestAgent.server: federation state anti-entropy synced +50:28.656 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:28.657 [INFO] TestAgent: Synced node info +50:28.785 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:35286 latency=23.404316ms +50:28.793 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/b from=127.0.0.1:35286 latency=895.129µs +50:28.806 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/food from=127.0.0.1:35286 latency=679.405µs +50:28.807 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/foo?recurse= from=127.0.0.1:35288 latency=769.249µs +50:28.808 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo/a from=127.0.0.1:35286 latency=55.238µs +50:28.808 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo/b from=127.0.0.1:35286 latency=50.207µs +50:28.809 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/food from=127.0.0.1:35286 latency=25.028µs +50:28.809 [INFO] TestAgent: Requesting shutdown +50:28.809 [INFO] TestAgent.server: shutting down server +50:28.809 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:28.809 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:28.809 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:28.809 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:28.809 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:28.809 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:28.809 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:28.809 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:28.810 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:28.810 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:28.810 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:28.810Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:28.810Z [INFO] agent.router.manager: shutting down +50:28.810 [INFO] TestAgent: consul server down +50:28.810 [INFO] TestAgent: shutdown complete +50:28.810 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16141 network=tcp +50:28.810 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16141 network=udp +50:28.810 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16142 network=tcp +50:29.312 [INFO] TestAgent: Waiting for endpoints to shut down +50:29.312 [INFO] TestAgent: Endpoints down +--- PASS: TestKVDeleteCommand_Recurse (1.15s) PASS -ok github.com/hashicorp/consul/command/kv/del 1.672s +ok github.com/hashicorp/consul/command/kv/del 1.308s === RUN TestKVExportCommand_noTabs === PAUSE TestKVExportCommand_noTabs === RUN TestKVExportCommand @@ -86568,69 +86713,70 @@ [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:17:53.130Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:53.169Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:53.180 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0b1a6b2e-24cc-7ed3-21b2-666645de6ef0 Address:127.0.0.1:29405}]" -17:53.180 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0b1a6b2e-24cc-7ed3-21b2-666645de6ef0.dc1 127.0.0.1 -17:53.181 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0b1a6b2e-24cc-7ed3-21b2-666645de6ef0 127.0.0.1 -2021-10-31T23:17:53.181Z [INFO] agent.router: Initializing LAN area manager -17:53.181 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=udp -17:53.181 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29405 [Follower]" leader= -17:53.182 [INFO] TestAgent.server: Adding LAN server: server="Node-0b1a6b2e-24cc-7ed3-21b2-666645de6ef0 (Addr: tcp/127.0.0.1:29405) (DC: dc1)" -17:53.182 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0b1a6b2e-24cc-7ed3-21b2-666645de6ef0.dc1 area=wan -17:53.182 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=tcp -17:53.183 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29401 network=tcp -17:53.183 [INFO] TestAgent: started state syncer -17:53.183 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29406 network=tcp -17:53.221 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:53.222 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29405 [Candidate]" term=2 -17:53.222 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:53.222 [DEBUG] TestAgent.server.raft: vote granted: from=0b1a6b2e-24cc-7ed3-21b2-666645de6ef0 term=2 tally=1 -17:53.222 [INFO] TestAgent.server.raft: election won: tally=1 -17:53.222 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29405 [Leader]" -17:53.222 [INFO] TestAgent.server: cluster leadership acquired -17:53.222 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29405 -17:53.223 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:53.225 [INFO] TestAgent.server: New leader elected: payload=Node-0b1a6b2e-24cc-7ed3-21b2-666645de6ef0 -17:53.257 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:53.257 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:53.257 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:53.257 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:53.257 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:53.258 [INFO] TestAgent.server: federation state anti-entropy synced -17:53.266 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0b1a6b2e-24cc-7ed3-21b2-666645de6ef0 -17:53.266 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0b1a6b2e-24cc-7ed3-21b2-666645de6ef0 -17:53.401 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:49668 latency=6.980907ms -17:53.422 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/b from=127.0.0.1:49668 latency=14.519614ms -17:53.427 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/c from=127.0.0.1:49668 latency=1.184939ms -17:53.434 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/bar from=127.0.0.1:49668 latency=911.893µs -17:53.439 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo?recurse= from=127.0.0.1:49670 latency=388.808µs -17:53.439 [INFO] TestAgent: Requesting shutdown -17:53.439 [INFO] TestAgent.server: shutting down server -17:53.439 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:53.439 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:53.439 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:53.439 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:53.439 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:53.439 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:53.439 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:53.439 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:53.439 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:53.439 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:53.439 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:53.439 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:53.439Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:53.440Z [INFO] agent.router.manager: shutting down -17:53.440 [INFO] TestAgent: consul server down -17:53.440 [INFO] TestAgent: shutdown complete -17:53.440 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=tcp -17:53.440 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=udp -17:53.440 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29401 network=tcp -17:53.940 [INFO] TestAgent: Waiting for endpoints to shut down -17:53.940 [INFO] TestAgent: Endpoints down ---- PASS: TestKVExportCommand (0.91s) +2022-12-04T05:50:30.704Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:30.749Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:30.752 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:abb6e781-93d8-6864-1914-3b4305df9291 Address:127.0.0.1:17153}]" +50:30.752 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17153 [Follower]" leader= +50:30.761 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-abb6e781-93d8-6864-1914-3b4305df9291.dc1 127.0.0.1 +50:30.762 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-abb6e781-93d8-6864-1914-3b4305df9291 127.0.0.1 +2022-12-04T05:50:30.762Z [INFO] agent.router: Initializing LAN area manager +50:30.763 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17148 network=udp +50:30.763 [INFO] TestAgent.server: Adding LAN server: server="Node-abb6e781-93d8-6864-1914-3b4305df9291 (Addr: tcp/127.0.0.1:17153) (DC: dc1)" +50:30.763 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-abb6e781-93d8-6864-1914-3b4305df9291.dc1 area=wan +50:30.764 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17148 network=tcp +50:30.764 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17149 network=tcp +50:30.765 [INFO] TestAgent: started state syncer +50:30.765 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17154 network=tcp +50:30.816 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:30.816 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17153 [Candidate]" term=2 +50:30.816 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:30.816 [DEBUG] TestAgent.server.raft: vote granted: from=abb6e781-93d8-6864-1914-3b4305df9291 term=2 tally=1 +50:30.816 [INFO] TestAgent.server.raft: election won: tally=1 +50:30.816 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17153 [Leader]" +50:30.817 [INFO] TestAgent.server: cluster leadership acquired +50:30.817 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17153 +50:30.818 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:30.820 [INFO] TestAgent.server: New leader elected: payload=Node-abb6e781-93d8-6864-1914-3b4305df9291 +50:30.870 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:30.870 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:30.870 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:30.870 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:30.870 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:30.871 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-abb6e781-93d8-6864-1914-3b4305df9291 +50:30.871 [INFO] TestAgent.server: member joined, marking health alive: member=Node-abb6e781-93d8-6864-1914-3b4305df9291 +50:30.880 [INFO] TestAgent.server: federation state anti-entropy synced +50:31.040 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:31.041 [INFO] TestAgent: Synced node info +50:31.132 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:36964 latency=6.51029ms +50:31.141 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/b from=127.0.0.1:36964 latency=622.423µs +50:31.142 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/c from=127.0.0.1:36964 latency=445.394µs +50:31.142 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/bar from=127.0.0.1:36964 latency=375.027µs +50:31.143 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo?recurse= from=127.0.0.1:36966 latency=365.637µs +50:31.148 [INFO] TestAgent: Requesting shutdown +50:31.148 [INFO] TestAgent.server: shutting down server +50:31.148 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:31.148 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:31.148 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:31.148 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:31.148 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:31.148 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:31.148 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:31.148 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:31.148 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:31.148 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:31.148 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:31.148Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:31.148Z [INFO] agent.router.manager: shutting down +50:31.149 [INFO] TestAgent: consul server down +50:31.149 [INFO] TestAgent: shutdown complete +50:31.149 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17148 network=tcp +50:31.149 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17148 network=udp +50:31.149 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17149 network=tcp +50:31.652 [INFO] TestAgent: Waiting for endpoints to shut down +50:31.652 [INFO] TestAgent: Endpoints down +--- PASS: TestKVExportCommand (1.07s) PASS -ok github.com/hashicorp/consul/command/kv/exp 1.747s +ok github.com/hashicorp/consul/command/kv/exp 2.126s === RUN TestKVGetCommand_noTabs === PAUSE TestKVGetCommand_noTabs === RUN TestKVGetCommand_Validation @@ -86654,559 +86800,545 @@ === RUN TestKVGetCommand_DetailedBase64 === PAUSE TestKVGetCommand_DetailedBase64 === CONT TestKVGetCommand_noTabs +--- PASS: TestKVGetCommand_noTabs (0.00s) === CONT TestKVGetCommand_DetailedBase64 [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestKVGetCommand_noTabs (0.01s) === CONT TestKVGetCommand_Empty -=== CONT TestKVGetCommand_RecurseBase64 === CONT TestKVGetCommand_Recurse === CONT TestKVGetCommand_Keys === CONT TestKVGetCommand_Detailed -=== CONT TestKVGetCommand_Base64 === CONT TestKVGetCommand_Missing +=== CONT TestKVGetCommand_RecurseBase64 === CONT TestKVGetCommand === CONT TestKVGetCommand_Validation ---- PASS: TestKVGetCommand_Validation (0.00s) +=== CONT TestKVGetCommand_Base64 +--- PASS: TestKVGetCommand_Validation (0.03s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:17:52.806Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:52.890Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:52.996 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:13b6e6a2-c644-2a07-13b7-37de141d7bf5 Address:127.0.0.1:19244}]" -17:52.997 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19244 [Follower]" leader= -17:53.062 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:53.062 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19244 [Candidate]" term=2 -17:53.062 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=13b6e6a2-c644-2a07-13b7-37de141d7bf5 fallback=127.0.0.1:19244 error="Could not find address for server id 13b6e6a2-c644-2a07-13b7-37de141d7bf5" -17:53.062 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:53.062 [DEBUG] TestAgent.server.raft: vote granted: from=13b6e6a2-c644-2a07-13b7-37de141d7bf5 term=2 tally=1 -17:53.062 [INFO] TestAgent.server.raft: election won: tally=1 -17:53.062 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19244 [Leader]" -17:53.062 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-13b6e6a2-c644-2a07-13b7-37de141d7bf5.dc1 127.0.0.1 -17:53.078 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-13b6e6a2-c644-2a07-13b7-37de141d7bf5 127.0.0.1 -2021-10-31T23:17:53.078Z [INFO] agent.router: Initializing LAN area manager -17:53.095 [INFO] TestAgent.server: Adding LAN server: server="Node-13b6e6a2-c644-2a07-13b7-37de141d7bf5 (Addr: tcp/127.0.0.1:19244) (DC: dc1)" -17:53.095 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-13b6e6a2-c644-2a07-13b7-37de141d7bf5.dc1 area=wan -17:53.095 [INFO] TestAgent.server: cluster leadership acquired -17:53.098 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19239 network=tcp -17:53.098 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19239 network=udp -17:53.102 [INFO] TestAgent.server: New leader elected: payload=Node-13b6e6a2-c644-2a07-13b7-37de141d7bf5 -17:53.102 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19244 -17:53.106 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19240 network=tcp -17:53.172 [INFO] TestAgent: started state syncer -17:53.172 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:53.174 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:53.174 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:53.174 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:53.174 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:53.174 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:53.174 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-13b6e6a2-c644-2a07-13b7-37de141d7bf5 -17:53.174 [INFO] TestAgent.server: member joined, marking health alive: member=Node-13b6e6a2-c644-2a07-13b7-37de141d7bf5 -17:53.174 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/not-a-real-key from=127.0.0.1:56966 latency=88.061µs -17:53.174 [INFO] TestAgent: Requesting shutdown -17:53.174 [INFO] TestAgent.server: shutting down server -17:53.174 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:53.174 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:53.174 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:53.174 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:53.174 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:53.174 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-13b6e6a2-c644-2a07-13b7-37de141d7bf5 -17:53.174 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:53.174Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:53.175Z [INFO] agent.router.manager: shutting down -17:53.175 [INFO] TestAgent: consul server down -17:53.175 [INFO] TestAgent: shutdown complete -17:53.175 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19239 network=tcp -17:53.175 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19239 network=udp -17:53.175 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19240 network=tcp -17:53.175 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:53.175 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:53.175 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:53.175 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:53.175 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19245 network=tcp -17:53.175 [ERROR] TestAgent: gRPC server failed: error="grpc: the server has been stopped" -17:53.176 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:53.176 [WARN] TestAgent: Syncing node info failed.: error="No cluster leader" -17:53.176 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:53.677 [INFO] TestAgent: Waiting for endpoints to shut down -17:53.677 [INFO] TestAgent: Endpoints down ---- PASS: TestKVGetCommand_Missing (1.09s) -2021-10-31T23:17:52.731Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:52.883Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:52.924 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f6933515-5694-6a13-9979-e8fef6eb8718 Address:127.0.0.1:19202}]" -17:52.931 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f6933515-5694-6a13-9979-e8fef6eb8718.dc1 127.0.0.1 -17:52.932 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f6933515-5694-6a13-9979-e8fef6eb8718 127.0.0.1 -2021-10-31T23:17:52.932Z [INFO] agent.router: Initializing LAN area manager -17:52.965 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19202 [Follower]" leader= -17:52.965 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=udp -17:52.993 [INFO] TestAgent.server: Adding LAN server: server="Node-f6933515-5694-6a13-9979-e8fef6eb8718 (Addr: tcp/127.0.0.1:19202) (DC: dc1)" -17:52.993 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f6933515-5694-6a13-9979-e8fef6eb8718.dc1 area=wan -17:52.998 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=tcp -17:53.061 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:53.061 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19202 [Candidate]" term=2 -17:53.062 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:53.062 [DEBUG] TestAgent.server.raft: vote granted: from=f6933515-5694-6a13-9979-e8fef6eb8718 term=2 tally=1 -17:53.062 [INFO] TestAgent.server.raft: election won: tally=1 -17:53.062 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19202 [Leader]" -17:53.062 [INFO] TestAgent.server: cluster leadership acquired -17:53.062 [INFO] TestAgent.server: New leader elected: payload=Node-f6933515-5694-6a13-9979-e8fef6eb8718 -17:53.063 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19202 -17:53.086 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19198 network=tcp -17:53.086 [INFO] TestAgent: started state syncer -17:53.087 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19203 network=tcp -17:53.154 [INFO] TestAgent: Synced node info -17:53.154 [DEBUG] TestAgent: Node info in sync -17:53.154 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:53.177 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/empty from=127.0.0.1:49952 latency=1.402963ms -17:53.190 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/empty from=127.0.0.1:49958 latency=231.813µs -17:53.191 [INFO] TestAgent: Requesting shutdown -17:53.204 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:53.204 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:53.204 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:53.204 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:53.204 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:53.204 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f6933515-5694-6a13-9979-e8fef6eb8718 -17:53.204 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f6933515-5694-6a13-9979-e8fef6eb8718 -17:53.204 [INFO] TestAgent.server: federation state anti-entropy synced -17:53.205 [INFO] TestAgent.server: shutting down server -17:53.205 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:53.205 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:53.205 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:53.205 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:53.205 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:53.206 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:53.206 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:53.206 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:53.206 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:53.206 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:53.206 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:53.206Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:53.206Z [INFO] agent.router.manager: shutting down -17:53.214 [INFO] TestAgent: consul server down -17:53.214 [INFO] TestAgent: shutdown complete -17:53.214 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=tcp -17:53.214 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=udp -17:53.214 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19198 network=tcp -17:53.717 [INFO] TestAgent: Waiting for endpoints to shut down -17:53.718 [INFO] TestAgent: Endpoints down ---- PASS: TestKVGetCommand_Empty (1.13s) -2021-10-31T23:17:52.788Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:52.959Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:52.983 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ea3c9cba-e302-c5b7-d5bb-35302e587cf1 Address:127.0.0.1:19223}]" -17:52.985 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ea3c9cba-e302-c5b7-d5bb-35302e587cf1.dc1 127.0.0.1 -17:52.987 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19223 [Follower]" leader= -17:52.990 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ea3c9cba-e302-c5b7-d5bb-35302e587cf1 127.0.0.1 -2021-10-31T23:17:52.990Z [INFO] agent.router: Initializing LAN area manager -17:52.995 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ea3c9cba-e302-c5b7-d5bb-35302e587cf1.dc1 area=wan -17:52.995 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19218 network=tcp -17:52.998 [INFO] TestAgent.server: Adding LAN server: server="Node-ea3c9cba-e302-c5b7-d5bb-35302e587cf1 (Addr: tcp/127.0.0.1:19223) (DC: dc1)" -17:53.038 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19218 network=udp -17:53.057 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:53.057 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19223 [Candidate]" term=2 -17:53.058 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:53.058 [DEBUG] TestAgent.server.raft: vote granted: from=ea3c9cba-e302-c5b7-d5bb-35302e587cf1 term=2 tally=1 -17:53.058 [INFO] TestAgent.server.raft: election won: tally=1 -17:53.058 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19223 [Leader]" -17:53.058 [INFO] TestAgent.server: cluster leadership acquired -17:53.058 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19223 -17:53.058 [INFO] TestAgent.server: New leader elected: payload=Node-ea3c9cba-e302-c5b7-d5bb-35302e587cf1 -17:53.059 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:53.060 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19219 network=tcp -17:53.060 [INFO] TestAgent: started state syncer -17:53.062 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19224 network=tcp -17:53.063 [INFO] TestAgent: Synced node info -17:53.085 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/bar from=127.0.0.1:50186 latency=22.802631ms -17:53.171 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:53.171 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:53.171 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:53.171 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:53.171 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:53.171 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ea3c9cba-e302-c5b7-d5bb-35302e587cf1 -17:53.171 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ea3c9cba-e302-c5b7-d5bb-35302e587cf1 -17:53.175 [INFO] TestAgent.server: federation state anti-entropy synced -17:53.201 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/baz from=127.0.0.1:50186 latency=107.294132ms -17:53.258 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/zip from=127.0.0.1:50186 latency=1.82909ms -17:53.259 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo/?keys=&separator=%2F from=127.0.0.1:50202 latency=252.758µs -17:53.260 [INFO] TestAgent: Requesting shutdown -17:53.260 [INFO] TestAgent.server: shutting down server -17:53.260 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:53.260 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:53.260 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:53.260 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:53.260 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:53.260 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:53.260 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:53.260 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:53.260 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:53.260 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:53.260 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:53.260Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:53.260Z [INFO] agent.router.manager: shutting down -17:53.260 [INFO] TestAgent: consul server down -17:53.260 [INFO] TestAgent: shutdown complete -17:53.260 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19218 network=tcp -17:53.260 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19218 network=udp -17:53.260 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19219 network=tcp -17:53.761 [INFO] TestAgent: Waiting for endpoints to shut down -17:53.762 [INFO] TestAgent: Endpoints down -2021-10-31T23:17:52.692Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:52.958Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:52.960 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:75dacbaf-cfc3-2a3b-8019-2c567093b6d8 Address:127.0.0.1:19195}]" -17:52.961 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-75dacbaf-cfc3-2a3b-8019-2c567093b6d8.dc1 127.0.0.1 -17:52.961 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= -17:53.047 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-75dacbaf-cfc3-2a3b-8019-2c567093b6d8 127.0.0.1 -2021-10-31T23:17:53.047Z [INFO] agent.router: Initializing LAN area manager -17:53.047 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp -17:53.047 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-75dacbaf-cfc3-2a3b-8019-2c567093b6d8.dc1 area=wan -17:53.047 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp -17:53.048 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp -17:53.048 [INFO] TestAgent: started state syncer -17:53.048 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp -17:53.048 [INFO] TestAgent.server: Adding LAN server: server="Node-75dacbaf-cfc3-2a3b-8019-2c567093b6d8 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" -17:53.058 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:53.058 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 -17:53.058 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:53.058 [DEBUG] TestAgent.server.raft: vote granted: from=75dacbaf-cfc3-2a3b-8019-2c567093b6d8 term=2 tally=1 -17:53.058 [INFO] TestAgent.server.raft: election won: tally=1 -17:53.058 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" -17:53.058 [INFO] TestAgent.server: cluster leadership acquired -17:53.058 [INFO] TestAgent.server: New leader elected: payload=Node-75dacbaf-cfc3-2a3b-8019-2c567093b6d8 -17:53.058 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 -17:53.198 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:53.235 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:53.235 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:53.235 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:53.235 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:53.235 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:53.236 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-75dacbaf-cfc3-2a3b-8019-2c567093b6d8 -17:53.236 [INFO] TestAgent.server: member joined, marking health alive: member=Node-75dacbaf-cfc3-2a3b-8019-2c567093b6d8 -17:53.255 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:35202 latency=49.442719ms -17:53.255 [INFO] TestAgent.server: federation state anti-entropy synced -17:53.257 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:35204 latency=373.163µs -17:53.258 [INFO] TestAgent: Requesting shutdown -17:53.258 [INFO] TestAgent.server: shutting down server -17:53.258 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:53.258 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:53.258 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:53.258 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:53.258 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:53.258 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:53.258 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:53.258 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:53.258 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -17:53.258 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:53.258 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:53.258 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:17:53.258Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:53.258Z [INFO] agent.router.manager: shutting down -17:53.259 [INFO] TestAgent: consul server down -17:53.259 [INFO] TestAgent: shutdown complete -17:53.259 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp -17:53.259 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp -17:53.259 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp -17:53.762 [INFO] TestAgent: Waiting for endpoints to shut down -17:53.762 [INFO] TestAgent: Endpoints down ---- PASS: TestKVGetCommand_Keys (1.17s) +2022-12-04T05:50:32.936Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:33.001Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:33.003 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cbbfe5b2-ce0b-ec2a-3a1d-0fd269da651a Address:127.0.0.1:22272}]" +50:33.004 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22272 [Follower]" leader= +50:33.017 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cbbfe5b2-ce0b-ec2a-3a1d-0fd269da651a.dc1 127.0.0.1 +50:33.018 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cbbfe5b2-ce0b-ec2a-3a1d-0fd269da651a 127.0.0.1 +2022-12-04T05:50:33.018Z [INFO] agent.router: Initializing LAN area manager +50:33.019 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22267 network=udp +50:33.019 [INFO] TestAgent.server: Adding LAN server: server="Node-cbbfe5b2-ce0b-ec2a-3a1d-0fd269da651a (Addr: tcp/127.0.0.1:22272) (DC: dc1)" +50:33.019 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cbbfe5b2-ce0b-ec2a-3a1d-0fd269da651a.dc1 area=wan +50:33.019 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22267 network=tcp +50:33.059 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:33.059 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22272 [Candidate]" term=2 +50:33.059 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:33.060 [DEBUG] TestAgent.server.raft: vote granted: from=cbbfe5b2-ce0b-ec2a-3a1d-0fd269da651a term=2 tally=1 +50:33.060 [INFO] TestAgent.server.raft: election won: tally=1 +50:33.060 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22272 [Leader]" +50:33.060 [INFO] TestAgent.server: cluster leadership acquired +50:33.060 [INFO] TestAgent.server: New leader elected: payload=Node-cbbfe5b2-ce0b-ec2a-3a1d-0fd269da651a +50:33.060 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22272 +50:33.061 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:33.093 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22268 network=tcp +50:33.093 [INFO] TestAgent: started state syncer +50:33.095 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22273 network=tcp +50:33.096 [INFO] TestAgent: Synced node info +50:33.098 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/bar from=127.0.0.1:53038 latency=1.028089ms +50:33.161 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/baz from=127.0.0.1:53038 latency=48.694386ms +50:33.162 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/zip from=127.0.0.1:53038 latency=516.016µs +50:33.199 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo/?keys=&separator=%2F from=127.0.0.1:53040 latency=35.049367ms +50:33.202 [INFO] TestAgent: Requesting shutdown +50:33.202 [INFO] TestAgent.server: shutting down server +50:33.202 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:33.202 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:33.203Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:33.203Z [INFO] agent.router.manager: shutting down +50:33.203 [INFO] TestAgent: consul server down +50:33.203 [INFO] TestAgent: shutdown complete +50:33.203 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22267 network=tcp +50:33.203 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22267 network=udp +50:33.203 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22268 network=tcp +50:33.205 [ERROR] TestAgent.server: failed to establish leadership: error="error generating CA root certificate: error computing next serial number: raft is already shutdown" +50:33.205 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=0 retry_limit=3 error="raft is already shutdown" +50:33.205 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=1 retry_limit=3 error="raft is already shutdown" +50:33.704 [INFO] TestAgent: Waiting for endpoints to shut down +50:33.704 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand_Keys (0.98s) +2022-12-04T05:50:32.926Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:32.999Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:33.003 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dc933c42-2134-96ac-0921-704bba07047b Address:127.0.0.1:22265}]" +50:33.004 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dc933c42-2134-96ac-0921-704bba07047b.dc1 127.0.0.1 +50:33.004 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22265 [Follower]" leader= +50:33.064 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:33.064 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22265 [Candidate]" term=2 +50:33.065 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=dc933c42-2134-96ac-0921-704bba07047b fallback=127.0.0.1:22265 error="Could not find address for server id dc933c42-2134-96ac-0921-704bba07047b" +50:33.065 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:33.065 [DEBUG] TestAgent.server.raft: vote granted: from=dc933c42-2134-96ac-0921-704bba07047b term=2 tally=1 +50:33.065 [INFO] TestAgent.server.raft: election won: tally=1 +50:33.065 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22265 [Leader]" +50:33.112 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dc933c42-2134-96ac-0921-704bba07047b 127.0.0.1 +2022-12-04T05:50:33.112Z [INFO] agent.router: Initializing LAN area manager +50:33.161 [INFO] TestAgent.server: Adding LAN server: server="Node-dc933c42-2134-96ac-0921-704bba07047b (Addr: tcp/127.0.0.1:22265) (DC: dc1)" +50:33.161 [INFO] TestAgent.server: cluster leadership acquired +50:33.161 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dc933c42-2134-96ac-0921-704bba07047b.dc1 area=wan +50:33.161 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22265 +50:33.161 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=udp +50:33.162 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=tcp +50:33.162 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:33.162 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22261 network=tcp +50:33.162 [INFO] TestAgent: started state syncer +50:33.163 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:33.163 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:33.163 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:33.163 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:33.163 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:33.163 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dc933c42-2134-96ac-0921-704bba07047b +50:33.163 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dc933c42-2134-96ac-0921-704bba07047b +50:33.164 [INFO] TestAgent.server: New leader elected: payload=Node-dc933c42-2134-96ac-0921-704bba07047b +50:33.164 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22266 network=tcp +50:33.199 [INFO] TestAgent: Synced node info +50:33.199 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dc933c42-2134-96ac-0921-704bba07047b +50:33.199 [INFO] TestAgent.server: federation state anti-entropy synced +50:33.202 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/empty from=127.0.0.1:45336 latency=1.929187ms +50:33.270 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/empty from=127.0.0.1:45338 latency=1.210013ms +50:33.271 [INFO] TestAgent: Requesting shutdown +50:33.271 [INFO] TestAgent.server: shutting down server +50:33.271 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:33.271 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:33.271 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:33.271 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:33.271 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:33.271 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:33.271 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:33.271 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:33.271 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:33.271 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:33.271 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:33.271Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:33.271Z [INFO] agent.router.manager: shutting down +50:33.271 [INFO] TestAgent: consul server down +50:33.271 [INFO] TestAgent: shutdown complete +50:33.271 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=tcp +50:33.271 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=udp +50:33.272 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22261 network=tcp +50:33.772 [INFO] TestAgent: Waiting for endpoints to shut down +50:33.772 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand_Empty (1.05s) +2022-12-04T05:50:32.864Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:33.115Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:33.158 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:65554650-fcb4-0ba2-e0ad-0ea0a7dea02e Address:127.0.0.1:22258}]" +50:33.159 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-65554650-fcb4-0ba2-e0ad-0ea0a7dea02e.dc1 127.0.0.1 +50:33.160 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-65554650-fcb4-0ba2-e0ad-0ea0a7dea02e 127.0.0.1 +2022-12-04T05:50:33.160Z [INFO] agent.router: Initializing LAN area manager +50:33.160 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=udp +50:33.160 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22258 [Follower]" leader= +50:33.160 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=tcp +50:33.160 [INFO] TestAgent.server: Adding LAN server: server="Node-65554650-fcb4-0ba2-e0ad-0ea0a7dea02e (Addr: tcp/127.0.0.1:22258) (DC: dc1)" +50:33.160 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-65554650-fcb4-0ba2-e0ad-0ea0a7dea02e.dc1 area=wan +50:33.161 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22254 network=tcp +50:33.161 [INFO] TestAgent: started state syncer +50:33.161 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22259 network=tcp +50:33.205 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:33.205 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22258 [Candidate]" term=2 +50:33.248 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:33.248 [DEBUG] TestAgent.server.raft: vote granted: from=65554650-fcb4-0ba2-e0ad-0ea0a7dea02e term=2 tally=1 +50:33.248 [INFO] TestAgent.server.raft: election won: tally=1 +50:33.248 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22258 [Leader]" +50:33.250 [INFO] TestAgent.server: cluster leadership acquired +50:33.250 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22258 +50:33.251 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:33.252 [INFO] TestAgent.server: New leader elected: payload=Node-65554650-fcb4-0ba2-e0ad-0ea0a7dea02e +50:33.257 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:33.257 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:33.257 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:33.257 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:33.257 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:33.257 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-65554650-fcb4-0ba2-e0ad-0ea0a7dea02e +50:33.257 [INFO] TestAgent.server: member joined, marking health alive: member=Node-65554650-fcb4-0ba2-e0ad-0ea0a7dea02e +50:33.259 [INFO] TestAgent.server: federation state anti-entropy synced +50:33.276 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:33.301 [INFO] TestAgent: Synced node info +50:33.301 [DEBUG] TestAgent: Node info in sync +50:33.391 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:57342 latency=2.8071ms +50:33.394 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:57344 latency=420.58µs +50:33.394 [INFO] TestAgent: Requesting shutdown +50:33.394 [INFO] TestAgent.server: shutting down server +50:33.394 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:33.394 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:33.394 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:33.394 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:33.394 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:33.394 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:33.394 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:33.394 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:33.394 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:33.394 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:33.394 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:33.394Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:33.394Z [INFO] agent.router.manager: shutting down +50:33.395 [INFO] TestAgent: consul server down +50:33.395 [INFO] TestAgent: shutdown complete +50:33.395 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=tcp +50:33.395 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=udp +50:33.395 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22254 network=tcp +50:33.896 [INFO] TestAgent: Waiting for endpoints to shut down +50:33.896 [INFO] TestAgent: Endpoints down --- PASS: TestKVGetCommand_DetailedBase64 (1.18s) -2021-10-31T23:17:52.832Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:52.960Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:52.961 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ba350f9b-4965-9067-69ad-5387ca8cf75e Address:127.0.0.1:19237}]" -17:52.962 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19237 [Follower]" leader= -17:52.998 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ba350f9b-4965-9067-69ad-5387ca8cf75e.dc1 127.0.0.1 -17:53.058 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:53.058 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19237 [Candidate]" term=2 -17:53.058 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=ba350f9b-4965-9067-69ad-5387ca8cf75e fallback=127.0.0.1:19237 error="Could not find address for server id ba350f9b-4965-9067-69ad-5387ca8cf75e" -17:53.058 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:53.058 [DEBUG] TestAgent.server.raft: vote granted: from=ba350f9b-4965-9067-69ad-5387ca8cf75e term=2 tally=1 -17:53.058 [INFO] TestAgent.server.raft: election won: tally=1 -17:53.058 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19237 [Leader]" -17:53.062 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ba350f9b-4965-9067-69ad-5387ca8cf75e 127.0.0.1 -2021-10-31T23:17:53.062Z [INFO] agent.router: Initializing LAN area manager -17:53.062 [INFO] TestAgent.server: cluster leadership acquired -17:53.062 [INFO] TestAgent.server: Adding LAN server: server="Node-ba350f9b-4965-9067-69ad-5387ca8cf75e (Addr: tcp/127.0.0.1:19237) (DC: dc1)" -17:53.062 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ba350f9b-4965-9067-69ad-5387ca8cf75e.dc1 area=wan -17:53.062 [INFO] TestAgent.server: New leader elected: payload=Node-ba350f9b-4965-9067-69ad-5387ca8cf75e -17:53.062 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19232 network=tcp -17:53.062 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19237 -17:53.062 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19232 network=udp -17:53.063 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19233 network=tcp -17:53.063 [INFO] TestAgent: started state syncer -17:53.063 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19238 network=tcp -17:53.086 [INFO] TestAgent: Synced node info -17:53.086 [DEBUG] TestAgent: Node info in sync -17:53.098 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:56140 latency=10.815015ms -17:53.170 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:53.174 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:53.174 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:53.174 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:53.174 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:53.174 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:53.175 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba350f9b-4965-9067-69ad-5387ca8cf75e -17:53.175 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ba350f9b-4965-9067-69ad-5387ca8cf75e -17:53.175 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ba350f9b-4965-9067-69ad-5387ca8cf75e -17:53.176 [INFO] TestAgent.server: federation state anti-entropy synced -17:53.255 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:56146 latency=76.231713ms -17:53.256 [INFO] TestAgent: Requesting shutdown -17:53.256 [INFO] TestAgent.server: shutting down server -17:53.256 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:53.256 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:53.256 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:53.256 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:53.256 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:53.256 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:53.256 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:53.256 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:53.256 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:53.256 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:53.256 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:53.259Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:53.259Z [INFO] agent.router.manager: shutting down -17:53.260 [INFO] TestAgent: consul server down -17:53.260 [INFO] TestAgent: shutdown complete -17:53.260 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19232 network=tcp -17:53.260 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19232 network=udp -17:53.260 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19233 network=tcp -17:53.762 [INFO] TestAgent: Waiting for endpoints to shut down -17:53.762 [INFO] TestAgent: Endpoints down ---- PASS: TestKVGetCommand_Base64 (1.17s) -2021-10-31T23:17:52.783Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:52.888Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:52.912 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7dd9533f-d785-a7d2-7d74-4d5181faec77 Address:127.0.0.1:19251}]" -17:52.913 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7dd9533f-d785-a7d2-7d74-4d5181faec77.dc1 127.0.0.1 -17:52.918 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19251 [Follower]" leader= -17:52.929 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7dd9533f-d785-a7d2-7d74-4d5181faec77 127.0.0.1 -2021-10-31T23:17:52.954Z [INFO] agent.router: Initializing LAN area manager -17:52.955 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19246 network=udp -17:52.955 [INFO] TestAgent.server: Adding LAN server: server="Node-7dd9533f-d785-a7d2-7d74-4d5181faec77 (Addr: tcp/127.0.0.1:19251) (DC: dc1)" -17:52.956 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7dd9533f-d785-a7d2-7d74-4d5181faec77.dc1 area=wan -17:52.956 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19246 network=tcp -17:52.956 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19247 network=tcp -17:52.956 [INFO] TestAgent: started state syncer -17:52.957 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19252 network=tcp -17:53.057 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:53.057 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19251 [Candidate]" term=2 -17:53.058 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:53.058 [DEBUG] TestAgent.server.raft: vote granted: from=7dd9533f-d785-a7d2-7d74-4d5181faec77 term=2 tally=1 -17:53.058 [INFO] TestAgent.server.raft: election won: tally=1 -17:53.058 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19251 [Leader]" -17:53.058 [INFO] TestAgent.server: cluster leadership acquired -17:53.058 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19251 -17:53.059 [INFO] TestAgent.server: New leader elected: payload=Node-7dd9533f-d785-a7d2-7d74-4d5181faec77 -17:53.063 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:53.203 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:53.203 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:53.203 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:53.203 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:53.203 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:53.203 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7dd9533f-d785-a7d2-7d74-4d5181faec77 -17:53.203 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7dd9533f-d785-a7d2-7d74-4d5181faec77 -17:53.204 [INFO] TestAgent.server: federation state anti-entropy synced -17:53.300 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:46922 latency=2.463924ms -17:53.303 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:46924 latency=278.032µs -17:53.303 [INFO] TestAgent: Requesting shutdown -17:53.303 [INFO] TestAgent.server: shutting down server -17:53.303 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:53.303 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:53.303 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:53.303 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:53.303 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:53.303 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:53.303 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:53.303 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:53.303 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:53.303 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:53.303 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:53.303 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:53.303Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:53.303Z [INFO] agent.router.manager: shutting down -17:53.303 [INFO] TestAgent: consul server down -17:53.303 [INFO] TestAgent: shutdown complete -17:53.303 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19246 network=tcp -17:53.303 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19246 network=udp -17:53.303 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19247 network=tcp -17:53.805 [INFO] TestAgent: Waiting for endpoints to shut down -17:53.805 [INFO] TestAgent: Endpoints down ---- PASS: TestKVGetCommand (1.22s) -2021-10-31T23:17:52.713Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:52.923Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:52.931 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1cc3519e-e379-2b7e-d832-f9c4e9abb541 Address:127.0.0.1:19209}]" -17:52.932 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1cc3519e-e379-2b7e-d832-f9c4e9abb541.dc1 127.0.0.1 -17:52.932 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1cc3519e-e379-2b7e-d832-f9c4e9abb541 127.0.0.1 -2021-10-31T23:17:52.932Z [INFO] agent.router: Initializing LAN area manager -17:52.966 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19209 [Follower]" leader= -17:52.966 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19204 network=tcp -17:53.046 [INFO] TestAgent.server: Adding LAN server: server="Node-1cc3519e-e379-2b7e-d832-f9c4e9abb541 (Addr: tcp/127.0.0.1:19209) (DC: dc1)" -17:53.046 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19204 network=udp -17:53.046 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19205 network=tcp -17:53.046 [INFO] TestAgent: started state syncer -17:53.047 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1cc3519e-e379-2b7e-d832-f9c4e9abb541.dc1 area=wan -17:53.047 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19210 network=tcp -17:53.176 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:53.176 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19209 [Candidate]" term=2 -17:53.176 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:53.176 [DEBUG] TestAgent.server.raft: vote granted: from=1cc3519e-e379-2b7e-d832-f9c4e9abb541 term=2 tally=1 -17:53.176 [INFO] TestAgent.server.raft: election won: tally=1 -17:53.176 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19209 [Leader]" -17:53.176 [INFO] TestAgent.server: cluster leadership acquired -17:53.176 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19209 -17:53.177 [INFO] TestAgent.server: New leader elected: payload=Node-1cc3519e-e379-2b7e-d832-f9c4e9abb541 -17:53.202 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:53.235 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:53.235 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:53.235 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:53.235 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:53.235 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:53.235 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1cc3519e-e379-2b7e-d832-f9c4e9abb541 -17:53.235 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1cc3519e-e379-2b7e-d832-f9c4e9abb541 -17:53.236 [INFO] TestAgent.server: federation state anti-entropy synced -17:53.313 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:38994 latency=577.725µs -17:53.322 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/b from=127.0.0.1:38994 latency=718.86µs -17:53.323 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/c from=127.0.0.1:38994 latency=327.706µs -17:53.324 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo?recurse= from=127.0.0.1:38996 latency=181.627µs -17:53.324 [INFO] TestAgent: Requesting shutdown -17:53.324 [INFO] TestAgent.server: shutting down server -17:53.324 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:53.324 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:53.324 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:53.324 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:53.324 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:53.324 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:53.324 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:53.324 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:53.324 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:53.324 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:53.324 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:53.324 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:53.324Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:53.324Z [INFO] agent.router.manager: shutting down -17:53.325 [INFO] TestAgent: consul server down -17:53.325 [INFO] TestAgent: shutdown complete -17:53.325 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19204 network=tcp -17:53.325 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19204 network=udp -17:53.325 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19205 network=tcp -17:53.825 [INFO] TestAgent: Waiting for endpoints to shut down -17:53.825 [INFO] TestAgent: Endpoints down ---- PASS: TestKVGetCommand_RecurseBase64 (1.24s) -2021-10-31T23:17:52.964Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:53.173Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:53.206 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:eb8ddb4e-27c7-3b7d-0f01-820a18be1aa9 Address:127.0.0.1:19216}]" -17:53.209 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19216 [Follower]" leader= -17:53.215 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-eb8ddb4e-27c7-3b7d-0f01-820a18be1aa9.dc1 127.0.0.1 -17:53.256 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-eb8ddb4e-27c7-3b7d-0f01-820a18be1aa9 127.0.0.1 -2021-10-31T23:17:53.256Z [INFO] agent.router: Initializing LAN area manager -17:53.257 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19211 network=udp -17:53.257 [INFO] TestAgent.server: Adding LAN server: server="Node-eb8ddb4e-27c7-3b7d-0f01-820a18be1aa9 (Addr: tcp/127.0.0.1:19216) (DC: dc1)" -17:53.257 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-eb8ddb4e-27c7-3b7d-0f01-820a18be1aa9.dc1 area=wan -17:53.258 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19211 network=tcp -17:53.260 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19212 network=tcp -17:53.260 [INFO] TestAgent: started state syncer -17:53.260 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19217 network=tcp -17:53.325 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:53.325 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19216 [Candidate]" term=2 -17:53.325 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:53.325 [DEBUG] TestAgent.server.raft: vote granted: from=eb8ddb4e-27c7-3b7d-0f01-820a18be1aa9 term=2 tally=1 -17:53.325 [INFO] TestAgent.server.raft: election won: tally=1 -17:53.325 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19216 [Leader]" -17:53.325 [INFO] TestAgent.server: cluster leadership acquired -17:53.325 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19216 -17:53.325 [INFO] TestAgent.server: New leader elected: payload=Node-eb8ddb4e-27c7-3b7d-0f01-820a18be1aa9 -17:53.326 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:53.327 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:53.327 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:53.327 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:53.327 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:53.327 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:53.327 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eb8ddb4e-27c7-3b7d-0f01-820a18be1aa9 -17:53.327 [INFO] TestAgent.server: member joined, marking health alive: member=Node-eb8ddb4e-27c7-3b7d-0f01-820a18be1aa9 -17:53.328 [INFO] TestAgent.server: federation state anti-entropy synced -17:53.465 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/c from=127.0.0.1:53246 latency=1.349814ms -17:53.470 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:53246 latency=548.54µs -17:53.478 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/b from=127.0.0.1:53246 latency=555.972µs -17:53.483 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo?recurse= from=127.0.0.1:53248 latency=373.363µs -17:53.486 [INFO] TestAgent: Requesting shutdown -17:53.486 [INFO] TestAgent.server: shutting down server -17:53.486 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:53.486 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:53.486 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:53.486 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:53.486 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:53.486 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:53.486 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:53.486 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -17:53.486 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:53.486 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:53.486 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:53.486 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:53.486Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:53.486Z [INFO] agent.router.manager: shutting down -17:53.486 [INFO] TestAgent: consul server down -17:53.486 [INFO] TestAgent: shutdown complete -17:53.486 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19211 network=tcp -17:53.486 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19211 network=udp -17:53.486 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19212 network=tcp -17:53.989 [INFO] TestAgent: Waiting for endpoints to shut down -17:53.989 [INFO] TestAgent: Endpoints down ---- PASS: TestKVGetCommand_Recurse (1.40s) -2021-10-31T23:17:53.055Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:53.260Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:53.278 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3925cf98-cace-1381-6887-daed1b4c8b42 Address:127.0.0.1:19230}]" -17:53.279 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3925cf98-cace-1381-6887-daed1b4c8b42.dc1 127.0.0.1 -17:53.280 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3925cf98-cace-1381-6887-daed1b4c8b42 127.0.0.1 -2021-10-31T23:17:53.280Z [INFO] agent.router: Initializing LAN area manager -17:53.280 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19225 network=udp -17:53.280 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19230 [Follower]" leader= -17:53.280 [INFO] TestAgent.server: Adding LAN server: server="Node-3925cf98-cace-1381-6887-daed1b4c8b42 (Addr: tcp/127.0.0.1:19230) (DC: dc1)" -17:53.280 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3925cf98-cace-1381-6887-daed1b4c8b42.dc1 area=wan -17:53.280 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19225 network=tcp -17:53.283 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19226 network=tcp -17:53.283 [INFO] TestAgent: started state syncer -17:53.283 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19231 network=tcp -17:53.345 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:53.345 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19230 [Candidate]" term=2 -17:53.346 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:53.346 [DEBUG] TestAgent.server.raft: vote granted: from=3925cf98-cace-1381-6887-daed1b4c8b42 term=2 tally=1 -17:53.346 [INFO] TestAgent.server.raft: election won: tally=1 -17:53.346 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19230 [Leader]" -17:53.346 [INFO] TestAgent.server: cluster leadership acquired -17:53.346 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19230 -17:53.347 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:53.348 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:53.348 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:53.348 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:53.348 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:53.348 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:53.348 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3925cf98-cace-1381-6887-daed1b4c8b42 -17:53.348 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3925cf98-cace-1381-6887-daed1b4c8b42 -17:53.349 [INFO] TestAgent.server: New leader elected: payload=Node-3925cf98-cace-1381-6887-daed1b4c8b42 -17:53.349 [INFO] TestAgent.server: federation state anti-entropy synced -17:53.401 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:53.402 [INFO] TestAgent: Synced node info -17:53.402 [DEBUG] TestAgent: Node info in sync -17:53.687 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:53534 latency=494.408µs -17:53.687 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:53536 latency=138.274µs -17:53.688 [INFO] TestAgent: Requesting shutdown -17:53.688 [INFO] TestAgent.server: shutting down server -17:53.688 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:53.688 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:53.688 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:53.688 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:53.688 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:53.688 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:53.688 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:53.688 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:53.688 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:53.688 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:53.688 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:53.688Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:53.688Z [INFO] agent.router.manager: shutting down -17:53.688 [INFO] TestAgent: consul server down -17:53.688 [INFO] TestAgent: shutdown complete -17:53.688 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19225 network=tcp -17:53.688 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19225 network=udp -17:53.688 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19226 network=tcp -17:54.190 [INFO] TestAgent: Waiting for endpoints to shut down -17:54.190 [INFO] TestAgent: Endpoints down ---- PASS: TestKVGetCommand_Detailed (1.60s) +2022-12-04T05:50:33.035Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:33.269Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:33.297 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:47dc73e3-cf19-6775-1dd7-6797f4f9b25c Address:127.0.0.1:22300}]" +50:33.298 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-47dc73e3-cf19-6775-1dd7-6797f4f9b25c.dc1 127.0.0.1 +50:33.299 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-47dc73e3-cf19-6775-1dd7-6797f4f9b25c 127.0.0.1 +2022-12-04T05:50:33.299Z [INFO] agent.router: Initializing LAN area manager +50:33.299 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22295 network=udp +50:33.299 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22300 [Follower]" leader= +50:33.300 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-47dc73e3-cf19-6775-1dd7-6797f4f9b25c.dc1 area=wan +50:33.300 [INFO] TestAgent.server: Adding LAN server: server="Node-47dc73e3-cf19-6775-1dd7-6797f4f9b25c (Addr: tcp/127.0.0.1:22300) (DC: dc1)" +50:33.300 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22295 network=tcp +50:33.309 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22296 network=tcp +50:33.309 [INFO] TestAgent: started state syncer +50:33.309 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22301 network=tcp +50:33.364 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:33.364 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22300 [Candidate]" term=2 +50:33.364 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:33.364 [DEBUG] TestAgent.server.raft: vote granted: from=47dc73e3-cf19-6775-1dd7-6797f4f9b25c term=2 tally=1 +50:33.364 [INFO] TestAgent.server.raft: election won: tally=1 +50:33.364 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22300 [Leader]" +50:33.364 [INFO] TestAgent.server: cluster leadership acquired +50:33.365 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22300 +50:33.365 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:33.367 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:33.367 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:33.367 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:33.367 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:33.367 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:33.367 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-47dc73e3-cf19-6775-1dd7-6797f4f9b25c +50:33.367 [INFO] TestAgent.server: member joined, marking health alive: member=Node-47dc73e3-cf19-6775-1dd7-6797f4f9b25c +50:33.367 [INFO] TestAgent.server: New leader elected: payload=Node-47dc73e3-cf19-6775-1dd7-6797f4f9b25c +50:33.368 [INFO] TestAgent.server: federation state anti-entropy synced +50:33.425 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:57466 latency=690.306µs +50:33.426 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:57468 latency=194.579µs +50:33.426 [INFO] TestAgent: Requesting shutdown +50:33.426 [INFO] TestAgent.server: shutting down server +50:33.426 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:33.426 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:33.426 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:33.426 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:33.427 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:33.427 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:33.427 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:33.427 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:33.427 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:33.427 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:33.427 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:33.427 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:33.427Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:33.427Z [INFO] agent.router.manager: shutting down +50:33.427 [INFO] TestAgent: consul server down +50:33.427 [INFO] TestAgent: shutdown complete +50:33.427 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22295 network=tcp +50:33.427 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22295 network=udp +50:33.427 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22296 network=tcp +50:33.928 [INFO] TestAgent: Waiting for endpoints to shut down +50:33.928 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand (1.21s) +2022-12-04T05:50:33.089Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:33.271Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:33.316 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2f655628-1445-edb0-56d6-903247026170 Address:127.0.0.1:22279}]" +50:33.318 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2f655628-1445-edb0-56d6-903247026170.dc1 127.0.0.1 +50:33.320 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22279 [Follower]" leader= +50:33.361 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2f655628-1445-edb0-56d6-903247026170 127.0.0.1 +2022-12-04T05:50:33.361Z [INFO] agent.router: Initializing LAN area manager +50:33.361 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22274 network=udp +50:33.362 [INFO] TestAgent.server: Adding LAN server: server="Node-2f655628-1445-edb0-56d6-903247026170 (Addr: tcp/127.0.0.1:22279) (DC: dc1)" +50:33.362 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2f655628-1445-edb0-56d6-903247026170.dc1 area=wan +50:33.362 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22274 network=tcp +50:33.363 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22275 network=tcp +50:33.363 [INFO] TestAgent: started state syncer +50:33.363 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22280 network=tcp +50:33.388 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:33.388 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22279 [Candidate]" term=2 +50:33.388 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:33.388 [DEBUG] TestAgent.server.raft: vote granted: from=2f655628-1445-edb0-56d6-903247026170 term=2 tally=1 +50:33.388 [INFO] TestAgent.server.raft: election won: tally=1 +50:33.388 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22279 [Leader]" +50:33.389 [INFO] TestAgent.server: cluster leadership acquired +50:33.389 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22279 +50:33.389 [INFO] TestAgent.server: New leader elected: payload=Node-2f655628-1445-edb0-56d6-903247026170 +50:33.392 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:33.394 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:33.394 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:33.394 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:33.394 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:33.394 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:33.394 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2f655628-1445-edb0-56d6-903247026170 +50:33.394 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2f655628-1445-edb0-56d6-903247026170 +50:33.395 [INFO] TestAgent.server: federation state anti-entropy synced +50:33.454 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/b from=127.0.0.1:39286 latency=712.079µs +50:33.455 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/c from=127.0.0.1:39286 latency=502.992µs +50:33.456 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:39286 latency=589.333µs +50:33.470 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo?recurse= from=127.0.0.1:39288 latency=1.474202ms +50:33.472 [INFO] TestAgent: Requesting shutdown +50:33.472 [INFO] TestAgent.server: shutting down server +50:33.472 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:33.472 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:33.472 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:33.472 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:33.472 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:33.472 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:33.472 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:33.472 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:33.472 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:33.472 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:33.472 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:33.473 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:33.473Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:33.473Z [INFO] agent.router.manager: shutting down +50:33.473 [INFO] TestAgent: consul server down +50:33.473 [INFO] TestAgent: shutdown complete +50:33.473 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22274 network=tcp +50:33.473 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22274 network=udp +50:33.473 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22275 network=tcp +50:33.976 [INFO] TestAgent: Waiting for endpoints to shut down +50:33.976 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand_Recurse (1.26s) +2022-12-04T05:50:32.939Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:33.071Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:33.081 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5b4e8ead-4f62-6cf2-87b5-d9b9d5ed8fd9 Address:127.0.0.1:22293}]" +50:33.082 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5b4e8ead-4f62-6cf2-87b5-d9b9d5ed8fd9.dc1 127.0.0.1 +50:33.082 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5b4e8ead-4f62-6cf2-87b5-d9b9d5ed8fd9 127.0.0.1 +2022-12-04T05:50:33.082Z [INFO] agent.router: Initializing LAN area manager +50:33.083 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22288 network=udp +50:33.084 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22293 [Follower]" leader= +50:33.085 [INFO] TestAgent.server: Adding LAN server: server="Node-5b4e8ead-4f62-6cf2-87b5-d9b9d5ed8fd9 (Addr: tcp/127.0.0.1:22293) (DC: dc1)" +50:33.085 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5b4e8ead-4f62-6cf2-87b5-d9b9d5ed8fd9.dc1 area=wan +50:33.085 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22288 network=tcp +50:33.086 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22289 network=tcp +50:33.086 [INFO] TestAgent: started state syncer +50:33.086 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22294 network=tcp +50:33.199 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:33.199 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22293 [Candidate]" term=2 +50:33.212 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:33.212 [DEBUG] TestAgent.server.raft: vote granted: from=5b4e8ead-4f62-6cf2-87b5-d9b9d5ed8fd9 term=2 tally=1 +50:33.212 [INFO] TestAgent.server.raft: election won: tally=1 +50:33.212 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22293 [Leader]" +50:33.260 [INFO] TestAgent.server: cluster leadership acquired +50:33.260 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22293 +50:33.261 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:33.263 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:33.263 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:33.263 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:33.263 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:33.263 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:33.263 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5b4e8ead-4f62-6cf2-87b5-d9b9d5ed8fd9 +50:33.263 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5b4e8ead-4f62-6cf2-87b5-d9b9d5ed8fd9 +50:33.264 [INFO] TestAgent.server: New leader elected: payload=Node-5b4e8ead-4f62-6cf2-87b5-d9b9d5ed8fd9 +50:33.264 [INFO] TestAgent.server: federation state anti-entropy synced +50:33.468 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:33.469 [INFO] TestAgent: Synced node info +50:33.476 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:33.476 [DEBUG] TestAgent: Node info in sync +50:33.476 [DEBUG] TestAgent: Node info in sync +50:33.496 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/not-a-real-key from=127.0.0.1:46216 latency=109.027µs +50:33.500 [INFO] TestAgent: Requesting shutdown +50:33.500 [INFO] TestAgent.server: shutting down server +50:33.500 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:33.500 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:33.500 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:33.500 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:33.500 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:33.500 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:33.500 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:33.500 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:33.500 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:33.500 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:33.501 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:33.501Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:33.501Z [INFO] agent.router.manager: shutting down +50:33.501 [INFO] TestAgent: consul server down +50:33.501 [INFO] TestAgent: shutdown complete +50:33.501 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22288 network=tcp +50:33.501 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22288 network=udp +50:33.501 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22289 network=tcp +50:34.004 [INFO] TestAgent: Waiting for endpoints to shut down +50:34.004 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand_Missing (1.28s) +2022-12-04T05:50:33.114Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:33.274Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:33.318 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ad700463-e448-d889-a714-ca5a7278ade1 Address:127.0.0.1:22307}]" +50:33.319 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22307 [Follower]" leader= +50:33.357 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ad700463-e448-d889-a714-ca5a7278ade1.dc1 127.0.0.1 +50:33.358 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ad700463-e448-d889-a714-ca5a7278ade1 127.0.0.1 +2022-12-04T05:50:33.358Z [INFO] agent.router: Initializing LAN area manager +50:33.358 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22302 network=udp +50:33.359 [INFO] TestAgent.server: Adding LAN server: server="Node-ad700463-e448-d889-a714-ca5a7278ade1 (Addr: tcp/127.0.0.1:22307) (DC: dc1)" +50:33.359 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ad700463-e448-d889-a714-ca5a7278ade1.dc1 area=wan +50:33.359 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22302 network=tcp +50:33.359 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22303 network=tcp +50:33.359 [INFO] TestAgent: started state syncer +50:33.360 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22308 network=tcp +50:33.384 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:33.384 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22307 [Candidate]" term=2 +50:33.385 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:33.385 [DEBUG] TestAgent.server.raft: vote granted: from=ad700463-e448-d889-a714-ca5a7278ade1 term=2 tally=1 +50:33.385 [INFO] TestAgent.server.raft: election won: tally=1 +50:33.385 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22307 [Leader]" +50:33.385 [INFO] TestAgent.server: cluster leadership acquired +50:33.385 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22307 +50:33.386 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:33.387 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:33.387 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:33.387 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:33.387 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:33.387 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:33.387 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ad700463-e448-d889-a714-ca5a7278ade1 +50:33.387 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ad700463-e448-d889-a714-ca5a7278ade1 +50:33.388 [INFO] TestAgent.server: New leader elected: payload=Node-ad700463-e448-d889-a714-ca5a7278ade1 +50:33.389 [INFO] TestAgent.server: federation state anti-entropy synced +50:33.509 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:59612 latency=659.964µs +50:33.511 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/b from=127.0.0.1:59612 latency=619.417µs +50:33.512 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/c from=127.0.0.1:59612 latency=557.798µs +50:33.533 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo?recurse= from=127.0.0.1:59614 latency=8.567146ms +50:33.544 [INFO] TestAgent: Requesting shutdown +50:33.544 [INFO] TestAgent.server: shutting down server +50:33.544 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:33.544 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:33.544 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:33.544 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:33.545 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:33.545 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:33.545 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:33.545 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:33.545 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:33.545 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:33.545 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:33.545 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:33.545Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:33.545Z [INFO] agent.router.manager: shutting down +50:33.545 [INFO] TestAgent: consul server down +50:33.545 [INFO] TestAgent: shutdown complete +50:33.545 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22302 network=tcp +50:33.545 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22302 network=udp +50:33.545 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22303 network=tcp +50:34.048 [INFO] TestAgent: Waiting for endpoints to shut down +50:34.048 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand_RecurseBase64 (1.33s) +2022-12-04T05:50:33.258Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:33.359Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:33.361 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4978c976-8606-ac7f-f1a3-4aafca3e0959 Address:127.0.0.1:22314}]" +50:33.361 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4978c976-8606-ac7f-f1a3-4aafca3e0959.dc1 127.0.0.1 +50:33.362 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4978c976-8606-ac7f-f1a3-4aafca3e0959 127.0.0.1 +2022-12-04T05:50:33.362Z [INFO] agent.router: Initializing LAN area manager +50:33.362 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22309 network=udp +50:33.362 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22314 [Follower]" leader= +50:33.363 [INFO] TestAgent.server: Adding LAN server: server="Node-4978c976-8606-ac7f-f1a3-4aafca3e0959 (Addr: tcp/127.0.0.1:22314) (DC: dc1)" +50:33.363 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4978c976-8606-ac7f-f1a3-4aafca3e0959.dc1 area=wan +50:33.363 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22309 network=tcp +50:33.364 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22310 network=tcp +50:33.364 [INFO] TestAgent: started state syncer +50:33.364 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22315 network=tcp +50:33.408 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:33.408 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22314 [Candidate]" term=2 +50:33.408 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:33.408 [DEBUG] TestAgent.server.raft: vote granted: from=4978c976-8606-ac7f-f1a3-4aafca3e0959 term=2 tally=1 +50:33.408 [INFO] TestAgent.server.raft: election won: tally=1 +50:33.408 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22314 [Leader]" +50:33.408 [INFO] TestAgent.server: cluster leadership acquired +50:33.409 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22314 +50:33.410 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:33.412 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:33.412 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:33.412 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:33.412 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:33.412 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:33.412 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4978c976-8606-ac7f-f1a3-4aafca3e0959 +50:33.412 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4978c976-8606-ac7f-f1a3-4aafca3e0959 +50:33.412 [INFO] TestAgent.server: New leader elected: payload=Node-4978c976-8606-ac7f-f1a3-4aafca3e0959 +50:33.417 [INFO] TestAgent.server: federation state anti-entropy synced +50:33.678 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:36856 latency=787.539µs +50:33.679 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:36858 latency=271.997µs +50:33.679 [INFO] TestAgent: Requesting shutdown +50:33.679 [INFO] TestAgent.server: shutting down server +50:33.679 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:33.679 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:33.679 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:33.679 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:33.679 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:33.679 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:33.679 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:33.679 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:33.679 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:33.679 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:33.679 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:33.680 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:33.680Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:33.680Z [INFO] agent.router.manager: shutting down +50:33.680 [INFO] TestAgent: consul server down +50:33.680 [INFO] TestAgent: shutdown complete +50:33.680 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22309 network=tcp +50:33.680 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22309 network=udp +50:33.680 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22310 network=tcp +50:34.184 [INFO] TestAgent: Waiting for endpoints to shut down +50:34.184 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand_Base64 (1.46s) +2022-12-04T05:50:33.038Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:33.200Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:33.265 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6ac1981c-09fc-ca01-00ac-7f469bd10a14 Address:127.0.0.1:22286}]" +50:33.266 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6ac1981c-09fc-ca01-00ac-7f469bd10a14.dc1 127.0.0.1 +50:33.267 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6ac1981c-09fc-ca01-00ac-7f469bd10a14 127.0.0.1 +2022-12-04T05:50:33.267Z [INFO] agent.router: Initializing LAN area manager +50:33.267 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22281 network=udp +50:33.267 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22286 [Follower]" leader= +50:33.268 [INFO] TestAgent.server: Adding LAN server: server="Node-6ac1981c-09fc-ca01-00ac-7f469bd10a14 (Addr: tcp/127.0.0.1:22286) (DC: dc1)" +50:33.268 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6ac1981c-09fc-ca01-00ac-7f469bd10a14.dc1 area=wan +50:33.268 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22281 network=tcp +50:33.269 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22282 network=tcp +50:33.269 [INFO] TestAgent: started state syncer +50:33.269 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22287 network=tcp +50:33.321 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:33.321 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22286 [Candidate]" term=2 +50:33.321 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:33.321 [DEBUG] TestAgent.server.raft: vote granted: from=6ac1981c-09fc-ca01-00ac-7f469bd10a14 term=2 tally=1 +50:33.322 [INFO] TestAgent.server.raft: election won: tally=1 +50:33.322 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22286 [Leader]" +50:33.322 [INFO] TestAgent.server: cluster leadership acquired +50:33.322 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:22286 +50:33.324 [INFO] TestAgent.server: New leader elected: payload=Node-6ac1981c-09fc-ca01-00ac-7f469bd10a14 +50:33.349 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:33.351 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:33.351 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:33.351 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:33.351 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:33.351 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:33.351 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6ac1981c-09fc-ca01-00ac-7f469bd10a14 +50:33.351 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6ac1981c-09fc-ca01-00ac-7f469bd10a14 +50:33.352 [INFO] TestAgent.server: federation state anti-entropy synced +50:33.416 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:33.417 [INFO] TestAgent: Synced node info +50:33.709 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:40288 latency=902.142µs +50:33.711 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:40290 latency=313.666µs +50:33.711 [INFO] TestAgent: Requesting shutdown +50:33.711 [INFO] TestAgent.server: shutting down server +50:33.711 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:33.711 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:33.711 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:33.711 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:33.711 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:33.711 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:33.711 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:33.711 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:33.711 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:33.711 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:33.711 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:33.712Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:33.712Z [INFO] agent.router.manager: shutting down +50:33.712 [INFO] TestAgent: consul server down +50:33.712 [INFO] TestAgent: shutdown complete +50:33.712 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22281 network=tcp +50:33.712 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22281 network=udp +50:33.712 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22282 network=tcp +50:34.216 [INFO] TestAgent: Waiting for endpoints to shut down +50:34.216 [INFO] TestAgent: Endpoints down +--- PASS: TestKVGetCommand_Detailed (1.50s) PASS -ok github.com/hashicorp/consul/command/kv/get 2.041s +ok github.com/hashicorp/consul/command/kv/get 2.590s === RUN TestKVImportCommand_noTabs === PAUSE TestKVImportCommand_noTabs === RUN TestKVImportCommand @@ -87217,69 +87349,70 @@ [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:17:58.135Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:17:58.174Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -17:58.176 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bc9b45eb-92b6-0a85-0ea2-604acc8b5c1a Address:127.0.0.1:30426}]" -17:58.177 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bc9b45eb-92b6-0a85-0ea2-604acc8b5c1a.dc1 127.0.0.1 -17:58.177 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= -17:58.186 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bc9b45eb-92b6-0a85-0ea2-604acc8b5c1a 127.0.0.1 -2021-10-31T23:17:58.186Z [INFO] agent.router: Initializing LAN area manager -17:58.186 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp -17:58.186 [INFO] TestAgent.server: Adding LAN server: server="Node-bc9b45eb-92b6-0a85-0ea2-604acc8b5c1a (Addr: tcp/127.0.0.1:30426) (DC: dc1)" -17:58.186 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bc9b45eb-92b6-0a85-0ea2-604acc8b5c1a.dc1 area=wan -17:58.186 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp -17:58.187 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp -17:58.187 [INFO] TestAgent: started state syncer -17:58.187 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp -17:58.249 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -17:58.249 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 -17:58.250 [DEBUG] TestAgent.server.raft: votes: needed=1 -17:58.250 [DEBUG] TestAgent.server.raft: vote granted: from=bc9b45eb-92b6-0a85-0ea2-604acc8b5c1a term=2 tally=1 -17:58.250 [INFO] TestAgent.server.raft: election won: tally=1 -17:58.250 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" -17:58.250 [INFO] TestAgent.server: cluster leadership acquired -17:58.250 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30426 -17:58.251 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -17:58.251 [INFO] TestAgent.server: New leader elected: payload=Node-bc9b45eb-92b6-0a85-0ea2-604acc8b5c1a -17:58.276 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -17:58.276 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -17:58.276 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -17:58.276 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -17:58.276 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -17:58.276 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bc9b45eb-92b6-0a85-0ea2-604acc8b5c1a -17:58.276 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bc9b45eb-92b6-0a85-0ea2-604acc8b5c1a -17:58.277 [INFO] TestAgent.server: federation state anti-entropy synced -17:58.318 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -17:58.319 [INFO] TestAgent: Synced node info -17:58.458 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:45084 latency=1.517075ms -17:58.483 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:45084 latency=16.880468ms -17:58.491 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:45086 latency=533.37µs -17:58.492 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo/a from=127.0.0.1:45086 latency=131.281µs -17:58.492 [INFO] TestAgent: Requesting shutdown -17:58.492 [INFO] TestAgent.server: shutting down server -17:58.492 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -17:58.492 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -17:58.492 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -17:58.492 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -17:58.492 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -17:58.492 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -17:58.492 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -17:58.492 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -17:58.492 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -17:58.492 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -17:58.492 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:17:58.492Z [INFO] agent.router.manager: shutting down -2021-10-31T23:17:58.492Z [INFO] agent.router.manager: shutting down -17:58.493 [INFO] TestAgent: consul server down -17:58.493 [INFO] TestAgent: shutdown complete -17:58.493 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp -17:58.494 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp -17:58.494 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp -17:58.997 [INFO] TestAgent: Waiting for endpoints to shut down -17:58.997 [INFO] TestAgent: Endpoints down ---- PASS: TestKVImportCommand (0.91s) +2022-12-04T05:50:41.509Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:41.554Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:41.557 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:289cd684-bde9-853e-83d3-22921cac0c41 Address:127.0.0.1:17153}]" +50:41.557 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17153 [Follower]" leader= +50:41.558 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-289cd684-bde9-853e-83d3-22921cac0c41.dc1 127.0.0.1 +50:41.558 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-289cd684-bde9-853e-83d3-22921cac0c41 127.0.0.1 +2022-12-04T05:50:41.558Z [INFO] agent.router: Initializing LAN area manager +50:41.559 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17148 network=udp +50:41.559 [INFO] TestAgent.server: Adding LAN server: server="Node-289cd684-bde9-853e-83d3-22921cac0c41 (Addr: tcp/127.0.0.1:17153) (DC: dc1)" +50:41.559 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-289cd684-bde9-853e-83d3-22921cac0c41.dc1 area=wan +50:41.559 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17148 network=tcp +50:41.559 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17149 network=tcp +50:41.560 [INFO] TestAgent: started state syncer +50:41.560 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17154 network=tcp +50:41.600 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:41.600 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17153 [Candidate]" term=2 +50:41.600 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:41.600 [DEBUG] TestAgent.server.raft: vote granted: from=289cd684-bde9-853e-83d3-22921cac0c41 term=2 tally=1 +50:41.600 [INFO] TestAgent.server.raft: election won: tally=1 +50:41.600 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17153 [Leader]" +50:41.600 [INFO] TestAgent.server: cluster leadership acquired +50:41.601 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17153 +50:41.601 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:41.604 [INFO] TestAgent.server: New leader elected: payload=Node-289cd684-bde9-853e-83d3-22921cac0c41 +50:41.625 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:41.625 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:41.625 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:41.625 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:41.625 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:41.625 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-289cd684-bde9-853e-83d3-22921cac0c41 +50:41.625 [INFO] TestAgent.server: member joined, marking health alive: member=Node-289cd684-bde9-853e-83d3-22921cac0c41 +50:41.626 [INFO] TestAgent.server: federation state anti-entropy synced +50:41.701 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:36970 latency=8.7204ms +50:41.713 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo/a from=127.0.0.1:36970 latency=906.13µs +50:41.730 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:36972 latency=732.396µs +50:41.730 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:41.731 [INFO] TestAgent: Synced node info +50:41.731 [DEBUG] TestAgent: Node info in sync +50:41.732 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo/a from=127.0.0.1:36972 latency=702.553µs +50:41.733 [INFO] TestAgent: Requesting shutdown +50:41.733 [INFO] TestAgent.server: shutting down server +50:41.733 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:41.733 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:41.733 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:41.733 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:41.733 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:41.733 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:41.733 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:41.733 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:41.733 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:41.733 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:41.733 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:41.733Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:41.733Z [INFO] agent.router.manager: shutting down +50:41.733 [INFO] TestAgent: consul server down +50:41.733 [INFO] TestAgent: shutdown complete +50:41.733 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17148 network=tcp +50:41.733 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17148 network=udp +50:41.733 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17149 network=tcp +50:42.233 [INFO] TestAgent: Waiting for endpoints to shut down +50:42.233 [INFO] TestAgent: Endpoints down +--- PASS: TestKVImportCommand (0.83s) PASS -ok github.com/hashicorp/consul/command/kv/imp 1.196s +ok github.com/hashicorp/consul/command/kv/imp 0.950s ? github.com/hashicorp/consul/command/kv/impexp [no test files] === RUN TestKVPutCommand_noTabs === PAUSE TestKVPutCommand_noTabs @@ -87304,479 +87437,500 @@ === RUN TestKVPutCommand_CAS === PAUSE TestKVPutCommand_CAS === CONT TestKVPutCommand_noTabs -=== CONT TestKVPutCommand_Stdin +--- PASS: TestKVPutCommand_noTabs (0.00s) === CONT TestKVPutCommand_CAS [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestKVPutCommand_Flags -=== CONT TestKVPutCommand_NegativeVal ---- PASS: TestKVPutCommand_noTabs (0.01s) -=== CONT TestKVPutCommand_EmptyDataQuoted -=== CONT TestKVPutCommand_FileNoExist ---- PASS: TestKVPutCommand_FileNoExist (0.00s) === CONT TestKVPutCommand_File +=== CONT TestKVPutCommand_EmptyDataQuoted === CONT TestKVPutCommand_Base64 -=== CONT TestKVPutCommand_Validation === CONT TestKVPutCommand +=== CONT TestKVPutCommand_Validation +=== CONT TestKVPutCommand_NegativeVal [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range ---- PASS: TestKVPutCommand_Validation (0.02s) -2021-10-31T23:18:01.983Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:02.019Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:02.035 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5e2dea9b-81db-f41c-977a-38e87bbb5a3b Address:127.0.0.1:12069}]" -18:02.035 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5e2dea9b-81db-f41c-977a-38e87bbb5a3b.dc1 127.0.0.1 -18:02.036 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5e2dea9b-81db-f41c-977a-38e87bbb5a3b 127.0.0.1 -2021-10-31T23:18:02.036Z [INFO] agent.router: Initializing LAN area manager -18:02.036 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12064 network=udp -18:02.036 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12069 [Follower]" leader= -18:02.037 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12064 network=tcp -18:02.037 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5e2dea9b-81db-f41c-977a-38e87bbb5a3b.dc1 area=wan -18:02.037 [INFO] TestAgent.server: Adding LAN server: server="Node-5e2dea9b-81db-f41c-977a-38e87bbb5a3b (Addr: tcp/127.0.0.1:12069) (DC: dc1)" -18:02.037 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12065 network=tcp -18:02.037 [INFO] TestAgent: started state syncer -18:02.037 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12070 network=tcp -18:02.101 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:02.101 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12069 [Candidate]" term=2 -18:02.102 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:02.102 [DEBUG] TestAgent.server.raft: vote granted: from=5e2dea9b-81db-f41c-977a-38e87bbb5a3b term=2 tally=1 -18:02.102 [INFO] TestAgent.server.raft: election won: tally=1 -18:02.102 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12069 [Leader]" -18:02.102 [INFO] TestAgent.server: cluster leadership acquired -18:02.102 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12069 -18:02.102 [INFO] TestAgent.server: New leader elected: payload=Node-5e2dea9b-81db-f41c-977a-38e87bbb5a3b -18:02.103 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:02.110 [INFO] TestAgent: Synced node info -18:02.110 [DEBUG] TestAgent: Node info in sync -18:02.161 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:54318 latency=1.6073ms -18:02.168 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:54320 latency=391.532µs -18:02.169 [INFO] TestAgent: Requesting shutdown -18:02.169 [INFO] TestAgent.server: shutting down server -18:02.169 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:02.169 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:02.169Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:02.169Z [INFO] agent.router.manager: shutting down -18:02.169 [INFO] TestAgent: consul server down -18:02.169 [INFO] TestAgent: shutdown complete -18:02.169 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12064 network=tcp -18:02.169 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12064 network=udp -18:02.169 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12065 network=tcp -18:02.187 [ERROR] TestAgent.server: failed to establish leadership: error="error generating CA root certificate: error computing next serial number: raft is already shutdown" -18:02.670 [INFO] TestAgent: Waiting for endpoints to shut down -18:02.670 [INFO] TestAgent: Endpoints down ---- PASS: TestKVPutCommand_NegativeVal (0.82s) -2021-10-31T23:18:02.106Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:02.174Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:02.191 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:677cf22d-1779-7a4c-f4e1-a25cb590444a Address:127.0.0.1:12076}]" -18:02.191 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-677cf22d-1779-7a4c-f4e1-a25cb590444a.dc1 127.0.0.1 -18:02.192 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12076 [Follower]" leader= -18:02.192 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-677cf22d-1779-7a4c-f4e1-a25cb590444a 127.0.0.1 -2021-10-31T23:18:02.192Z [INFO] agent.router: Initializing LAN area manager -18:02.192 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12071 network=udp -18:02.193 [INFO] TestAgent.server: Adding LAN server: server="Node-677cf22d-1779-7a4c-f4e1-a25cb590444a (Addr: tcp/127.0.0.1:12076) (DC: dc1)" -18:02.193 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-677cf22d-1779-7a4c-f4e1-a25cb590444a.dc1 area=wan -18:02.193 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12071 network=tcp -18:02.195 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12072 network=tcp -18:02.195 [INFO] TestAgent: started state syncer -18:02.217 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12077 network=tcp -18:02.332 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:02.332 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12076 [Candidate]" term=2 -18:02.332 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:02.332 [DEBUG] TestAgent.server.raft: vote granted: from=677cf22d-1779-7a4c-f4e1-a25cb590444a term=2 tally=1 -18:02.332 [INFO] TestAgent.server.raft: election won: tally=1 -18:02.332 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12076 [Leader]" -18:02.332 [INFO] TestAgent.server: cluster leadership acquired -18:02.342 [INFO] TestAgent.server: New leader elected: payload=Node-677cf22d-1779-7a4c-f4e1-a25cb590444a -18:02.342 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12076 -18:02.343 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:02.382 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:02.382 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:02.382 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:02.382 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:02.382 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:02.382 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-677cf22d-1779-7a4c-f4e1-a25cb590444a -18:02.382 [INFO] TestAgent.server: member joined, marking health alive: member=Node-677cf22d-1779-7a4c-f4e1-a25cb590444a -18:02.383 [INFO] TestAgent.server: federation state anti-entropy synced -18:02.397 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:48040 latency=619.531µs -18:02.414 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:48044 latency=414.65µs -18:02.415 [INFO] TestAgent: Requesting shutdown -18:02.416 [INFO] TestAgent.server: shutting down server -18:02.416 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:02.416 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:02.416 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:02.416 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:02.416 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:02.416 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:02.416 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:02.416 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -18:02.416 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:02.416 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:02.416 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:02.416 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:02.416Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:02.416Z [INFO] agent.router.manager: shutting down -18:02.416 [INFO] TestAgent: consul server down -18:02.416 [INFO] TestAgent: shutdown complete -18:02.416 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12071 network=tcp -18:02.416 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12071 network=udp -18:02.416 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12072 network=tcp -18:02.917 [INFO] TestAgent: Waiting for endpoints to shut down -18:02.917 [INFO] TestAgent: Endpoints down ---- PASS: TestKVPutCommand_EmptyDataQuoted (1.07s) -2021-10-31T23:18:02.123Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:02.222Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:02.224 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a1860018-1398-7453-9cc1-a7cb9b4531ea Address:127.0.0.1:12090}]" -18:02.225 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a1860018-1398-7453-9cc1-a7cb9b4531ea.dc1 127.0.0.1 -18:02.225 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12090 [Follower]" leader= -18:02.245 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a1860018-1398-7453-9cc1-a7cb9b4531ea 127.0.0.1 -2021-10-31T23:18:02.245Z [INFO] agent.router: Initializing LAN area manager -18:02.246 [INFO] TestAgent.server: Adding LAN server: server="Node-a1860018-1398-7453-9cc1-a7cb9b4531ea (Addr: tcp/127.0.0.1:12090) (DC: dc1)" -18:02.246 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a1860018-1398-7453-9cc1-a7cb9b4531ea.dc1 area=wan -18:02.246 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12085 network=udp -18:02.246 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12085 network=tcp -18:02.250 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12086 network=tcp -18:02.326 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:02.326 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12090 [Candidate]" term=2 -18:02.326 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:02.326 [DEBUG] TestAgent.server.raft: vote granted: from=a1860018-1398-7453-9cc1-a7cb9b4531ea term=2 tally=1 -18:02.326 [INFO] TestAgent.server.raft: election won: tally=1 -18:02.326 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12090 [Leader]" -18:02.326 [INFO] TestAgent.server: cluster leadership acquired -18:02.326 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12090 -18:02.327 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:02.327 [INFO] TestAgent.server: New leader elected: payload=Node-a1860018-1398-7453-9cc1-a7cb9b4531ea -18:02.328 [INFO] TestAgent: started state syncer -18:02.329 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12091 network=tcp -18:02.350 [INFO] TestAgent: Synced node info -18:02.367 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:02.367 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:02.367 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:02.367 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:02.367 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:02.368 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a1860018-1398-7453-9cc1-a7cb9b4531ea -18:02.368 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a1860018-1398-7453-9cc1-a7cb9b4531ea -18:02.368 [INFO] TestAgent.server: federation state anti-entropy synced -18:02.381 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:53182 latency=49.437643ms -18:02.418 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:53186 latency=4.557848ms -18:02.426 [INFO] TestAgent: Requesting shutdown -18:02.426 [INFO] TestAgent.server: shutting down server -18:02.426 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:02.426 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:02.426 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:02.426 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:02.426 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:02.426 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:02.426 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:02.426 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:02.426 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:02.426 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:02.426 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:02.426Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:02.426Z [INFO] agent.router.manager: shutting down -18:02.426 [INFO] TestAgent: consul server down -18:02.426 [INFO] TestAgent: shutdown complete -18:02.426 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12085 network=tcp -18:02.426 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12085 network=udp -18:02.426 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12086 network=tcp -18:02.929 [INFO] TestAgent: Waiting for endpoints to shut down -18:02.929 [INFO] TestAgent: Endpoints down ---- PASS: TestKVPutCommand_Base64 (1.08s) -2021-10-31T23:18:02.067Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:02.229Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:02.246 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0ab8de14-11c6-887e-8926-4cb5ab4fe8d5 Address:127.0.0.1:12048}]" -18:02.246 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12048 [Follower]" leader= -18:02.246 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0ab8de14-11c6-887e-8926-4cb5ab4fe8d5.dc1 127.0.0.1 -18:02.247 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0ab8de14-11c6-887e-8926-4cb5ab4fe8d5 127.0.0.1 -2021-10-31T23:18:02.247Z [INFO] agent.router: Initializing LAN area manager -18:02.248 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=udp -18:02.248 [INFO] TestAgent.server: Adding LAN server: server="Node-0ab8de14-11c6-887e-8926-4cb5ab4fe8d5 (Addr: tcp/127.0.0.1:12048) (DC: dc1)" -18:02.248 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0ab8de14-11c6-887e-8926-4cb5ab4fe8d5.dc1 area=wan -18:02.253 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=tcp -18:02.328 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12044 network=tcp -18:02.328 [INFO] TestAgent: started state syncer -18:02.328 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12049 network=tcp -18:02.332 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:02.332 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12048 [Candidate]" term=2 -18:02.332 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:02.332 [DEBUG] TestAgent.server.raft: vote granted: from=0ab8de14-11c6-887e-8926-4cb5ab4fe8d5 term=2 tally=1 -18:02.332 [INFO] TestAgent.server.raft: election won: tally=1 -18:02.332 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12048 [Leader]" -18:02.332 [INFO] TestAgent.server: cluster leadership acquired -18:02.347 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12048 -18:02.347 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:02.348 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:02.348 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:02.348 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:02.348 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:02.348 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:02.349 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0ab8de14-11c6-887e-8926-4cb5ab4fe8d5 -18:02.349 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0ab8de14-11c6-887e-8926-4cb5ab4fe8d5 -18:02.349 [INFO] TestAgent.server: New leader elected: payload=Node-0ab8de14-11c6-887e-8926-4cb5ab4fe8d5 -18:02.366 [INFO] TestAgent.server: federation state anti-entropy synced -18:02.443 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:60692 latency=543.068µs -18:02.444 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:60694 latency=705.621µs -18:02.444 [INFO] TestAgent: Requesting shutdown -18:02.444 [INFO] TestAgent.server: shutting down server -18:02.444 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:02.444 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:02.444 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:02.444 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:02.444 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:02.444 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:02.444 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:02.444 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -18:02.444 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:02.444 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:02.444 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:02.444 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:02.444Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:02.444Z [INFO] agent.router.manager: shutting down -18:02.445 [INFO] TestAgent: consul server down -18:02.445 [INFO] TestAgent: shutdown complete -18:02.445 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=tcp -18:02.445 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=udp -18:02.445 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12044 network=tcp -18:02.946 [INFO] TestAgent: Waiting for endpoints to shut down -18:02.946 [INFO] TestAgent: Endpoints down ---- PASS: TestKVPutCommand_Stdin (1.10s) -2021-10-31T23:18:02.065Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:02.190Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:02.215 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:85303cbe-6ab1-bf33-dd34-699cbd653041 Address:127.0.0.1:12055}]" -18:02.216 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-85303cbe-6ab1-bf33-dd34-699cbd653041.dc1 127.0.0.1 -18:02.217 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-85303cbe-6ab1-bf33-dd34-699cbd653041 127.0.0.1 -2021-10-31T23:18:02.217Z [INFO] agent.router: Initializing LAN area manager -18:02.217 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=udp -18:02.217 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12055 [Follower]" leader= -18:02.218 [INFO] TestAgent.server: Adding LAN server: server="Node-85303cbe-6ab1-bf33-dd34-699cbd653041 (Addr: tcp/127.0.0.1:12055) (DC: dc1)" -18:02.218 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-85303cbe-6ab1-bf33-dd34-699cbd653041.dc1 area=wan -18:02.218 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12050 network=tcp -18:02.219 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12051 network=tcp -18:02.219 [INFO] TestAgent: started state syncer -18:02.219 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12056 network=tcp -18:02.332 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:02.332 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12055 [Candidate]" term=2 -18:02.332 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:02.332 [DEBUG] TestAgent.server.raft: vote granted: from=85303cbe-6ab1-bf33-dd34-699cbd653041 term=2 tally=1 -18:02.332 [INFO] TestAgent.server.raft: election won: tally=1 -18:02.332 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12055 [Leader]" -18:02.332 [INFO] TestAgent.server: cluster leadership acquired -18:02.346 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12055 -18:02.347 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:02.348 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:02.348 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:02.348 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:02.348 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:02.348 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:02.348 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-85303cbe-6ab1-bf33-dd34-699cbd653041 -18:02.348 [INFO] TestAgent.server: member joined, marking health alive: member=Node-85303cbe-6ab1-bf33-dd34-699cbd653041 -18:02.349 [INFO] TestAgent.server: New leader elected: payload=Node-85303cbe-6ab1-bf33-dd34-699cbd653041 -18:02.350 [INFO] TestAgent.server: federation state anti-entropy synced -18:02.386 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:02.389 [INFO] TestAgent: Synced node info -18:02.542 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:44468 latency=665.209µs -18:02.550 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo?cas=123 from=127.0.0.1:44470 latency=442.634µs -18:02.551 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:44468 latency=724.345µs -18:02.553 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo?cas=13 from=127.0.0.1:44472 latency=434.23µs -18:02.553 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:44468 latency=124.391µs -18:02.553 [INFO] TestAgent: Requesting shutdown -18:02.553 [INFO] TestAgent.server: shutting down server -18:02.553 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:02.553 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:02.553 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:02.553 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:02.553 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:02.553 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:02.553 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:02.553 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -18:02.553 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:02.553 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:02.553 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:18:02.554Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:02.554Z [INFO] agent.router.manager: shutting down -18:02.554 [INFO] TestAgent: consul server down -18:02.554 [INFO] TestAgent: shutdown complete -18:02.554 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=tcp -18:02.554 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12050 network=udp -18:02.554 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12051 network=tcp -18:03.057 [INFO] TestAgent: Waiting for endpoints to shut down -18:03.057 [INFO] TestAgent: Endpoints down ---- PASS: TestKVPutCommand_CAS (1.21s) -2021-10-31T23:18:02.187Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:02.334Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:02.344 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e0191a85-b9c3-e9e6-fdd4-b509a3121764 Address:127.0.0.1:12097}]" -18:02.344 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e0191a85-b9c3-e9e6-fdd4-b509a3121764.dc1 127.0.0.1 -18:02.345 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e0191a85-b9c3-e9e6-fdd4-b509a3121764 127.0.0.1 -2021-10-31T23:18:02.345Z [INFO] agent.router: Initializing LAN area manager -18:02.345 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12092 network=udp -18:02.347 [INFO] TestAgent.server: Adding LAN server: server="Node-e0191a85-b9c3-e9e6-fdd4-b509a3121764 (Addr: tcp/127.0.0.1:12097) (DC: dc1)" -18:02.348 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12092 network=tcp -18:02.348 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12093 network=tcp -18:02.348 [INFO] TestAgent: started state syncer -18:02.349 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e0191a85-b9c3-e9e6-fdd4-b509a3121764.dc1 area=wan -18:02.349 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12098 network=tcp -18:02.349 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12097 [Follower]" leader= -18:02.397 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:02.397 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12097 [Candidate]" term=2 -18:02.411 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:02.411 [DEBUG] TestAgent.server.raft: vote granted: from=e0191a85-b9c3-e9e6-fdd4-b509a3121764 term=2 tally=1 -18:02.411 [INFO] TestAgent.server.raft: election won: tally=1 -18:02.411 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12097 [Leader]" -18:02.411 [INFO] TestAgent.server: cluster leadership acquired -18:02.411 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12097 -18:02.411 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:02.413 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:02.413 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:02.413 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:02.413 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:02.413 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:02.413 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e0191a85-b9c3-e9e6-fdd4-b509a3121764 -18:02.413 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e0191a85-b9c3-e9e6-fdd4-b509a3121764 -18:02.414 [INFO] TestAgent.server: New leader elected: payload=Node-e0191a85-b9c3-e9e6-fdd4-b509a3121764 -18:02.418 [INFO] TestAgent.server: federation state anti-entropy synced -18:02.633 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:40138 latency=532.479µs -18:02.642 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:40140 latency=338.564µs -18:02.644 [INFO] TestAgent: Requesting shutdown -18:02.644 [INFO] TestAgent.server: shutting down server -18:02.644 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:02.644 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:02.644 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:02.644 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:02.644 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:02.644 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:02.644 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:02.644 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -18:02.644 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:02.644 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:02.644 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:02.644 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:02.644Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:02.644Z [INFO] agent.router.manager: shutting down -18:02.644 [INFO] TestAgent: consul server down -18:02.644 [INFO] TestAgent: shutdown complete -18:02.644 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12092 network=tcp -18:02.644 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12092 network=udp -18:02.645 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12093 network=tcp -18:03.146 [INFO] TestAgent: Waiting for endpoints to shut down -18:03.146 [INFO] TestAgent: Endpoints down ---- PASS: TestKVPutCommand (1.30s) -2021-10-31T23:18:02.126Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:02.197Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:02.226 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9df2912a-bf97-d614-f35c-9f1a96b1627f Address:127.0.0.1:12062}]" -18:02.227 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9df2912a-bf97-d614-f35c-9f1a96b1627f.dc1 127.0.0.1 -18:02.227 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9df2912a-bf97-d614-f35c-9f1a96b1627f 127.0.0.1 -2021-10-31T23:18:02.227Z [INFO] agent.router: Initializing LAN area manager -18:02.228 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12057 network=udp -18:02.228 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12062 [Follower]" leader= -18:02.228 [INFO] TestAgent.server: Adding LAN server: server="Node-9df2912a-bf97-d614-f35c-9f1a96b1627f (Addr: tcp/127.0.0.1:12062) (DC: dc1)" -18:02.228 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9df2912a-bf97-d614-f35c-9f1a96b1627f.dc1 area=wan -18:02.228 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12057 network=tcp -18:02.229 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12058 network=tcp -18:02.229 [INFO] TestAgent: started state syncer -18:02.229 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12063 network=tcp -18:02.326 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:02.326 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12062 [Candidate]" term=2 -18:02.326 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:02.326 [DEBUG] TestAgent.server.raft: vote granted: from=9df2912a-bf97-d614-f35c-9f1a96b1627f term=2 tally=1 -18:02.326 [INFO] TestAgent.server.raft: election won: tally=1 -18:02.326 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12062 [Leader]" -18:02.326 [INFO] TestAgent.server: cluster leadership acquired -18:02.326 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12062 -18:02.327 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:02.329 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:02.329 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:02.329 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:02.329 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:02.329 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:02.329 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9df2912a-bf97-d614-f35c-9f1a96b1627f -18:02.329 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9df2912a-bf97-d614-f35c-9f1a96b1627f -18:02.332 [INFO] TestAgent.server: New leader elected: payload=Node-9df2912a-bf97-d614-f35c-9f1a96b1627f -18:02.350 [INFO] TestAgent.server: federation state anti-entropy synced -18:02.557 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:02.558 [INFO] TestAgent: Synced node info -18:02.645 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo?flags=12345 from=127.0.0.1:48516 latency=3.076588ms -18:02.650 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:48518 latency=772.396µs -18:02.651 [INFO] TestAgent: Requesting shutdown -18:02.651 [INFO] TestAgent.server: shutting down server -18:02.651 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:02.651 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:02.651 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:02.651 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:02.651 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:02.651 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:02.651 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:02.651 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:02.651 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:02.651 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:02.651 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:02.651Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:02.651Z [INFO] agent.router.manager: shutting down -18:02.651 [INFO] TestAgent: consul server down -18:02.651 [INFO] TestAgent: shutdown complete -18:02.651 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12057 network=tcp -18:02.651 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12057 network=udp -18:02.651 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12058 network=tcp -18:03.153 [INFO] TestAgent: Waiting for endpoints to shut down -18:03.153 [INFO] TestAgent: Endpoints down ---- PASS: TestKVPutCommand_Flags (1.31s) -2021-10-31T23:18:02.101Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:02.330Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:02.346 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0de3a854-9be1-543f-9df3-b3f887228d60 Address:127.0.0.1:12083}]" -18:02.346 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0de3a854-9be1-543f-9df3-b3f887228d60.dc1 127.0.0.1 -18:02.347 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0de3a854-9be1-543f-9df3-b3f887228d60 127.0.0.1 -2021-10-31T23:18:02.347Z [INFO] agent.router: Initializing LAN area manager -18:02.347 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12078 network=udp -18:02.348 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12083 [Follower]" leader= -18:02.349 [INFO] TestAgent.server: Adding LAN server: server="Node-0de3a854-9be1-543f-9df3-b3f887228d60 (Addr: tcp/127.0.0.1:12083) (DC: dc1)" -18:02.349 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0de3a854-9be1-543f-9df3-b3f887228d60.dc1 area=wan -18:02.349 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12078 network=tcp -18:02.349 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12079 network=tcp -18:02.350 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12084 network=tcp -18:02.350 [INFO] TestAgent: started state syncer -18:02.393 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:02.394 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12083 [Candidate]" term=2 -18:02.394 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:02.394 [DEBUG] TestAgent.server.raft: vote granted: from=0de3a854-9be1-543f-9df3-b3f887228d60 term=2 tally=1 -18:02.394 [INFO] TestAgent.server.raft: election won: tally=1 -18:02.394 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12083 [Leader]" -18:02.394 [INFO] TestAgent.server: cluster leadership acquired -18:02.394 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12083 -18:02.395 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:02.397 [INFO] TestAgent.server: New leader elected: payload=Node-0de3a854-9be1-543f-9df3-b3f887228d60 -18:02.397 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:02.397 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:02.397 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:02.397 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:02.397 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:02.397 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0de3a854-9be1-543f-9df3-b3f887228d60 -18:02.397 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0de3a854-9be1-543f-9df3-b3f887228d60 -18:02.410 [INFO] TestAgent.server: federation state anti-entropy synced -18:02.752 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:02.753 [INFO] TestAgent: Synced node info -18:02.753 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:39678 latency=893.382µs -18:02.759 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:39680 latency=272.982µs -18:02.760 [INFO] TestAgent: Requesting shutdown -18:02.760 [INFO] TestAgent.server: shutting down server -18:02.760 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:02.760 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:02.760 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:02.760 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:02.760 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:02.760 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:02.760 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:02.760 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:02.760 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:02.760 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:02.760 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:02.760Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:02.760Z [INFO] agent.router.manager: shutting down -18:02.760 [INFO] TestAgent: consul server down -18:02.760 [INFO] TestAgent: shutdown complete -18:02.760 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12078 network=tcp -18:02.760 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12078 network=udp -18:02.760 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12079 network=tcp -18:03.261 [INFO] TestAgent: Waiting for endpoints to shut down -18:03.261 [INFO] TestAgent: Endpoints down ---- PASS: TestKVPutCommand_File (1.41s) +=== CONT TestKVPutCommand_Flags +=== CONT TestKVPutCommand_Stdin +=== CONT TestKVPutCommand_FileNoExist +--- PASS: TestKVPutCommand_Validation (0.04s) +--- PASS: TestKVPutCommand_FileNoExist (0.02s) +2022-12-04T05:50:41.660Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:41.726Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:41.728 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cc63f373-eb91-af96-f349-30e239556237 Address:127.0.0.1:24307}]" +50:41.728 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24307 [Follower]" leader= +50:41.745 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cc63f373-eb91-af96-f349-30e239556237.dc1 127.0.0.1 +50:41.748 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cc63f373-eb91-af96-f349-30e239556237 127.0.0.1 +2022-12-04T05:50:41.748Z [INFO] agent.router: Initializing LAN area manager +50:41.749 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cc63f373-eb91-af96-f349-30e239556237.dc1 area=wan +50:41.749 [INFO] TestAgent.server: Adding LAN server: server="Node-cc63f373-eb91-af96-f349-30e239556237 (Addr: tcp/127.0.0.1:24307) (DC: dc1)" +50:41.749 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=udp +50:41.749 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24302 network=tcp +50:41.751 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24303 network=tcp +50:41.751 [INFO] TestAgent: started state syncer +50:41.772 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24308 network=tcp +50:41.782 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:41.782 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24307 [Candidate]" term=2 +50:41.783 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:41.783 [DEBUG] TestAgent.server.raft: vote granted: from=cc63f373-eb91-af96-f349-30e239556237 term=2 tally=1 +50:41.783 [INFO] TestAgent.server.raft: election won: tally=1 +50:41.783 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24307 [Leader]" +50:41.783 [INFO] TestAgent.server: cluster leadership acquired +50:41.783 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24307 +50:41.784 [INFO] TestAgent.server: New leader elected: payload=Node-cc63f373-eb91-af96-f349-30e239556237 +50:41.788 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:41.802 [INFO] TestAgent: Synced node info +50:41.802 [DEBUG] TestAgent: Node info in sync +50:41.819 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:41.819 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:41.819 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:41.819 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:41.819 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:41.819 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cc63f373-eb91-af96-f349-30e239556237 +50:41.819 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cc63f373-eb91-af96-f349-30e239556237 +50:41.820 [INFO] TestAgent.server: federation state anti-entropy synced +50:41.849 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:41.849 [DEBUG] TestAgent: Node info in sync +50:41.895 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:49736 latency=8.619382ms +50:41.902 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:49738 latency=2.030493ms +50:41.903 [INFO] TestAgent: Requesting shutdown +50:41.903 [INFO] TestAgent.server: shutting down server +50:41.903 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:41.903 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:41.903 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:41.903 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:41.903 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:41.903 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:41.903 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:41.903 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:41.903 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:41.903 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:41.903 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:41.903Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:41.903Z [INFO] agent.router.manager: shutting down +50:41.904 [INFO] TestAgent: consul server down +50:41.904 [INFO] TestAgent: shutdown complete +50:41.904 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=tcp +50:41.904 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24302 network=udp +50:41.904 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24303 network=tcp +50:42.404 [INFO] TestAgent: Waiting for endpoints to shut down +50:42.404 [INFO] TestAgent: Endpoints down +--- PASS: TestKVPutCommand_File (0.90s) +2022-12-04T05:50:41.709Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:41.729Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:41.787 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:250e91b4-0b8e-8bca-5cbc-854299e05e05 Address:127.0.0.1:24314}]" +50:41.788 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24314 [Follower]" leader= +50:41.788 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-250e91b4-0b8e-8bca-5cbc-854299e05e05.dc1 127.0.0.1 +50:41.789 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-250e91b4-0b8e-8bca-5cbc-854299e05e05 127.0.0.1 +2022-12-04T05:50:41.789Z [INFO] agent.router: Initializing LAN area manager +50:41.803 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24309 network=udp +50:41.803 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24309 network=tcp +50:41.805 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24310 network=tcp +50:41.805 [INFO] TestAgent: started state syncer +50:41.805 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24315 network=tcp +50:41.809 [INFO] TestAgent.server: Adding LAN server: server="Node-250e91b4-0b8e-8bca-5cbc-854299e05e05 (Addr: tcp/127.0.0.1:24314) (DC: dc1)" +50:41.809 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-250e91b4-0b8e-8bca-5cbc-854299e05e05.dc1 area=wan +50:41.856 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:41.856 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24314 [Candidate]" term=2 +50:41.860 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:41.860 [DEBUG] TestAgent.server.raft: vote granted: from=250e91b4-0b8e-8bca-5cbc-854299e05e05 term=2 tally=1 +50:41.860 [INFO] TestAgent.server.raft: election won: tally=1 +50:41.860 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24314 [Leader]" +50:41.860 [INFO] TestAgent.server: cluster leadership acquired +50:41.860 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24314 +50:41.860 [INFO] TestAgent.server: New leader elected: payload=Node-250e91b4-0b8e-8bca-5cbc-854299e05e05 +50:41.873 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:41.893 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:41.893 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:41.893 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:41.893 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:41.893 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:41.893 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-250e91b4-0b8e-8bca-5cbc-854299e05e05 +50:41.893 [INFO] TestAgent.server: member joined, marking health alive: member=Node-250e91b4-0b8e-8bca-5cbc-854299e05e05 +50:41.894 [INFO] TestAgent.server: federation state anti-entropy synced +50:41.905 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:37414 latency=644.262µs +50:41.906 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:37418 latency=215.358µs +50:41.906 [INFO] TestAgent: Requesting shutdown +50:41.906 [INFO] TestAgent.server: shutting down server +50:41.906 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:41.906 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:41.906 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:41.906 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:41.906 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:41.906 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:41.906 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:41.906 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:41.906 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:41.906 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:41.906 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:41.907 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:41.907Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:41.907Z [INFO] agent.router.manager: shutting down +50:41.907 [INFO] TestAgent: consul server down +50:41.907 [INFO] TestAgent: shutdown complete +50:41.907 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24309 network=tcp +50:41.907 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24309 network=udp +50:41.907 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24310 network=tcp +50:42.407 [INFO] TestAgent: Waiting for endpoints to shut down +50:42.407 [INFO] TestAgent: Endpoints down +--- PASS: TestKVPutCommand_EmptyDataQuoted (0.90s) +2022-12-04T05:50:41.722Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:41.809Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:41.858 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3b998a0f-3a9d-f421-90d8-ae1adeeca649 Address:127.0.0.1:24342}]" +50:41.858 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24342 [Follower]" leader= +50:41.859 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3b998a0f-3a9d-f421-90d8-ae1adeeca649.dc1 127.0.0.1 +50:41.859 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3b998a0f-3a9d-f421-90d8-ae1adeeca649 127.0.0.1 +2022-12-04T05:50:41.859Z [INFO] agent.router: Initializing LAN area manager +50:41.859 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24337 network=udp +50:41.860 [INFO] TestAgent.server: Adding LAN server: server="Node-3b998a0f-3a9d-f421-90d8-ae1adeeca649 (Addr: tcp/127.0.0.1:24342) (DC: dc1)" +50:41.860 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3b998a0f-3a9d-f421-90d8-ae1adeeca649.dc1 area=wan +50:41.860 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24337 network=tcp +50:41.861 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24338 network=tcp +50:41.861 [INFO] TestAgent: started state syncer +50:41.861 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24343 network=tcp +50:41.909 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:41.909 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24342 [Candidate]" term=2 +50:41.910 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:41.910 [DEBUG] TestAgent.server.raft: vote granted: from=3b998a0f-3a9d-f421-90d8-ae1adeeca649 term=2 tally=1 +50:41.910 [INFO] TestAgent.server.raft: election won: tally=1 +50:41.910 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24342 [Leader]" +50:41.910 [INFO] TestAgent.server: cluster leadership acquired +50:41.910 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24342 +50:41.911 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:41.912 [INFO] TestAgent.server: New leader elected: payload=Node-3b998a0f-3a9d-f421-90d8-ae1adeeca649 +50:41.912 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:41.912 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:41.912 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:41.912 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:41.912 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:41.912 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3b998a0f-3a9d-f421-90d8-ae1adeeca649 +50:41.912 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3b998a0f-3a9d-f421-90d8-ae1adeeca649 +50:41.913 [INFO] TestAgent.server: federation state anti-entropy synced +50:41.919 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo?flags=12345 from=127.0.0.1:42646 latency=459.637µs +50:41.919 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:42648 latency=162.593µs +50:41.920 [INFO] TestAgent: Requesting shutdown +50:41.920 [INFO] TestAgent.server: shutting down server +50:41.920 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:41.920 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:41.920 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:41.920 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:41.920 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:41.920 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:41.920 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:41.920 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:41.920 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:41.920 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +50:41.920 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:41.920 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:50:41.920Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:41.920Z [INFO] agent.router.manager: shutting down +50:41.920 [INFO] TestAgent: consul server down +50:41.920 [INFO] TestAgent: shutdown complete +50:41.920 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24337 network=tcp +50:41.920 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24337 network=udp +50:41.920 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24338 network=tcp +50:42.420 [INFO] TestAgent: Waiting for endpoints to shut down +50:42.420 [INFO] TestAgent: Endpoints down +--- PASS: TestKVPutCommand_Flags (0.89s) +2022-12-04T05:50:41.639Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:41.674Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:41.692 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dd07f41a-fd3d-3cb7-273a-366bb8ba68e6 Address:127.0.0.1:24300}]" +50:41.693 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dd07f41a-fd3d-3cb7-273a-366bb8ba68e6.dc1 127.0.0.1 +50:41.694 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dd07f41a-fd3d-3cb7-273a-366bb8ba68e6 127.0.0.1 +2022-12-04T05:50:41.694Z [INFO] agent.router: Initializing LAN area manager +50:41.695 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp +50:41.695 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dd07f41a-fd3d-3cb7-273a-366bb8ba68e6.dc1 area=wan +50:41.695 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp +50:41.695 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= +50:41.706 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp +50:41.706 [INFO] TestAgent: started state syncer +50:41.707 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp +50:41.707 [INFO] TestAgent.server: Adding LAN server: server="Node-dd07f41a-fd3d-3cb7-273a-366bb8ba68e6 (Addr: tcp/127.0.0.1:24300) (DC: dc1)" +50:41.772 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:41.772 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 +50:41.773 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:41.773 [DEBUG] TestAgent.server.raft: vote granted: from=dd07f41a-fd3d-3cb7-273a-366bb8ba68e6 term=2 tally=1 +50:41.773 [INFO] TestAgent.server.raft: election won: tally=1 +50:41.773 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" +50:41.773 [INFO] TestAgent.server: cluster leadership acquired +50:41.773 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24300 +50:41.773 [INFO] TestAgent.server: New leader elected: payload=Node-dd07f41a-fd3d-3cb7-273a-366bb8ba68e6 +50:41.775 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:41.814 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:41.814 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:41.814 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:41.814 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:41.814 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:41.814 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dd07f41a-fd3d-3cb7-273a-366bb8ba68e6 +50:41.814 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dd07f41a-fd3d-3cb7-273a-366bb8ba68e6 +50:41.815 [INFO] TestAgent.server: federation state anti-entropy synced +50:41.897 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:41.898 [INFO] TestAgent: Synced node info +50:41.898 [DEBUG] TestAgent: Node info in sync +50:41.915 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:32884 latency=534.015µs +50:41.918 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo?cas=123 from=127.0.0.1:32888 latency=632.349µs +50:41.918 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:32884 latency=356.68µs +50:41.920 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo?cas=13 from=127.0.0.1:32890 latency=297.487µs +50:41.920 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:32884 latency=132.504µs +50:41.923 [INFO] TestAgent: Requesting shutdown +50:41.923 [INFO] TestAgent.server: shutting down server +50:41.923 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:41.923 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:41.923 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:41.923 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:41.923 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:41.923 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:41.923 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:41.923 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:41.923 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:41.923 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:41.923 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:41.923Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:41.923Z [INFO] agent.router.manager: shutting down +50:41.923 [INFO] TestAgent: consul server down +50:41.923 [INFO] TestAgent: shutdown complete +50:41.923 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp +50:41.923 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp +50:41.923 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp +50:42.424 [INFO] TestAgent: Waiting for endpoints to shut down +50:42.424 [INFO] TestAgent: Endpoints down +--- PASS: TestKVPutCommand_CAS (0.92s) +2022-12-04T05:50:41.694Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:41.855Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:41.862 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:499a5aee-b796-d2b7-1db2-7a5ed604d3ba Address:127.0.0.1:24335}]" +50:41.863 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24335 [Follower]" leader= +50:41.875 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-499a5aee-b796-d2b7-1db2-7a5ed604d3ba.dc1 127.0.0.1 +50:41.894 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-499a5aee-b796-d2b7-1db2-7a5ed604d3ba 127.0.0.1 +2022-12-04T05:50:41.894Z [INFO] agent.router: Initializing LAN area manager +50:41.895 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24330 network=udp +50:41.895 [INFO] TestAgent.server: Adding LAN server: server="Node-499a5aee-b796-d2b7-1db2-7a5ed604d3ba (Addr: tcp/127.0.0.1:24335) (DC: dc1)" +50:41.895 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-499a5aee-b796-d2b7-1db2-7a5ed604d3ba.dc1 area=wan +50:41.895 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24330 network=tcp +50:41.897 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24331 network=tcp +50:41.897 [INFO] TestAgent: started state syncer +50:41.897 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24336 network=tcp +50:41.930 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:41.930 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24335 [Candidate]" term=2 +50:41.931 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:41.931 [DEBUG] TestAgent.server.raft: vote granted: from=499a5aee-b796-d2b7-1db2-7a5ed604d3ba term=2 tally=1 +50:41.931 [INFO] TestAgent.server.raft: election won: tally=1 +50:41.931 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24335 [Leader]" +50:41.931 [INFO] TestAgent.server: cluster leadership acquired +50:41.931 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24335 +50:41.931 [INFO] TestAgent.server: New leader elected: payload=Node-499a5aee-b796-d2b7-1db2-7a5ed604d3ba +50:41.932 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:41.933 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:41.933 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:41.933 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:41.933 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:41.933 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:41.933 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-499a5aee-b796-d2b7-1db2-7a5ed604d3ba +50:41.933 [INFO] TestAgent.server: member joined, marking health alive: member=Node-499a5aee-b796-d2b7-1db2-7a5ed604d3ba +50:41.934 [INFO] TestAgent.server: federation state anti-entropy synced +50:41.972 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:35072 latency=354.653µs +50:41.973 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:35074 latency=135.606µs +50:41.973 [INFO] TestAgent: Requesting shutdown +50:41.973 [INFO] TestAgent.server: shutting down server +50:41.973 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:41.973 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:41.973 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:41.973 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:41.973 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:41.973 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:41.973 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:41.973 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:41.973 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:41.973 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:41.973 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:41.973 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:41.974Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:41.974Z [INFO] agent.router.manager: shutting down +50:41.974 [INFO] TestAgent: consul server down +50:41.974 [INFO] TestAgent: shutdown complete +50:41.974 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24330 network=tcp +50:41.974 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24330 network=udp +50:41.974 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24331 network=tcp +50:42.474 [INFO] TestAgent: Waiting for endpoints to shut down +50:42.474 [INFO] TestAgent: Endpoints down +--- PASS: TestKVPutCommand_NegativeVal (0.94s) +2022-12-04T05:50:41.665Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:41.744Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:41.748 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9de9b0b9-a873-f394-bf2f-d60cde85e1f6 Address:127.0.0.1:24328}]" +50:41.749 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24328 [Follower]" leader= +50:41.750 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9de9b0b9-a873-f394-bf2f-d60cde85e1f6.dc1 127.0.0.1 +50:41.780 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9de9b0b9-a873-f394-bf2f-d60cde85e1f6 127.0.0.1 +2022-12-04T05:50:41.780Z [INFO] agent.router: Initializing LAN area manager +50:41.781 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24323 network=udp +50:41.781 [INFO] TestAgent.server: Adding LAN server: server="Node-9de9b0b9-a873-f394-bf2f-d60cde85e1f6 (Addr: tcp/127.0.0.1:24328) (DC: dc1)" +50:41.781 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9de9b0b9-a873-f394-bf2f-d60cde85e1f6.dc1 area=wan +50:41.781 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24323 network=tcp +50:41.782 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24324 network=tcp +50:41.782 [INFO] TestAgent: started state syncer +50:41.782 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24329 network=tcp +50:41.788 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:41.788 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24328 [Candidate]" term=2 +50:41.789 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:41.789 [DEBUG] TestAgent.server.raft: vote granted: from=9de9b0b9-a873-f394-bf2f-d60cde85e1f6 term=2 tally=1 +50:41.789 [INFO] TestAgent.server.raft: election won: tally=1 +50:41.789 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24328 [Leader]" +50:41.789 [INFO] TestAgent.server: cluster leadership acquired +50:41.789 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24328 +50:41.802 [INFO] TestAgent.server: New leader elected: payload=Node-9de9b0b9-a873-f394-bf2f-d60cde85e1f6 +50:41.803 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:41.819 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:41.819 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:41.819 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:41.819 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:41.819 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:41.819 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9de9b0b9-a873-f394-bf2f-d60cde85e1f6 +50:41.819 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9de9b0b9-a873-f394-bf2f-d60cde85e1f6 +50:41.820 [INFO] TestAgent.server: federation state anti-entropy synced +50:42.145 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:48366 latency=1.84838ms +50:42.150 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:48368 latency=279.918µs +50:42.150 [INFO] TestAgent: Requesting shutdown +50:42.150 [INFO] TestAgent.server: shutting down server +50:42.150 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:42.150 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:42.150 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:42.150 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:42.150 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:42.151 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:42.151 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:42.151 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:42.151 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:42.151 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:42.151 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:42.151 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:42.151Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:42.151Z [INFO] agent.router.manager: shutting down +50:42.151 [INFO] TestAgent: consul server down +50:42.151 [INFO] TestAgent: shutdown complete +50:42.151 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24323 network=tcp +50:42.151 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24323 network=udp +50:42.151 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24324 network=tcp +50:42.651 [INFO] TestAgent: Waiting for endpoints to shut down +50:42.651 [INFO] TestAgent: Endpoints down +--- PASS: TestKVPutCommand (1.15s) +2022-12-04T05:50:41.749Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:41.788Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:41.806 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:4f3b22f3-3bce-d681-90cf-2d10588c7dd2 Address:127.0.0.1:24349}]" +50:41.807 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-4f3b22f3-3bce-d681-90cf-2d10588c7dd2.dc1 127.0.0.1 +50:41.807 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-4f3b22f3-3bce-d681-90cf-2d10588c7dd2 127.0.0.1 +2022-12-04T05:50:41.807Z [INFO] agent.router: Initializing LAN area manager +50:41.808 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24349 [Follower]" leader= +50:41.816 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-4f3b22f3-3bce-d681-90cf-2d10588c7dd2.dc1 area=wan +50:41.817 [INFO] TestAgent.server: Adding LAN server: server="Node-4f3b22f3-3bce-d681-90cf-2d10588c7dd2 (Addr: tcp/127.0.0.1:24349) (DC: dc1)" +50:41.852 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24344 network=udp +50:41.853 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24344 network=tcp +50:41.853 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24345 network=tcp +50:41.853 [INFO] TestAgent: started state syncer +50:41.853 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24350 network=tcp +50:41.858 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:41.858 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24349 [Candidate]" term=2 +50:41.859 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:41.859 [DEBUG] TestAgent.server.raft: vote granted: from=4f3b22f3-3bce-d681-90cf-2d10588c7dd2 term=2 tally=1 +50:41.859 [INFO] TestAgent.server.raft: election won: tally=1 +50:41.859 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24349 [Leader]" +50:41.859 [INFO] TestAgent.server: cluster leadership acquired +50:41.859 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24349 +50:41.860 [INFO] TestAgent.server: New leader elected: payload=Node-4f3b22f3-3bce-d681-90cf-2d10588c7dd2 +50:41.875 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:41.895 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:41.895 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:41.895 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:41.895 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:41.895 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:41.895 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-4f3b22f3-3bce-d681-90cf-2d10588c7dd2 +50:41.895 [INFO] TestAgent.server: member joined, marking health alive: member=Node-4f3b22f3-3bce-d681-90cf-2d10588c7dd2 +50:41.897 [INFO] TestAgent.server: federation state anti-entropy synced +50:42.166 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:49292 latency=442.849µs +50:42.168 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:49294 latency=700.305µs +50:42.168 [INFO] TestAgent: Requesting shutdown +50:42.168 [INFO] TestAgent.server: shutting down server +50:42.168 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:42.168 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:42.168 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:42.168 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:42.168 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:42.168 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:42.168 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:42.168 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:42.168 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:42.168 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:42.168 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:42.168 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:42.168Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:42.169Z [INFO] agent.router.manager: shutting down +50:42.169 [INFO] TestAgent: consul server down +50:42.169 [INFO] TestAgent: shutdown complete +50:42.169 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24344 network=tcp +50:42.169 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24344 network=udp +50:42.169 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24345 network=tcp +50:42.669 [INFO] TestAgent: Waiting for endpoints to shut down +50:42.669 [INFO] TestAgent: Endpoints down +--- PASS: TestKVPutCommand_Stdin (1.14s) +2022-12-04T05:50:41.689Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:41.814Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:41.853 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:09246b4e-ad0f-5b5c-2587-4cb19446524f Address:127.0.0.1:24321}]" +50:41.854 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24321 [Follower]" leader= +50:41.855 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-09246b4e-ad0f-5b5c-2587-4cb19446524f.dc1 127.0.0.1 +50:41.855 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-09246b4e-ad0f-5b5c-2587-4cb19446524f 127.0.0.1 +2022-12-04T05:50:41.855Z [INFO] agent.router: Initializing LAN area manager +50:41.856 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-09246b4e-ad0f-5b5c-2587-4cb19446524f.dc1 area=wan +50:41.856 [INFO] TestAgent.server: Adding LAN server: server="Node-09246b4e-ad0f-5b5c-2587-4cb19446524f (Addr: tcp/127.0.0.1:24321) (DC: dc1)" +50:41.857 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24316 network=tcp +50:41.857 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24316 network=udp +50:41.886 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24317 network=tcp +50:41.887 [INFO] TestAgent: started state syncer +50:41.887 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24322 network=tcp +50:41.902 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:41.902 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24321 [Candidate]" term=2 +50:41.902 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:41.902 [DEBUG] TestAgent.server.raft: vote granted: from=09246b4e-ad0f-5b5c-2587-4cb19446524f term=2 tally=1 +50:41.902 [INFO] TestAgent.server.raft: election won: tally=1 +50:41.902 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24321 [Leader]" +50:41.902 [INFO] TestAgent.server: cluster leadership acquired +50:41.903 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24321 +50:41.903 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:41.904 [INFO] TestAgent.server: New leader elected: payload=Node-09246b4e-ad0f-5b5c-2587-4cb19446524f +50:41.905 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:41.905 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:41.905 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:41.905 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:41.905 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:41.905 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-09246b4e-ad0f-5b5c-2587-4cb19446524f +50:41.905 [INFO] TestAgent.server: member joined, marking health alive: member=Node-09246b4e-ad0f-5b5c-2587-4cb19446524f +50:41.906 [INFO] TestAgent.server: federation state anti-entropy synced +50:41.966 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:41.967 [INFO] TestAgent: Synced node info +50:41.967 [DEBUG] TestAgent: Node info in sync +50:42.237 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:42.237 [DEBUG] TestAgent: Node info in sync +50:42.293 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/foo from=127.0.0.1:50000 latency=444.538µs +50:42.293 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/foo from=127.0.0.1:50002 latency=171.117µs +50:42.294 [INFO] TestAgent: Requesting shutdown +50:42.294 [INFO] TestAgent.server: shutting down server +50:42.294 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:42.294 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:42.294 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:42.294 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:42.294 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:42.294 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:42.294 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:42.294 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:42.294 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:42.294 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:42.294 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:42.294Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:42.294Z [INFO] agent.router.manager: shutting down +50:42.294 [INFO] TestAgent: consul server down +50:42.294 [INFO] TestAgent: shutdown complete +50:42.294 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24316 network=tcp +50:42.294 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24316 network=udp +50:42.294 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24317 network=tcp +50:42.795 [INFO] TestAgent: Waiting for endpoints to shut down +50:42.795 [INFO] TestAgent: Endpoints down +--- PASS: TestKVPutCommand_Base64 (1.29s) PASS -ok github.com/hashicorp/consul/command/kv/put 2.193s +ok github.com/hashicorp/consul/command/kv/put 1.419s === RUN TestLeaveCommand_noTabs === PAUSE TestLeaveCommand_noTabs === RUN TestLeaveCommand @@ -87784,142 +87938,142 @@ === RUN TestLeaveCommand_FailOnNonFlagArgs === PAUSE TestLeaveCommand_FailOnNonFlagArgs === CONT TestLeaveCommand_noTabs ---- PASS: TestLeaveCommand_noTabs (0.00s) === CONT TestLeaveCommand_FailOnNonFlagArgs -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestLeaveCommand +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestLeaveCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:18:05.562Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:05.588Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:05.591 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:533ca61b-c4fa-af3c-60c6-ae3a15e654ec Address:127.0.0.1:19195}]" -18:05.592 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-533ca61b-c4fa-af3c-60c6-ae3a15e654ec.dc1 127.0.0.1 -18:05.606 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-533ca61b-c4fa-af3c-60c6-ae3a15e654ec 127.0.0.1 -2021-10-31T23:18:05.606Z [INFO] agent.router: Initializing LAN area manager -18:05.607 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp -18:05.607 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= -18:05.607 [INFO] TestAgent.server: Adding LAN server: server="Node-533ca61b-c4fa-af3c-60c6-ae3a15e654ec (Addr: tcp/127.0.0.1:19195) (DC: dc1)" -18:05.607 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-533ca61b-c4fa-af3c-60c6-ae3a15e654ec.dc1 area=wan -18:05.607 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp -18:05.608 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp -18:05.608 [INFO] TestAgent: started state syncer -18:05.608 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp -18:05.653 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:05.653 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 -18:05.654 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:05.654 [DEBUG] TestAgent.server.raft: vote granted: from=533ca61b-c4fa-af3c-60c6-ae3a15e654ec term=2 tally=1 -18:05.654 [INFO] TestAgent.server.raft: election won: tally=1 -18:05.654 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" -18:05.654 [INFO] TestAgent.server: cluster leadership acquired -18:05.654 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 -18:05.655 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:05.657 [INFO] TestAgent.server: New leader elected: payload=Node-533ca61b-c4fa-af3c-60c6-ae3a15e654ec -18:05.684 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:05.684 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:05.684 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:05.684 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:05.684 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:05.685 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-533ca61b-c4fa-af3c-60c6-ae3a15e654ec -18:05.685 [INFO] TestAgent.server: member joined, marking health alive: member=Node-533ca61b-c4fa-af3c-60c6-ae3a15e654ec -18:05.686 [INFO] TestAgent.server: federation state anti-entropy synced -18:05.742 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:05.742 [INFO] TestAgent: Synced node info -18:05.922 [INFO] TestAgent: Requesting shutdown -18:05.922 [INFO] TestAgent.server: shutting down server -18:05.922 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:05.922 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:05.922 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:05.922 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:05.922 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:05.922 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:05.922 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:05.922 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:05.922 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:05.922 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:05.922 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:05.922Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:05.922Z [INFO] agent.router.manager: shutting down -18:05.923 [INFO] TestAgent: consul server down -18:05.923 [INFO] TestAgent: shutdown complete -18:05.923 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp -18:05.923 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp -18:05.923 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp -18:06.425 [INFO] TestAgent: Waiting for endpoints to shut down -18:06.425 [INFO] TestAgent: Endpoints down ---- PASS: TestLeaveCommand_FailOnNonFlagArgs (0.95s) -2021-10-31T23:18:05.562Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:05.576Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:05.606 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0b23c9ed-b2ad-e431-30e4-5167617b4e41 Address:127.0.0.1:19202}]" -18:05.606 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0b23c9ed-b2ad-e431-30e4-5167617b4e41.dc1 127.0.0.1 -18:05.607 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0b23c9ed-b2ad-e431-30e4-5167617b4e41 127.0.0.1 -18:05.608 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19202 [Follower]" leader= -2021-10-31T23:18:05.613Z [INFO] agent.router: Initializing LAN area manager -18:05.614 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=udp -18:05.614 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0b23c9ed-b2ad-e431-30e4-5167617b4e41.dc1 area=wan -18:05.614 [INFO] TestAgent.server: Adding LAN server: server="Node-0b23c9ed-b2ad-e431-30e4-5167617b4e41 (Addr: tcp/127.0.0.1:19202) (DC: dc1)" -18:05.614 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=tcp -18:05.615 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19198 network=tcp -18:05.615 [INFO] TestAgent: started state syncer -18:05.615 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19203 network=tcp -18:05.673 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:05.673 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19202 [Candidate]" term=2 -18:05.674 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:05.674 [DEBUG] TestAgent.server.raft: vote granted: from=0b23c9ed-b2ad-e431-30e4-5167617b4e41 term=2 tally=1 -18:05.674 [INFO] TestAgent.server.raft: election won: tally=1 -18:05.674 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19202 [Leader]" -18:05.674 [INFO] TestAgent.server: cluster leadership acquired -18:05.674 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19202 -18:05.675 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:05.675 [INFO] TestAgent.server: New leader elected: payload=Node-0b23c9ed-b2ad-e431-30e4-5167617b4e41 -18:05.687 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:05.687 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:05.687 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:05.687 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:05.687 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:05.687 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0b23c9ed-b2ad-e431-30e4-5167617b4e41 -18:05.687 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0b23c9ed-b2ad-e431-30e4-5167617b4e41 -18:05.688 [INFO] TestAgent.server: federation state anti-entropy synced -18:05.718 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:05.718 [INFO] TestAgent: Synced node info -18:06.054 [INFO] TestAgent.server: server starting leave -18:06.054 [INFO] TestAgent.server.serf.wan: serf: EventMemberLeave: Node-0b23c9ed-b2ad-e431-30e4-5167617b4e41.dc1 127.0.0.1 -18:06.054 [INFO] TestAgent.server: Handled event for server in area: event=member-leave server=Node-0b23c9ed-b2ad-e431-30e4-5167617b4e41.dc1 area=wan -2021-10-31T23:18:06.054Z [INFO] agent.router.manager: shutting down -18:07.902 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:07.902 [DEBUG] TestAgent: Node info in sync -18:07.902 [DEBUG] TestAgent: Node info in sync -18:09.058 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-0b23c9ed-b2ad-e431-30e4-5167617b4e41 127.0.0.1 -18:09.058 [INFO] TestAgent.server: Removing LAN server: server="Node-0b23c9ed-b2ad-e431-30e4-5167617b4e41 (Addr: tcp/127.0.0.1:19202) (DC: dc1)" -18:09.058 [WARN] TestAgent.server: deregistering self should be done by follower: name=Node-0b23c9ed-b2ad-e431-30e4-5167617b4e41 -18:09.677 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -18:11.677 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -18:12.060 [INFO] TestAgent.server: Waiting to drain RPC traffic: drain_time=5s -18:13.677 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -18:15.678 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" -18:15.678 [ERROR] TestAgent.server.autopilot: Error promoting servers: error="error getting server raft protocol versions: No servers found" -18:17.062 [INFO] TestAgent: Requesting shutdown -18:17.062 [INFO] TestAgent.server: shutting down server -18:17.062 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:17.062 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:17.062 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:17.062 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:17.062 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:17.062 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:17.062 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:17.062 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:17.062 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:18:17.062Z [INFO] agent.router.manager: shutting down -18:17.062 [INFO] TestAgent: consul server down -18:17.062 [INFO] TestAgent: shutdown complete -18:17.062 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/leave from=127.0.0.1:50026 latency=11.008256969s -18:17.065 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=tcp -18:17.066 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=udp -18:17.066 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19198 network=tcp -18:17.569 [INFO] TestAgent: Waiting for endpoints to shut down -18:17.569 [INFO] TestAgent: Endpoints down ---- PASS: TestLeaveCommand (12.09s) +2022-12-04T05:50:43.613Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:43.629Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:43.631 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5b19b176-e325-12e1-dffb-cd9aa58065a6 Address:127.0.0.1:21237}]" +50:43.631 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21237 [Follower]" leader= +50:43.631 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5b19b176-e325-12e1-dffb-cd9aa58065a6.dc1 127.0.0.1 +50:43.632 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5b19b176-e325-12e1-dffb-cd9aa58065a6 127.0.0.1 +2022-12-04T05:50:43.632Z [INFO] agent.router: Initializing LAN area manager +50:43.632 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5b19b176-e325-12e1-dffb-cd9aa58065a6.dc1 area=wan +50:43.632 [INFO] TestAgent.server: Adding LAN server: server="Node-5b19b176-e325-12e1-dffb-cd9aa58065a6 (Addr: tcp/127.0.0.1:21237) (DC: dc1)" +50:43.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=udp +50:43.633 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=tcp +50:43.633 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21233 network=tcp +50:43.634 [INFO] TestAgent: started state syncer +50:43.634 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21238 network=tcp +50:43.698 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:43.698 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21237 [Candidate]" term=2 +50:43.699 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:43.699 [DEBUG] TestAgent.server.raft: vote granted: from=5b19b176-e325-12e1-dffb-cd9aa58065a6 term=2 tally=1 +50:43.699 [INFO] TestAgent.server.raft: election won: tally=1 +50:43.699 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21237 [Leader]" +50:43.699 [INFO] TestAgent.server: cluster leadership acquired +50:43.699 [INFO] TestAgent.server: New leader elected: payload=Node-5b19b176-e325-12e1-dffb-cd9aa58065a6 +50:43.699 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21237 +50:43.700 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:43.715 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:43.715 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:43.715 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:43.715 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:43.715 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:43.715 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5b19b176-e325-12e1-dffb-cd9aa58065a6 +50:43.715 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5b19b176-e325-12e1-dffb-cd9aa58065a6 +50:43.715 [INFO] TestAgent.server: federation state anti-entropy synced +50:43.844 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:43.845 [INFO] TestAgent: Synced node info +50:43.896 [INFO] TestAgent: Requesting shutdown +50:43.896 [INFO] TestAgent.server: shutting down server +50:43.896 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:43.896 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:43.896 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:43.896 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:43.896 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:43.896 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:43.896 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:43.896 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:43.896 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:43.896 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:43.896 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:43.896Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:43.896Z [INFO] agent.router.manager: shutting down +50:43.896 [INFO] TestAgent: consul server down +50:43.896 [INFO] TestAgent: shutdown complete +50:43.896 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=tcp +50:43.896 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=udp +50:43.896 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21233 network=tcp +50:44.397 [INFO] TestAgent: Waiting for endpoints to shut down +50:44.397 [INFO] TestAgent: Endpoints down +--- PASS: TestLeaveCommand_FailOnNonFlagArgs (0.82s) +2022-12-04T05:50:43.618Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:43.633Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:43.635 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:28602584-3d11-109c-59a0-7888b91ee839 Address:127.0.0.1:21244}]" +50:43.635 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21244 [Follower]" leader= +50:43.640 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-28602584-3d11-109c-59a0-7888b91ee839.dc1 127.0.0.1 +50:43.641 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-28602584-3d11-109c-59a0-7888b91ee839 127.0.0.1 +2022-12-04T05:50:43.641Z [INFO] agent.router: Initializing LAN area manager +50:43.641 [INFO] TestAgent.server: Adding LAN server: server="Node-28602584-3d11-109c-59a0-7888b91ee839 (Addr: tcp/127.0.0.1:21244) (DC: dc1)" +50:43.641 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-28602584-3d11-109c-59a0-7888b91ee839.dc1 area=wan +50:43.641 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21239 network=tcp +50:43.641 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21239 network=udp +50:43.642 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21240 network=tcp +50:43.642 [INFO] TestAgent: started state syncer +50:43.642 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21245 network=tcp +50:43.698 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:43.698 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21244 [Candidate]" term=2 +50:43.698 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:43.698 [DEBUG] TestAgent.server.raft: vote granted: from=28602584-3d11-109c-59a0-7888b91ee839 term=2 tally=1 +50:43.698 [INFO] TestAgent.server.raft: election won: tally=1 +50:43.698 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21244 [Leader]" +50:43.698 [INFO] TestAgent.server: cluster leadership acquired +50:43.698 [INFO] TestAgent.server: New leader elected: payload=Node-28602584-3d11-109c-59a0-7888b91ee839 +50:43.698 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21244 +50:43.699 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:43.714 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:43.714 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:43.714 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:43.714 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:43.714 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:43.714 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-28602584-3d11-109c-59a0-7888b91ee839 +50:43.714 [INFO] TestAgent.server: member joined, marking health alive: member=Node-28602584-3d11-109c-59a0-7888b91ee839 +50:43.715 [INFO] TestAgent.server: federation state anti-entropy synced +50:44.031 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:44.031 [INFO] TestAgent: Synced node info +50:44.031 [DEBUG] TestAgent: Node info in sync +50:44.073 [INFO] TestAgent.server: server starting leave +50:44.074 [INFO] TestAgent.server.serf.wan: serf: EventMemberLeave: Node-28602584-3d11-109c-59a0-7888b91ee839.dc1 127.0.0.1 +50:44.074 [INFO] TestAgent.server: Handled event for server in area: event=member-leave server=Node-28602584-3d11-109c-59a0-7888b91ee839.dc1 area=wan +2022-12-04T05:50:44.074Z [INFO] agent.router.manager: shutting down +50:45.814 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:45.814 [DEBUG] TestAgent: Node info in sync +50:47.074 [INFO] TestAgent.server.serf.lan: serf: EventMemberLeave: Node-28602584-3d11-109c-59a0-7888b91ee839 127.0.0.1 +50:47.074 [INFO] TestAgent.server: Removing LAN server: server="Node-28602584-3d11-109c-59a0-7888b91ee839 (Addr: tcp/127.0.0.1:21244) (DC: dc1)" +50:47.074 [WARN] TestAgent.server: deregistering self should be done by follower: name=Node-28602584-3d11-109c-59a0-7888b91ee839 +50:47.699 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +50:49.699 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +50:50.074 [INFO] TestAgent.server: Waiting to drain RPC traffic: drain_time=5s +50:51.699 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +50:53.699 [ERROR] TestAgent.server.autopilot: Error updating cluster health: error="error getting server raft protocol versions: No servers found" +50:53.699 [ERROR] TestAgent.server.autopilot: Error promoting servers: error="error getting server raft protocol versions: No servers found" +50:55.074 [INFO] TestAgent: Requesting shutdown +50:55.074 [INFO] TestAgent.server: shutting down server +50:55.074 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:55.074 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:55.074 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:55.074 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:55.074 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:55.074 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:55.074 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:55.074 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:55.074 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:50:55.074Z [INFO] agent.router.manager: shutting down +50:55.075 [INFO] TestAgent: consul server down +50:55.075 [INFO] TestAgent: shutdown complete +50:55.075 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/leave from=127.0.0.1:35478 latency=11.001130994s +50:55.075 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21239 network=tcp +50:55.075 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21239 network=udp +50:55.075 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21240 network=tcp +50:55.575 [INFO] TestAgent: Waiting for endpoints to shut down +50:55.575 [INFO] TestAgent: Endpoints down +--- PASS: TestLeaveCommand (12.00s) PASS -ok github.com/hashicorp/consul/command/leave 12.583s +ok github.com/hashicorp/consul/command/leave 12.053s === RUN TestLockCommand_noTabs === PAUSE TestLockCommand_noTabs === RUN TestLockCommand_BadArgs @@ -87943,662 +88097,668 @@ === RUN TestLockCommand_ChildExitCode === PAUSE TestLockCommand_ChildExitCode === CONT TestLockCommand_noTabs ---- PASS: TestLockCommand_noTabs (0.00s) -=== CONT TestLockCommand_ChildExitCode -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestLockCommand_TrySemaphore +=== CONT TestLockCommand_MonitorRetry_Lock_Default === CONT TestLockCommand_MonitorRetry_Lock_Arg +=== CONT TestLockCommand_TryLock === CONT TestLockCommand_MonitorRetry_Semaphore_Default -=== CONT TestLockCommand_MonitorRetry_Lock_Default +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +=== CONT TestLockCommand === CONT TestLockCommand_MonitorRetry_Semaphore_Arg +=== CONT TestLockCommand_ChildExitCode === CONT TestLockCommand_NoShell -=== CONT TestLockCommand_TryLock -=== CONT TestLockCommand +=== CONT TestLockCommand_TrySemaphore === CONT TestLockCommand_BadArgs ---- PASS: TestLockCommand_BadArgs (0.00s) +--- PASS: TestLockCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range +--- PASS: TestLockCommand_BadArgs (0.00s) === RUN TestLockCommand_ChildExitCode/clean_exit === RUN TestLockCommand_ChildExitCode/error_exit === RUN TestLockCommand_ChildExitCode/not_propagated -2021-10-31T23:18:06.754Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:06.905Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:07.030 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5549d409-44fe-5038-ef68-a3f4e8a0bce4 Address:127.0.0.1:29426}]" -18:07.031 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5549d409-44fe-5038-ef68-a3f4e8a0bce4.dc1 127.0.0.1 -18:07.032 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5549d409-44fe-5038-ef68-a3f4e8a0bce4 127.0.0.1 -2021-10-31T23:18:07.032Z [INFO] agent.router: Initializing LAN area manager -18:07.032 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29421 network=udp -18:07.032 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29426 [Follower]" leader= -18:07.035 [INFO] TestAgent.server: Adding LAN server: server="Node-5549d409-44fe-5038-ef68-a3f4e8a0bce4 (Addr: tcp/127.0.0.1:29426) (DC: dc1)" -18:07.035 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5549d409-44fe-5038-ef68-a3f4e8a0bce4.dc1 area=wan -18:07.098 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29421 network=tcp -18:07.099 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29422 network=tcp -18:07.099 [INFO] TestAgent: started state syncer -18:07.099 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29427 network=tcp -18:07.102 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:07.102 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29426 [Candidate]" term=2 -18:07.102 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:07.102 [DEBUG] TestAgent.server.raft: vote granted: from=5549d409-44fe-5038-ef68-a3f4e8a0bce4 term=2 tally=1 -18:07.102 [INFO] TestAgent.server.raft: election won: tally=1 -18:07.102 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29426 [Leader]" -18:07.103 [INFO] TestAgent.server: cluster leadership acquired -18:07.103 [INFO] TestAgent.server: New leader elected: payload=Node-5549d409-44fe-5038-ef68-a3f4e8a0bce4 -18:07.103 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29426 -18:07.126 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:07.129 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:07.129 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:07.129 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:07.129 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:07.129 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:07.129 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5549d409-44fe-5038-ef68-a3f4e8a0bce4 -18:07.129 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5549d409-44fe-5038-ef68-a3f4e8a0bce4 -18:07.150 [INFO] TestAgent.server: federation state anti-entropy synced -18:07.220 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:37474 latency=2.05426ms -18:07.234 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:37474 latency=2.060987ms -18:07.238 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:37474 latency=82.27µs -18:07.241 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=802940c9-835e-888e-7ce3-71edbe5091a3&flags=3304740253564472344 from=127.0.0.1:37474 latency=2.620664ms -18:07.288 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=802940c9-835e-888e-7ce3-71edbe5091a3 from=127.0.0.1:37484 latency=812.533µs -18:07.290 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:37484 latency=261.416µs -18:07.317 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:37474 latency=50.377524ms -18:07.318 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/802940c9-835e-888e-7ce3-71edbe5091a3 from=127.0.0.1:37486 latency=28.450571ms -18:07.320 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=13 from=127.0.0.1:37474 latency=619.302µs -18:07.320 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=14 from=127.0.0.1:37484 latency=1.106179ms -18:07.320 [INFO] TestAgent: Requesting shutdown -18:07.320 [INFO] TestAgent.server: shutting down server -18:07.320 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:07.320 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:07.320 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:07.320 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:07.320 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:07.320 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:07.320 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:07.320 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -18:07.320 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:07.320 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:07.320 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:07.320 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:07.320Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:07.321Z [INFO] agent.router.manager: shutting down -18:07.321 [INFO] TestAgent: consul server down -18:07.321 [INFO] TestAgent: shutdown complete -18:07.321 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29421 network=tcp -18:07.321 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29421 network=udp -18:07.321 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29422 network=tcp -18:07.821 [INFO] TestAgent: Waiting for endpoints to shut down -18:07.821 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand_MonitorRetry_Lock_Default (1.25s) -2021-10-31T23:18:06.835Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:06.927Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:06.982 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:13e143ad-4070-4d20-47c7-cfcdfa541f11 Address:127.0.0.1:29433}]" -18:06.983 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-13e143ad-4070-4d20-47c7-cfcdfa541f11.dc1 127.0.0.1 -18:06.984 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-13e143ad-4070-4d20-47c7-cfcdfa541f11 127.0.0.1 -2021-10-31T23:18:06.984Z [INFO] agent.router: Initializing LAN area manager -18:06.995 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-13e143ad-4070-4d20-47c7-cfcdfa541f11.dc1 area=wan -18:06.995 [INFO] TestAgent.server: Adding LAN server: server="Node-13e143ad-4070-4d20-47c7-cfcdfa541f11 (Addr: tcp/127.0.0.1:29433) (DC: dc1)" -18:06.995 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29428 network=udp -18:06.995 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29428 network=tcp -18:06.996 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29429 network=tcp -18:06.996 [INFO] TestAgent: started state syncer -18:06.997 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29434 network=tcp -18:07.002 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29433 [Follower]" leader= -18:07.047 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:07.047 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29433 [Candidate]" term=2 -18:07.047 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:07.047 [DEBUG] TestAgent.server.raft: vote granted: from=13e143ad-4070-4d20-47c7-cfcdfa541f11 term=2 tally=1 -18:07.047 [INFO] TestAgent.server.raft: election won: tally=1 -18:07.047 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29433 [Leader]" -18:07.047 [INFO] TestAgent.server: cluster leadership acquired -18:07.047 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29433 -18:07.048 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:07.050 [INFO] TestAgent.server: New leader elected: payload=Node-13e143ad-4070-4d20-47c7-cfcdfa541f11 -18:07.062 [INFO] TestAgent: Synced node info -18:07.069 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:07.070 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:07.070 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:07.070 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:07.070 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:07.071 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-13e143ad-4070-4d20-47c7-cfcdfa541f11 -18:07.071 [INFO] TestAgent.server: member joined, marking health alive: member=Node-13e143ad-4070-4d20-47c7-cfcdfa541f11 -18:07.072 [INFO] TestAgent.server: federation state anti-entropy synced -18:07.230 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:33114 latency=29.861415ms -18:07.233 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:33114 latency=508.422µs -18:07.238 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/96de3e87-f1ce-9621-8641-638a99061e1f?acquire=96de3e87-f1ce-9621-8641-638a99061e1f&flags=16210313421097356768 from=127.0.0.1:33114 latency=510.827µs -18:07.241 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse=&wait=15000ms from=127.0.0.1:33114 latency=2.53655ms -18:07.288 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=0&flags=16210313421097356768 from=127.0.0.1:33114 latency=8.058608ms -18:07.320 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&recurse= from=127.0.0.1:33114 latency=333.581µs -18:07.330 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:33130 latency=245.692µs -18:07.331 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=15&flags=16210313421097356768 from=127.0.0.1:33130 latency=402.862µs -18:07.331 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&index=15&recurse= from=127.0.0.1:33114 latency=11.364259ms -18:07.332 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/96de3e87-f1ce-9621-8641-638a99061e1f from=127.0.0.1:33130 latency=528.674µs -18:07.334 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse= from=127.0.0.1:33130 latency=999.377µs -18:07.339 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=16 from=127.0.0.1:33130 latency=541.514µs -18:07.339 [INFO] TestAgent: Requesting shutdown -18:07.339 [INFO] TestAgent.server: shutting down server -18:07.339 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:07.339 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:07.339 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:07.339 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:07.339 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:07.339 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:07.339 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:07.339 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:07.339 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:07.339 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:07.339 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:07.340Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:07.340Z [INFO] agent.router.manager: shutting down -18:07.340 [INFO] TestAgent: consul server down -18:07.340 [INFO] TestAgent: shutdown complete -18:07.340 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29428 network=tcp -18:07.340 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29428 network=udp -18:07.340 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29429 network=tcp -18:07.841 [INFO] TestAgent: Waiting for endpoints to shut down -18:07.841 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand_MonitorRetry_Semaphore_Arg (1.27s) -2021-10-31T23:18:06.964Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:07.018Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:07.020 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b37e65f6-e36b-4735-8470-d15dc9923eec Address:127.0.0.1:29454}]" -18:07.021 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b37e65f6-e36b-4735-8470-d15dc9923eec.dc1 127.0.0.1 -18:07.022 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b37e65f6-e36b-4735-8470-d15dc9923eec 127.0.0.1 -2021-10-31T23:18:07.023Z [INFO] agent.router: Initializing LAN area manager -18:07.062 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29454 [Follower]" leader= -18:07.070 [INFO] TestAgent.server: Adding LAN server: server="Node-b37e65f6-e36b-4735-8470-d15dc9923eec (Addr: tcp/127.0.0.1:29454) (DC: dc1)" -18:07.070 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b37e65f6-e36b-4735-8470-d15dc9923eec.dc1 area=wan -18:07.096 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29449 network=udp -18:07.099 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29449 network=tcp -18:07.100 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29450 network=tcp -18:07.100 [INFO] TestAgent: started state syncer -18:07.100 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29455 network=tcp -18:07.138 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:07.138 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29454 [Candidate]" term=2 -18:07.138 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:07.138 [DEBUG] TestAgent.server.raft: vote granted: from=b37e65f6-e36b-4735-8470-d15dc9923eec term=2 tally=1 -18:07.138 [INFO] TestAgent.server.raft: election won: tally=1 -18:07.138 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29454 [Leader]" -18:07.138 [INFO] TestAgent.server: cluster leadership acquired -18:07.138 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29454 -18:07.139 [INFO] TestAgent.server: New leader elected: payload=Node-b37e65f6-e36b-4735-8470-d15dc9923eec -18:07.162 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:07.163 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:07.163 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:07.163 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:07.163 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:07.163 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:07.163 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b37e65f6-e36b-4735-8470-d15dc9923eec -18:07.163 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b37e65f6-e36b-4735-8470-d15dc9923eec -18:07.164 [INFO] TestAgent.server: federation state anti-entropy synced -18:07.195 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:07.195 [INFO] TestAgent: Synced node info -18:07.197 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:07.197 [DEBUG] TestAgent: Node info in sync -18:07.197 [DEBUG] TestAgent: Node info in sync -18:07.322 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:47934 latency=34.856247ms -18:07.325 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:47934 latency=658.049µs -18:07.325 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:47934 latency=79.802µs -18:07.326 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=328b977e-011c-175f-24cd-58856de2cede&flags=3304740253564472344 from=127.0.0.1:47934 latency=414.498µs -18:07.327 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:47934 latency=189.222µs -18:07.344 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=328b977e-011c-175f-24cd-58856de2cede from=127.0.0.1:47950 latency=571.545µs -18:07.345 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:47934 latency=17.348163ms -18:07.346 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/328b977e-011c-175f-24cd-58856de2cede from=127.0.0.1:47934 latency=787.624µs -18:07.347 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:47950 latency=1.961896ms -18:07.354 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:47950 latency=598.981µs -18:07.355 [INFO] TestAgent: Requesting shutdown -18:07.355 [INFO] TestAgent.server: shutting down server -18:07.355 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:07.355 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:07.355 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:07.355 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:07.355 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:07.355 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:07.355 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:07.355 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:07.355 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:07.355 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:07.355 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:07.355Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:07.355Z [INFO] agent.router.manager: shutting down -18:07.355 [INFO] TestAgent: consul server down -18:07.355 [INFO] TestAgent: shutdown complete -18:07.355 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29449 network=tcp -18:07.355 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29449 network=udp -18:07.355 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29450 network=tcp -18:07.857 [INFO] TestAgent: Waiting for endpoints to shut down -18:07.857 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand (1.29s) -2021-10-31T23:18:07.097Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:07.219Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:07.226 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:39e2ce14-ff52-00bc-47e6-3374c82a7834 Address:127.0.0.1:29461}]" -18:07.227 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-39e2ce14-ff52-00bc-47e6-3374c82a7834.dc1 127.0.0.1 -18:07.227 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-39e2ce14-ff52-00bc-47e6-3374c82a7834 127.0.0.1 -2021-10-31T23:18:07.227Z [INFO] agent.router: Initializing LAN area manager -18:07.228 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29456 network=udp -18:07.228 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29461 [Follower]" leader= -18:07.228 [INFO] TestAgent.server: Adding LAN server: server="Node-39e2ce14-ff52-00bc-47e6-3374c82a7834 (Addr: tcp/127.0.0.1:29461) (DC: dc1)" -18:07.228 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-39e2ce14-ff52-00bc-47e6-3374c82a7834.dc1 area=wan -18:07.228 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29456 network=tcp -18:07.229 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29457 network=tcp -18:07.229 [INFO] TestAgent: started state syncer -18:07.229 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29462 network=tcp -18:07.276 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:07.276 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29461 [Candidate]" term=2 -18:07.276 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:07.276 [DEBUG] TestAgent.server.raft: vote granted: from=39e2ce14-ff52-00bc-47e6-3374c82a7834 term=2 tally=1 -18:07.276 [INFO] TestAgent.server.raft: election won: tally=1 -18:07.276 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29461 [Leader]" -18:07.276 [INFO] TestAgent.server: cluster leadership acquired -18:07.277 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29461 -18:07.277 [INFO] TestAgent.server: New leader elected: payload=Node-39e2ce14-ff52-00bc-47e6-3374c82a7834 -18:07.280 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:07.314 [INFO] TestAgent: Synced node info -18:07.314 [DEBUG] TestAgent: Node info in sync -18:07.315 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:07.315 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:07.315 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:07.315 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:07.315 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:07.315 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-39e2ce14-ff52-00bc-47e6-3374c82a7834 -18:07.315 [INFO] TestAgent.server: member joined, marking health alive: member=Node-39e2ce14-ff52-00bc-47e6-3374c82a7834 -18:07.316 [INFO] TestAgent.server: federation state anti-entropy synced -18:07.330 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:38436 latency=1.237131ms -18:07.332 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:38436 latency=552.147µs -18:07.333 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:38436 latency=44.958µs -18:07.335 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=f43913ce-08e3-cb64-8964-787c35ed5d72&flags=3304740253564472344 from=127.0.0.1:38436 latency=2.382958ms -18:07.338 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:38436 latency=242.534µs -18:07.366 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=f43913ce-08e3-cb64-8964-787c35ed5d72 from=127.0.0.1:38446 latency=708.022µs -18:07.366 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:38436 latency=26.500248ms -18:07.374 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/f43913ce-08e3-cb64-8964-787c35ed5d72 from=127.0.0.1:38446 latency=723.019µs -18:07.374 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:38436 latency=118.678µs -18:07.376 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:38436 latency=308.581µs -18:07.376 [INFO] TestAgent: Requesting shutdown -18:07.376 [INFO] TestAgent.server: shutting down server -18:07.376 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:07.376 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:07.376 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:07.376 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:07.376 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:07.376 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:07.376 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:07.376 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:07.376 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:07.376 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:07.377 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:07.377Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:07.377Z [INFO] agent.router.manager: shutting down -18:07.377 [INFO] TestAgent: consul server down -18:07.377 [INFO] TestAgent: shutdown complete -18:07.377 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29456 network=tcp -18:07.377 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29456 network=udp -18:07.377 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29457 network=tcp -18:07.877 [INFO] TestAgent: Waiting for endpoints to shut down -18:07.877 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand_MonitorRetry_Lock_Arg (1.31s) -2021-10-31T23:18:06.801Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:07.097Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:07.182 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5f788d1b-d594-bd3a-04f0-22a8080bb111 Address:127.0.0.1:29419}]" -18:07.196 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5f788d1b-d594-bd3a-04f0-22a8080bb111.dc1 127.0.0.1 -18:07.197 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5f788d1b-d594-bd3a-04f0-22a8080bb111 127.0.0.1 -2021-10-31T23:18:07.197Z [INFO] agent.router: Initializing LAN area manager -18:07.198 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29419 [Follower]" leader= -18:07.199 [INFO] TestAgent.server: Adding LAN server: server="Node-5f788d1b-d594-bd3a-04f0-22a8080bb111 (Addr: tcp/127.0.0.1:29419) (DC: dc1)" -18:07.199 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5f788d1b-d594-bd3a-04f0-22a8080bb111.dc1 area=wan -18:07.199 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29414 network=tcp -18:07.200 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29414 network=udp -18:07.230 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29415 network=tcp -18:07.230 [INFO] TestAgent: started state syncer -18:07.230 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29420 network=tcp -18:07.263 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:07.263 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29419 [Candidate]" term=2 -18:07.264 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:07.264 [DEBUG] TestAgent.server.raft: vote granted: from=5f788d1b-d594-bd3a-04f0-22a8080bb111 term=2 tally=1 -18:07.264 [INFO] TestAgent.server.raft: election won: tally=1 -18:07.264 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29419 [Leader]" -18:07.264 [INFO] TestAgent.server: cluster leadership acquired -18:07.264 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29419 -18:07.264 [INFO] TestAgent.server: New leader elected: payload=Node-5f788d1b-d594-bd3a-04f0-22a8080bb111 -18:07.267 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:07.318 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:07.318 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:07.318 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:07.318 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:07.318 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:07.318 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5f788d1b-d594-bd3a-04f0-22a8080bb111 -18:07.318 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5f788d1b-d594-bd3a-04f0-22a8080bb111 -18:07.319 [INFO] TestAgent.server: federation state anti-entropy synced -18:07.345 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:35882 latency=1.329332ms -18:07.354 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:35882 latency=874.175µs -18:07.366 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/4b266000-5bdf-9b76-4283-bf5359cc1ca3?acquire=4b266000-5bdf-9b76-4283-bf5359cc1ca3&flags=16210313421097356768 from=127.0.0.1:35882 latency=722.711µs -18:07.374 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse=&wait=15000ms from=127.0.0.1:35882 latency=368.865µs -18:07.375 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=0&flags=16210313421097356768 from=127.0.0.1:35882 latency=431.447µs -18:07.387 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&recurse= from=127.0.0.1:35882 latency=9.789694ms -18:07.398 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:35896 latency=292.815µs -18:07.399 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=14&flags=16210313421097356768 from=127.0.0.1:35896 latency=530.549µs -18:07.400 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/4b266000-5bdf-9b76-4283-bf5359cc1ca3 from=127.0.0.1:35896 latency=413.043µs -18:07.401 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse= from=127.0.0.1:35896 latency=188.932µs -18:07.402 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:35896 latency=791.097µs -18:07.402 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&index=14&recurse= from=127.0.0.1:35882 latency=281.879µs -18:07.402 [INFO] TestAgent: Requesting shutdown -18:07.403 [INFO] TestAgent.server: shutting down server -18:07.403 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:07.403 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:07.403 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:07.403 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:07.403 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:07.403 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/4b266000-5bdf-9b76-4283-bf5359cc1ca3 from=127.0.0.1:35900 latency=2.104162ms -18:07.403 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:07.403 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:07.403 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -18:07.403 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -18:07.403 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:07.403 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:07.403 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:18:07.403Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:07.403Z [INFO] agent.router.manager: shutting down -18:07.403 [INFO] TestAgent: consul server down -18:07.403 [INFO] TestAgent: shutdown complete -18:07.403 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29414 network=tcp -18:07.403 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29414 network=udp -18:07.403 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29415 network=tcp -18:07.904 [INFO] TestAgent: Waiting for endpoints to shut down -18:07.904 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand_MonitorRetry_Semaphore_Default (1.34s) -2021-10-31T23:18:06.696Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:06.929Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:06.932 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cf69e4ea-a2ab-7c4a-9bcf-05d904926eb9 Address:127.0.0.1:29405}]" -18:06.942 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cf69e4ea-a2ab-7c4a-9bcf-05d904926eb9.dc1 127.0.0.1 -18:06.942 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cf69e4ea-a2ab-7c4a-9bcf-05d904926eb9 127.0.0.1 -2021-10-31T23:18:06.943Z [INFO] agent.router: Initializing LAN area manager -18:06.962 [INFO] TestAgent.server: Adding LAN server: server="Node-cf69e4ea-a2ab-7c4a-9bcf-05d904926eb9 (Addr: tcp/127.0.0.1:29405) (DC: dc1)" -18:06.974 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=tcp -18:07.030 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29405 [Follower]" leader= -18:07.030 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=udp -18:07.031 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29401 network=tcp -18:07.031 [INFO] TestAgent: started state syncer -18:07.031 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cf69e4ea-a2ab-7c4a-9bcf-05d904926eb9.dc1 area=wan -18:07.031 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29406 network=tcp -18:07.095 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:07.095 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29405 [Candidate]" term=2 -18:07.096 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:07.096 [DEBUG] TestAgent.server.raft: vote granted: from=cf69e4ea-a2ab-7c4a-9bcf-05d904926eb9 term=2 tally=1 -18:07.096 [INFO] TestAgent.server.raft: election won: tally=1 -18:07.096 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29405 [Leader]" -18:07.096 [INFO] TestAgent.server: cluster leadership acquired -18:07.096 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29405 -18:07.097 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:07.099 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:07.099 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:07.099 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:07.099 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:07.099 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:07.099 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cf69e4ea-a2ab-7c4a-9bcf-05d904926eb9 -18:07.099 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cf69e4ea-a2ab-7c4a-9bcf-05d904926eb9 -18:07.100 [INFO] TestAgent.server: New leader elected: payload=Node-cf69e4ea-a2ab-7c4a-9bcf-05d904926eb9 -18:07.100 [INFO] TestAgent.server: federation state anti-entropy synced -18:07.130 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:07.157 [INFO] TestAgent: Synced node info -18:07.326 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:49738 latency=5.836375ms -18:07.329 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:49738 latency=610.364µs -18:07.334 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:49738 latency=47.476µs -18:07.335 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=6c31a053-85fe-d46b-8be6-7b7cb0f85c7c&flags=3304740253564472344 from=127.0.0.1:49738 latency=659.575µs -18:07.339 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:49738 latency=299.913µs -18:07.375 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=6c31a053-85fe-d46b-8be6-7b7cb0f85c7c from=127.0.0.1:49752 latency=444.857µs -18:07.376 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:49738 latency=35.789838ms -18:07.377 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:49752 latency=281.481µs -18:07.377 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/6c31a053-85fe-d46b-8be6-7b7cb0f85c7c from=127.0.0.1:49754 latency=450.439µs -18:07.380 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:49754 latency=488.942µs -18:07.382 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:49756 latency=1.170621ms -18:07.393 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:49756 latency=1.020994ms -18:07.394 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:49756 latency=70.468µs -18:07.395 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=5c88bd05-f4a1-b664-9721-3a9939d1354c&flags=3304740253564472344 from=127.0.0.1:49756 latency=489.411µs -18:07.397 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:49756 latency=298.787µs -18:07.411 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=5c88bd05-f4a1-b664-9721-3a9939d1354c from=127.0.0.1:49764 latency=733.986µs -18:07.411 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=19 from=127.0.0.1:49756 latency=9.26789ms -18:07.413 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/5c88bd05-f4a1-b664-9721-3a9939d1354c from=127.0.0.1:49756 latency=742.704µs -18:07.413 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:49764 latency=175.065µs -18:07.415 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=20 from=127.0.0.1:49756 latency=684.507µs -18:07.424 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:49766 latency=1.877889ms -18:07.429 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:49766 latency=877.155µs -18:07.430 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:49766 latency=109.452µs -18:07.431 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=b7b7c222-2f96-8982-5bf6-cfa1c30f6847&flags=3304740253564472344 from=127.0.0.1:49766 latency=834.988µs -18:07.439 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:49766 latency=252.06µs -18:07.445 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=b7b7c222-2f96-8982-5bf6-cfa1c30f6847 from=127.0.0.1:49768 latency=606.593µs -18:07.446 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=24 from=127.0.0.1:49766 latency=5.325338ms -18:07.451 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/b7b7c222-2f96-8982-5bf6-cfa1c30f6847 from=127.0.0.1:49766 latency=645.226µs -18:07.451 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:49768 latency=147.126µs -18:07.452 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=25 from=127.0.0.1:49768 latency=379.32µs -18:07.452 [INFO] TestAgent: Requesting shutdown -18:07.452 [INFO] TestAgent.server: shutting down server -18:07.452 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:07.452 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:07.452 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:07.452 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:07.452 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:07.452 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:07.452 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:07.452 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:07.452 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:07.452 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:07.452 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:07.452Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:07.452Z [INFO] agent.router.manager: shutting down -18:07.452 [INFO] TestAgent: consul server down -18:07.452 [INFO] TestAgent: shutdown complete -18:07.452 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=tcp -18:07.452 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=udp -18:07.452 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29401 network=tcp -18:07.953 [INFO] TestAgent: Waiting for endpoints to shut down -18:07.953 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand_ChildExitCode (1.39s) - --- PASS: TestLockCommand_ChildExitCode/clean_exit (0.06s) - --- PASS: TestLockCommand_ChildExitCode/error_exit (0.04s) - --- PASS: TestLockCommand_ChildExitCode/not_propagated (0.03s) -2021-10-31T23:18:06.820Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:07.002Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:07.014 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:178dd940-3ca6-5b30-b461-9a40ae221e62 Address:127.0.0.1:29440}]" -18:07.015 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-178dd940-3ca6-5b30-b461-9a40ae221e62.dc1 127.0.0.1 -18:07.016 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-178dd940-3ca6-5b30-b461-9a40ae221e62 127.0.0.1 -2021-10-31T23:18:07.016Z [INFO] agent.router: Initializing LAN area manager -18:07.042 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29440 [Follower]" leader= -18:07.054 [INFO] TestAgent.server: Adding LAN server: server="Node-178dd940-3ca6-5b30-b461-9a40ae221e62 (Addr: tcp/127.0.0.1:29440) (DC: dc1)" -18:07.054 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-178dd940-3ca6-5b30-b461-9a40ae221e62.dc1 area=wan -18:07.095 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29435 network=tcp -18:07.100 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29435 network=udp -18:07.101 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29436 network=tcp -18:07.101 [INFO] TestAgent: started state syncer -18:07.101 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29441 network=tcp -18:07.102 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:07.102 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29440 [Candidate]" term=2 -18:07.102 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:07.102 [DEBUG] TestAgent.server.raft: vote granted: from=178dd940-3ca6-5b30-b461-9a40ae221e62 term=2 tally=1 -18:07.102 [INFO] TestAgent.server.raft: election won: tally=1 -18:07.102 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29440 [Leader]" -18:07.103 [INFO] TestAgent.server: cluster leadership acquired -18:07.103 [INFO] TestAgent.server: New leader elected: payload=Node-178dd940-3ca6-5b30-b461-9a40ae221e62 -18:07.103 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29440 -18:07.195 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:07.198 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:07.198 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:07.198 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:07.198 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:07.198 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:07.198 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-178dd940-3ca6-5b30-b461-9a40ae221e62 -18:07.198 [INFO] TestAgent.server: member joined, marking health alive: member=Node-178dd940-3ca6-5b30-b461-9a40ae221e62 -18:07.200 [INFO] TestAgent.server: federation state anti-entropy synced -18:07.416 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:34924 latency=16.263593ms -18:07.418 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:07.418 [INFO] TestAgent: Synced node info -18:07.431 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:34924 latency=10.531444ms -18:07.432 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:34924 latency=54.436µs -18:07.440 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=75a91bfe-1eaf-07a2-4136-22a72a78cf38&flags=3304740253564472344 from=127.0.0.1:34924 latency=7.582336ms -18:07.441 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:34924 latency=307.252µs -18:07.454 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=75a91bfe-1eaf-07a2-4136-22a72a78cf38 from=127.0.0.1:34934 latency=618.936µs -18:07.455 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:34924 latency=13.521126ms -18:07.456 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/75a91bfe-1eaf-07a2-4136-22a72a78cf38 from=127.0.0.1:34924 latency=464.498µs -18:07.456 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:34934 latency=140.507µs -18:07.457 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:34934 latency=325.401µs -18:07.457 [INFO] TestAgent: Requesting shutdown -18:07.457 [INFO] TestAgent.server: shutting down server -18:07.457 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:07.457 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:07.457 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:07.457 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:07.457 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:07.457 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:07.457 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:07.457 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:07.457 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:07.457 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:07.457 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:07.457Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:07.457Z [INFO] agent.router.manager: shutting down -18:07.457 [INFO] TestAgent: consul server down -18:07.457 [INFO] TestAgent: shutdown complete -18:07.457 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29435 network=tcp -18:07.457 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29435 network=udp -18:07.458 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29436 network=tcp -18:07.958 [INFO] TestAgent: Waiting for endpoints to shut down -18:07.958 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand_NoShell (1.39s) -2021-10-31T23:18:07.099Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:07.165Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:07.181 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d23f6d14-f201-16d0-cf48-090d2d026c4c Address:127.0.0.1:29412}]" -18:07.181 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d23f6d14-f201-16d0-cf48-090d2d026c4c.dc1 127.0.0.1 -18:07.183 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d23f6d14-f201-16d0-cf48-090d2d026c4c 127.0.0.1 -2021-10-31T23:18:07.183Z [INFO] agent.router: Initializing LAN area manager -18:07.194 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29412 [Follower]" leader= -18:07.196 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29407 network=udp -18:07.196 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d23f6d14-f201-16d0-cf48-090d2d026c4c.dc1 area=wan -18:07.196 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29407 network=tcp -18:07.197 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29408 network=tcp -18:07.198 [INFO] TestAgent: started state syncer -18:07.198 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29413 network=tcp -18:07.198 [INFO] TestAgent.server: Adding LAN server: server="Node-d23f6d14-f201-16d0-cf48-090d2d026c4c (Addr: tcp/127.0.0.1:29412) (DC: dc1)" -18:07.264 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:07.264 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29412 [Candidate]" term=2 -18:07.265 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:07.265 [DEBUG] TestAgent.server.raft: vote granted: from=d23f6d14-f201-16d0-cf48-090d2d026c4c term=2 tally=1 -18:07.265 [INFO] TestAgent.server.raft: election won: tally=1 -18:07.265 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29412 [Leader]" -18:07.265 [INFO] TestAgent.server: cluster leadership acquired -18:07.265 [INFO] TestAgent.server: New leader elected: payload=Node-d23f6d14-f201-16d0-cf48-090d2d026c4c -18:07.270 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29412 -18:07.314 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:07.316 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:07.316 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:07.316 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:07.316 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:07.316 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:07.316 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d23f6d14-f201-16d0-cf48-090d2d026c4c -18:07.316 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d23f6d14-f201-16d0-cf48-090d2d026c4c -18:07.316 [INFO] TestAgent.server: federation state anti-entropy synced -18:07.414 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:07.414 [INFO] TestAgent: Synced node info -18:07.414 [DEBUG] TestAgent: Node info in sync -18:07.607 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:46086 latency=1.315476ms -18:07.614 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:46086 latency=721.156µs -18:07.615 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/7263f2ba-3bcd-71c2-efd3-92c29ea013ce?acquire=7263f2ba-3bcd-71c2-efd3-92c29ea013ce&flags=16210313421097356768 from=127.0.0.1:46086 latency=403.298µs -18:07.616 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse=&wait=10000ms from=127.0.0.1:46086 latency=161.973µs -18:07.616 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=0&flags=16210313421097356768 from=127.0.0.1:46086 latency=380.879µs -18:07.618 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&recurse= from=127.0.0.1:46086 latency=351.869µs -18:07.635 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:46088 latency=322.585µs -18:07.636 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=15&flags=16210313421097356768 from=127.0.0.1:46088 latency=442.31µs -18:07.636 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&index=15&recurse= from=127.0.0.1:46086 latency=17.636203ms -18:07.637 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/7263f2ba-3bcd-71c2-efd3-92c29ea013ce from=127.0.0.1:46088 latency=362.27µs -18:07.642 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse= from=127.0.0.1:46088 latency=331.018µs -18:07.642 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/7263f2ba-3bcd-71c2-efd3-92c29ea013ce from=127.0.0.1:46086 latency=621.238µs -18:07.646 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=16 from=127.0.0.1:46086 latency=529.047µs -18:07.646 [INFO] TestAgent: Requesting shutdown -18:07.646 [INFO] TestAgent.server: shutting down server -18:07.646 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:07.646 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:07.646 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:07.646 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:07.646 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:07.646 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:07.646 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:07.646 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:07.646 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:07.646 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:07.647 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:07.647Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:07.647Z [INFO] agent.router.manager: shutting down -18:07.647 [INFO] TestAgent: consul server down -18:07.647 [INFO] TestAgent: shutdown complete -18:07.647 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29407 network=tcp -18:07.647 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29407 network=udp -18:07.647 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29408 network=tcp -18:08.153 [INFO] TestAgent: Waiting for endpoints to shut down -18:08.153 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand_TrySemaphore (1.58s) -2021-10-31T23:18:06.910Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:06.960Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:06.974 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f141a053-719d-3ec0-bb3e-54fe76937ca6 Address:127.0.0.1:29447}]" -18:06.979 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29447 [Follower]" leader= -18:06.982 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f141a053-719d-3ec0-bb3e-54fe76937ca6.dc1 127.0.0.1 -18:07.004 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f141a053-719d-3ec0-bb3e-54fe76937ca6 127.0.0.1 -2021-10-31T23:18:07.004Z [INFO] agent.router: Initializing LAN area manager -18:07.041 [INFO] TestAgent.server: Adding LAN server: server="Node-f141a053-719d-3ec0-bb3e-54fe76937ca6 (Addr: tcp/127.0.0.1:29447) (DC: dc1)" -18:07.042 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f141a053-719d-3ec0-bb3e-54fe76937ca6.dc1 area=wan -18:07.068 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:07.069 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29447 [Candidate]" term=2 -18:07.069 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:07.069 [DEBUG] TestAgent.server.raft: vote granted: from=f141a053-719d-3ec0-bb3e-54fe76937ca6 term=2 tally=1 -18:07.069 [INFO] TestAgent.server.raft: election won: tally=1 -18:07.069 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29447 [Leader]" -18:07.069 [INFO] TestAgent.server: cluster leadership acquired -18:07.069 [INFO] TestAgent.server: New leader elected: payload=Node-f141a053-719d-3ec0-bb3e-54fe76937ca6 -18:07.070 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29447 -18:07.070 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:07.072 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:07.072 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:07.072 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:07.072 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:07.072 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:07.072 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f141a053-719d-3ec0-bb3e-54fe76937ca6 -18:07.072 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f141a053-719d-3ec0-bb3e-54fe76937ca6 -18:07.095 [INFO] TestAgent.server: federation state anti-entropy synced -18:07.098 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29442 network=udp -18:07.098 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29442 network=tcp -18:07.099 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29443 network=tcp -18:07.099 [INFO] TestAgent: started state syncer -18:07.100 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29448 network=tcp -18:07.100 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:07.101 [INFO] TestAgent: Synced node info -18:07.128 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:59794 latency=27.224806ms -18:07.216 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:59794 latency=16.720685ms -18:07.229 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=10000ms from=127.0.0.1:59794 latency=110.412µs -18:07.231 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=a74fe141-ab25-b734-ffac-9539f0ee206b&flags=3304740253564472344 from=127.0.0.1:59794 latency=1.453193ms -18:07.234 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:59794 latency=446.014µs -18:07.260 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:59794 latency=22.241272ms -18:07.262 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=a74fe141-ab25-b734-ffac-9539f0ee206b from=127.0.0.1:59800 latency=3.241927ms -18:07.263 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/a74fe141-ab25-b734-ffac-9539f0ee206b from=127.0.0.1:59794 latency=650.23µs -18:07.264 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:59800 latency=206.974µs -18:07.265 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:59800 latency=672.788µs -18:07.265 [INFO] TestAgent: Requesting shutdown -18:07.279 [INFO] TestAgent.server: shutting down server -18:07.279 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:07.280 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:07.280 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:07.280 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:07.280 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:07.280 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:07.280 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:07.280 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:07.280 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:07.280 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:07.280 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:07.280Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:07.280Z [INFO] agent.router.manager: shutting down -18:07.285 [INFO] TestAgent: consul server down -18:07.285 [INFO] TestAgent: shutdown complete -18:07.287 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29442 network=tcp -18:07.287 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29442 network=udp -18:07.287 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29443 network=tcp -18:08.291 [WARN] TestAgent: Timeout stopping server: protocol=HTTP address=127.0.0.1:29443 network=tcp -18:08.291 [INFO] TestAgent: Waiting for endpoints to shut down -18:08.291 [INFO] TestAgent: Endpoints down ---- PASS: TestLockCommand_TryLock (1.72s) +2022-12-04T05:50:44.572Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:44.600Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:44.619 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:54bdfd41-2b8d-04e2-26cb-53415005490a Address:127.0.0.1:13125}]" +50:44.621 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13125 [Follower]" leader= +50:44.622 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-54bdfd41-2b8d-04e2-26cb-53415005490a.dc1 127.0.0.1 +50:44.658 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-54bdfd41-2b8d-04e2-26cb-53415005490a 127.0.0.1 +2022-12-04T05:50:44.658Z [INFO] agent.router: Initializing LAN area manager +50:44.660 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13120 network=udp +50:44.660 [INFO] TestAgent.server: Adding LAN server: server="Node-54bdfd41-2b8d-04e2-26cb-53415005490a (Addr: tcp/127.0.0.1:13125) (DC: dc1)" +50:44.660 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-54bdfd41-2b8d-04e2-26cb-53415005490a.dc1 area=wan +50:44.660 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13120 network=tcp +50:44.662 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13121 network=tcp +50:44.662 [INFO] TestAgent: started state syncer +50:44.665 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13126 network=tcp +50:44.673 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:44.673 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13125 [Candidate]" term=2 +50:44.673 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:44.673 [DEBUG] TestAgent.server.raft: vote granted: from=54bdfd41-2b8d-04e2-26cb-53415005490a term=2 tally=1 +50:44.673 [INFO] TestAgent.server.raft: election won: tally=1 +50:44.673 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13125 [Leader]" +50:44.673 [INFO] TestAgent.server: cluster leadership acquired +50:44.673 [INFO] TestAgent.server: New leader elected: payload=Node-54bdfd41-2b8d-04e2-26cb-53415005490a +50:44.673 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13125 +50:44.674 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:44.675 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:44.675 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:44.675 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:44.675 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:44.675 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:44.675 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-54bdfd41-2b8d-04e2-26cb-53415005490a +50:44.675 [INFO] TestAgent.server: member joined, marking health alive: member=Node-54bdfd41-2b8d-04e2-26cb-53415005490a +50:44.676 [INFO] TestAgent.server: federation state anti-entropy synced +50:44.753 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:49460 latency=1.11557ms +50:44.755 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:49460 latency=569.423µs +50:44.755 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:49460 latency=35.019µs +50:44.756 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=eabc4e12-c898-62f3-f094-822a8d641798&flags=3304740253564472344 from=127.0.0.1:49460 latency=472.315µs +50:44.756 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:49460 latency=145.863µs +50:44.795 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=eabc4e12-c898-62f3-f094-822a8d641798 from=127.0.0.1:49462 latency=360.715µs +50:44.796 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=13 from=127.0.0.1:49460 latency=38.734987ms +50:44.796 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:49462 latency=174.237µs +50:44.796 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/eabc4e12-c898-62f3-f094-822a8d641798 from=127.0.0.1:49460 latency=437.571µs +50:44.797 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=14 from=127.0.0.1:49462 latency=459.155µs +50:44.797 [INFO] TestAgent: Requesting shutdown +50:44.797 [INFO] TestAgent.server: shutting down server +50:44.797 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:44.797 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:44.797 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:44.797 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:44.797 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:44.797 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:44.797 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:44.797 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:44.797 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:44.797 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +50:44.797 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:44.797 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:50:44.797Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:44.797Z [INFO] agent.router.manager: shutting down +50:44.798 [INFO] TestAgent: consul server down +50:44.798 [INFO] TestAgent: shutdown complete +50:44.798 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13120 network=tcp +50:44.798 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13120 network=udp +50:44.798 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13121 network=tcp +50:45.298 [INFO] TestAgent: Waiting for endpoints to shut down +50:45.298 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand (0.78s) +2022-12-04T05:50:44.570Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:44.600Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:44.604 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aeee8def-3fcb-c933-d7e9-efe7c90e63b3 Address:127.0.0.1:13118}]" +50:44.604 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aeee8def-3fcb-c933-d7e9-efe7c90e63b3.dc1 127.0.0.1 +50:44.605 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aeee8def-3fcb-c933-d7e9-efe7c90e63b3 127.0.0.1 +2022-12-04T05:50:44.605Z [INFO] agent.router: Initializing LAN area manager +50:44.605 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13113 network=udp +50:44.605 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13118 [Follower]" leader= +50:44.608 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aeee8def-3fcb-c933-d7e9-efe7c90e63b3.dc1 area=wan +50:44.608 [INFO] TestAgent.server: Adding LAN server: server="Node-aeee8def-3fcb-c933-d7e9-efe7c90e63b3 (Addr: tcp/127.0.0.1:13118) (DC: dc1)" +50:44.608 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13113 network=tcp +50:44.609 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13114 network=tcp +50:44.609 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13119 network=tcp +50:44.609 [INFO] TestAgent: started state syncer +50:44.643 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:44.643 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13118 [Candidate]" term=2 +50:44.643 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:44.643 [DEBUG] TestAgent.server.raft: vote granted: from=aeee8def-3fcb-c933-d7e9-efe7c90e63b3 term=2 tally=1 +50:44.643 [INFO] TestAgent.server.raft: election won: tally=1 +50:44.643 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13118 [Leader]" +50:44.644 [INFO] TestAgent.server: cluster leadership acquired +50:44.644 [INFO] TestAgent.server: New leader elected: payload=Node-aeee8def-3fcb-c933-d7e9-efe7c90e63b3 +50:44.644 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13118 +50:44.645 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:44.667 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:44.667 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:44.667 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:44.667 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:44.667 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:44.667 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aeee8def-3fcb-c933-d7e9-efe7c90e63b3 +50:44.667 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aeee8def-3fcb-c933-d7e9-efe7c90e63b3 +50:44.668 [INFO] TestAgent.server: federation state anti-entropy synced +50:44.839 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:44.840 [INFO] TestAgent: Synced node info +50:44.885 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:48986 latency=938.07µs +50:44.888 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:48986 latency=631.024µs +50:44.888 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/97bba9b7-ef1a-8aae-dec2-d311f4609c38?acquire=97bba9b7-ef1a-8aae-dec2-d311f4609c38&flags=16210313421097356768 from=127.0.0.1:48986 latency=268.469µs +50:44.889 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse=&wait=10000ms from=127.0.0.1:48986 latency=154.83µs +50:44.890 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=0&flags=16210313421097356768 from=127.0.0.1:48986 latency=877.62µs +50:44.891 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&recurse= from=127.0.0.1:48986 latency=161.494µs +50:44.894 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:48988 latency=198.9µs +50:44.895 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=15&flags=16210313421097356768 from=127.0.0.1:48988 latency=386.7µs +50:44.895 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&index=15&recurse= from=127.0.0.1:48986 latency=3.657133ms +50:44.895 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/97bba9b7-ef1a-8aae-dec2-d311f4609c38 from=127.0.0.1:48988 latency=418.66µs +50:44.896 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse= from=127.0.0.1:48988 latency=178.763µs +50:44.896 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/97bba9b7-ef1a-8aae-dec2-d311f4609c38 from=127.0.0.1:48986 latency=401.789µs +50:44.897 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=16 from=127.0.0.1:48988 latency=321.71µs +50:44.897 [INFO] TestAgent: Requesting shutdown +50:44.897 [INFO] TestAgent.server: shutting down server +50:44.897 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:44.897 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:44.897 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:44.897 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:44.897 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:44.897 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:44.897 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:44.897 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:44.897 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:44.897 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:44.897 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:44.897Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:44.897Z [INFO] agent.router.manager: shutting down +50:44.897 [INFO] TestAgent: consul server down +50:44.897 [INFO] TestAgent: shutdown complete +50:44.897 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13113 network=tcp +50:44.897 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13113 network=udp +50:44.897 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13114 network=tcp +50:45.397 [INFO] TestAgent: Waiting for endpoints to shut down +50:45.397 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand_TrySemaphore (0.88s) +2022-12-04T05:50:44.573Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:44.597Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:44.612 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:108c92c2-e0df-23ed-dc7c-1aca0a3e3dcb Address:127.0.0.1:13083}]" +50:44.614 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-108c92c2-e0df-23ed-dc7c-1aca0a3e3dcb.dc1 127.0.0.1 +50:44.614 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13083 [Follower]" leader= +50:44.621 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-108c92c2-e0df-23ed-dc7c-1aca0a3e3dcb 127.0.0.1 +2022-12-04T05:50:44.621Z [INFO] agent.router: Initializing LAN area manager +50:44.622 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13078 network=udp +50:44.622 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13078 network=tcp +50:44.622 [INFO] TestAgent.server: Adding LAN server: server="Node-108c92c2-e0df-23ed-dc7c-1aca0a3e3dcb (Addr: tcp/127.0.0.1:13083) (DC: dc1)" +50:44.622 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-108c92c2-e0df-23ed-dc7c-1aca0a3e3dcb.dc1 area=wan +50:44.623 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13079 network=tcp +50:44.623 [INFO] TestAgent: started state syncer +50:44.623 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13084 network=tcp +50:44.685 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:44.685 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13083 [Candidate]" term=2 +50:44.685 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:44.685 [DEBUG] TestAgent.server.raft: vote granted: from=108c92c2-e0df-23ed-dc7c-1aca0a3e3dcb term=2 tally=1 +50:44.685 [INFO] TestAgent.server.raft: election won: tally=1 +50:44.685 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13083 [Leader]" +50:44.685 [INFO] TestAgent.server: cluster leadership acquired +50:44.685 [INFO] TestAgent.server: New leader elected: payload=Node-108c92c2-e0df-23ed-dc7c-1aca0a3e3dcb +50:44.685 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13083 +50:44.685 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:44.686 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:44.686 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:44.686 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:44.686 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:44.686 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:44.686 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-108c92c2-e0df-23ed-dc7c-1aca0a3e3dcb +50:44.686 [INFO] TestAgent.server: member joined, marking health alive: member=Node-108c92c2-e0df-23ed-dc7c-1aca0a3e3dcb +50:44.687 [INFO] TestAgent.server: federation state anti-entropy synced +50:44.929 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:44.929 [INFO] TestAgent: Synced node info +50:44.929 [DEBUG] TestAgent: Node info in sync +50:44.934 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:44.934 [DEBUG] TestAgent: Node info in sync +50:44.956 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:42360 latency=1.022615ms +50:44.959 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:42360 latency=491.843µs +50:44.959 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/2094bdf4-0ec7-7431-919c-be7f4b108fb3?acquire=2094bdf4-0ec7-7431-919c-be7f4b108fb3&flags=16210313421097356768 from=127.0.0.1:42360 latency=317.069µs +50:44.960 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse=&wait=15000ms from=127.0.0.1:42360 latency=150.52µs +50:44.960 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=0&flags=16210313421097356768 from=127.0.0.1:42360 latency=320.844µs +50:44.961 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&recurse= from=127.0.0.1:42360 latency=234.544µs +50:44.964 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:42362 latency=203.421µs +50:44.965 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=15&flags=16210313421097356768 from=127.0.0.1:42362 latency=393.439µs +50:44.965 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&index=15&recurse= from=127.0.0.1:42360 latency=3.121923ms +50:44.965 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/2094bdf4-0ec7-7431-919c-be7f4b108fb3 from=127.0.0.1:42362 latency=395.909µs +50:44.966 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse= from=127.0.0.1:42362 latency=104.975µs +50:44.966 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/2094bdf4-0ec7-7431-919c-be7f4b108fb3 from=127.0.0.1:42360 latency=394.121µs +50:44.966 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=16 from=127.0.0.1:42362 latency=244.771µs +50:44.966 [INFO] TestAgent: Requesting shutdown +50:44.966 [INFO] TestAgent.server: shutting down server +50:44.966 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:44.966 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:44.966 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:44.966 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:44.966 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:44.967 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:44.967 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:44.967 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:44.967 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:44.967 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:44.967 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:44.967Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:44.967Z [INFO] agent.router.manager: shutting down +50:44.967 [INFO] TestAgent: consul server down +50:44.967 [INFO] TestAgent: shutdown complete +50:44.967 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13078 network=tcp +50:44.967 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13078 network=udp +50:44.967 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13079 network=tcp +50:45.467 [INFO] TestAgent: Waiting for endpoints to shut down +50:45.467 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand_MonitorRetry_Semaphore_Default (0.95s) +2022-12-04T05:50:44.597Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:44.622Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:44.664 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aa6d35a4-d2bf-f72e-0009-fe61fdc2e0f3 Address:127.0.0.1:13097}]" +50:44.664 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aa6d35a4-d2bf-f72e-0009-fe61fdc2e0f3.dc1 127.0.0.1 +50:44.665 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aa6d35a4-d2bf-f72e-0009-fe61fdc2e0f3 127.0.0.1 +2022-12-04T05:50:44.665Z [INFO] agent.router: Initializing LAN area manager +50:44.665 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13097 [Follower]" leader= +50:44.666 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13092 network=udp +50:44.666 [INFO] TestAgent.server: Adding LAN server: server="Node-aa6d35a4-d2bf-f72e-0009-fe61fdc2e0f3 (Addr: tcp/127.0.0.1:13097) (DC: dc1)" +50:44.666 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aa6d35a4-d2bf-f72e-0009-fe61fdc2e0f3.dc1 area=wan +50:44.666 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13092 network=tcp +50:44.667 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13093 network=tcp +50:44.667 [INFO] TestAgent: started state syncer +50:44.667 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13098 network=tcp +50:44.727 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:44.728 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13097 [Candidate]" term=2 +50:44.728 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:44.728 [DEBUG] TestAgent.server.raft: vote granted: from=aa6d35a4-d2bf-f72e-0009-fe61fdc2e0f3 term=2 tally=1 +50:44.728 [INFO] TestAgent.server.raft: election won: tally=1 +50:44.728 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13097 [Leader]" +50:44.728 [INFO] TestAgent.server: cluster leadership acquired +50:44.728 [INFO] TestAgent.server: New leader elected: payload=Node-aa6d35a4-d2bf-f72e-0009-fe61fdc2e0f3 +50:44.728 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13097 +50:44.736 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:44.738 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:44.738 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:44.738 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:44.738 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:44.738 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:44.738 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aa6d35a4-d2bf-f72e-0009-fe61fdc2e0f3 +50:44.738 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aa6d35a4-d2bf-f72e-0009-fe61fdc2e0f3 +50:44.739 [INFO] TestAgent.server: federation state anti-entropy synced +50:44.924 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:44.924 [INFO] TestAgent: Synced node info +50:45.000 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:57024 latency=1.23602ms +50:45.003 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:57024 latency=502.641µs +50:45.004 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/1c61926e-c727-f2c4-9acc-3ed5ba8e3b2e?acquire=1c61926e-c727-f2c4-9acc-3ed5ba8e3b2e&flags=16210313421097356768 from=127.0.0.1:57024 latency=269.897µs +50:45.004 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse=&wait=15000ms from=127.0.0.1:57024 latency=122.001µs +50:45.005 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=0&flags=16210313421097356768 from=127.0.0.1:57024 latency=349.26µs +50:45.006 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&recurse= from=127.0.0.1:57024 latency=185.71µs +50:45.009 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:57026 latency=286.491µs +50:45.010 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?cas=15&flags=16210313421097356768 from=127.0.0.1:57026 latency=387.832µs +50:45.010 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?consistent=&index=15&recurse= from=127.0.0.1:57024 latency=3.368831ms +50:45.010 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/1c61926e-c727-f2c4-9acc-3ed5ba8e3b2e from=127.0.0.1:57026 latency=426.122µs +50:45.011 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix?recurse= from=127.0.0.1:57026 latency=212.747µs +50:45.011 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/1c61926e-c727-f2c4-9acc-3ed5ba8e3b2e from=127.0.0.1:57024 latency=481.824µs +50:45.013 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=16 from=127.0.0.1:57026 latency=538.051µs +50:45.013 [INFO] TestAgent: Requesting shutdown +50:45.013 [INFO] TestAgent.server: shutting down server +50:45.013 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:45.013 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:45.013 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:45.013 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:45.013 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:45.013 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:45.013 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:45.014 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:45.014 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:45.014 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:45.014 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:45.014Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:45.014Z [INFO] agent.router.manager: shutting down +50:45.014 [INFO] TestAgent: consul server down +50:45.014 [INFO] TestAgent: shutdown complete +50:45.014 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13092 network=tcp +50:45.014 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13092 network=udp +50:45.014 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13093 network=tcp +50:45.514 [INFO] TestAgent: Waiting for endpoints to shut down +50:45.514 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand_MonitorRetry_Semaphore_Arg (1.00s) +2022-12-04T05:50:44.579Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:44.599Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:44.609 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cc3b0832-d74d-bb13-e48e-94ad6ae9c53e Address:127.0.0.1:13104}]" +50:44.611 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13104 [Follower]" leader= +50:44.615 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cc3b0832-d74d-bb13-e48e-94ad6ae9c53e.dc1 127.0.0.1 +50:44.621 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cc3b0832-d74d-bb13-e48e-94ad6ae9c53e 127.0.0.1 +2022-12-04T05:50:44.623Z [INFO] agent.router: Initializing LAN area manager +50:44.624 [INFO] TestAgent.server: Adding LAN server: server="Node-cc3b0832-d74d-bb13-e48e-94ad6ae9c53e (Addr: tcp/127.0.0.1:13104) (DC: dc1)" +50:44.624 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cc3b0832-d74d-bb13-e48e-94ad6ae9c53e.dc1 area=wan +50:44.624 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13099 network=udp +50:44.624 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13099 network=tcp +50:44.662 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13100 network=tcp +50:44.662 [INFO] TestAgent: started state syncer +50:44.665 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13105 network=tcp +50:44.666 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:44.666 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13104 [Candidate]" term=2 +50:44.666 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:44.666 [DEBUG] TestAgent.server.raft: vote granted: from=cc3b0832-d74d-bb13-e48e-94ad6ae9c53e term=2 tally=1 +50:44.666 [INFO] TestAgent.server.raft: election won: tally=1 +50:44.666 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13104 [Leader]" +50:44.666 [INFO] TestAgent.server: cluster leadership acquired +50:44.667 [INFO] TestAgent.server: New leader elected: payload=Node-cc3b0832-d74d-bb13-e48e-94ad6ae9c53e +50:44.667 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13104 +50:44.667 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:44.669 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:44.669 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:44.669 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:44.669 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:44.669 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:44.669 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cc3b0832-d74d-bb13-e48e-94ad6ae9c53e +50:44.669 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cc3b0832-d74d-bb13-e48e-94ad6ae9c53e +50:44.670 [INFO] TestAgent.server: federation state anti-entropy synced +50:44.729 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:58250 latency=2.00731ms +50:44.740 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:58250 latency=931.761µs +50:44.740 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:58250 latency=50.042µs +50:44.741 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=3ea4eee6-639e-1a4a-1df2-d4bf0fc04b7f&flags=3304740253564472344 from=127.0.0.1:58250 latency=520.748µs +50:44.743 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:58250 latency=1.281502ms +50:44.746 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=3ea4eee6-639e-1a4a-1df2-d4bf0fc04b7f from=127.0.0.1:58252 latency=568.994µs +50:44.746 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=13 from=127.0.0.1:58250 latency=1.991872ms +50:44.746 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:58252 latency=239.115µs +50:44.747 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/3ea4eee6-639e-1a4a-1df2-d4bf0fc04b7f from=127.0.0.1:58250 latency=423.511µs +50:44.747 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=14 from=127.0.0.1:58252 latency=391.313µs +50:44.750 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:58256 latency=1.065874ms +50:44.752 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:58256 latency=832.885µs +50:44.753 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:58256 latency=46.136µs +50:44.753 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=4b77e62c-efd5-c151-a7ae-262faeb65f3d&flags=3304740253564472344 from=127.0.0.1:58256 latency=352.428µs +50:44.754 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:58256 latency=157.098µs +50:44.793 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=4b77e62c-efd5-c151-a7ae-262faeb65f3d from=127.0.0.1:58258 latency=503.18µs +50:44.793 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=18 from=127.0.0.1:58256 latency=38.547151ms +50:44.794 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:58258 latency=273.149µs +50:44.794 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/4b77e62c-efd5-c151-a7ae-262faeb65f3d from=127.0.0.1:58256 latency=412.097µs +50:44.795 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=19 from=127.0.0.1:58258 latency=357.1µs +50:44.797 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:58260 latency=1.147032ms +50:44.799 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:58260 latency=617.3µs +50:44.800 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:58260 latency=80.868µs +50:44.800 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=13172e2f-7c5d-62f6-c2d4-c909f07ebcc5&flags=3304740253564472344 from=127.0.0.1:58260 latency=386.754µs +50:44.801 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:58260 latency=155.718µs +50:44.805 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=13172e2f-7c5d-62f6-c2d4-c909f07ebcc5 from=127.0.0.1:58264 latency=377.679µs +50:44.805 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=23 from=127.0.0.1:58260 latency=3.729775ms +50:44.806 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:58264 latency=92.838µs +50:44.806 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/13172e2f-7c5d-62f6-c2d4-c909f07ebcc5 from=127.0.0.1:58260 latency=598.231µs +50:44.807 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=24 from=127.0.0.1:58264 latency=615.992µs +50:44.812 [INFO] TestAgent: Requesting shutdown +50:44.812 [INFO] TestAgent.server: shutting down server +50:44.812 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:44.812 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:44.812 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:44.812 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:44.812 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:44.812 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:44.812 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:44.812 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:44.812 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:44.812 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:44.812 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:44.812 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:44.812Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:44.813Z [INFO] agent.router.manager: shutting down +50:44.813 [INFO] TestAgent: consul server down +50:44.813 [INFO] TestAgent: shutdown complete +50:44.813 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13099 network=tcp +50:44.813 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13099 network=udp +50:44.813 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13100 network=tcp +50:45.813 [WARN] TestAgent: Timeout stopping server: protocol=HTTP address=127.0.0.1:13100 network=tcp +50:45.813 [INFO] TestAgent: Waiting for endpoints to shut down +50:45.813 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand_ChildExitCode (1.29s) + --- PASS: TestLockCommand_ChildExitCode/clean_exit (0.02s) + --- PASS: TestLockCommand_ChildExitCode/error_exit (0.05s) + --- PASS: TestLockCommand_ChildExitCode/not_propagated (0.02s) +2022-12-04T05:50:44.594Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:44.661Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:44.664 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:c4f9016d-c596-510b-878f-c96bd6ae32ce Address:127.0.0.1:13069}]" +50:44.665 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-c4f9016d-c596-510b-878f-c96bd6ae32ce.dc1 127.0.0.1 +50:44.665 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= +50:44.665 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-c4f9016d-c596-510b-878f-c96bd6ae32ce 127.0.0.1 +2022-12-04T05:50:44.666Z [INFO] agent.router: Initializing LAN area manager +50:44.666 [INFO] TestAgent.server: Adding LAN server: server="Node-c4f9016d-c596-510b-878f-c96bd6ae32ce (Addr: tcp/127.0.0.1:13069) (DC: dc1)" +50:44.666 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp +50:44.666 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-c4f9016d-c596-510b-878f-c96bd6ae32ce.dc1 area=wan +50:44.666 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp +50:44.666 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp +50:44.666 [INFO] TestAgent: started state syncer +50:44.666 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp +50:44.717 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:44.717 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 +50:44.717 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:44.717 [DEBUG] TestAgent.server.raft: vote granted: from=c4f9016d-c596-510b-878f-c96bd6ae32ce term=2 tally=1 +50:44.717 [INFO] TestAgent.server.raft: election won: tally=1 +50:44.717 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" +50:44.717 [INFO] TestAgent.server: cluster leadership acquired +50:44.717 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13069 +50:44.717 [INFO] TestAgent.server: New leader elected: payload=Node-c4f9016d-c596-510b-878f-c96bd6ae32ce +50:44.718 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:44.719 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:44.719 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:44.719 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:44.719 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:44.719 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:44.719 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-c4f9016d-c596-510b-878f-c96bd6ae32ce +50:44.719 [INFO] TestAgent.server: member joined, marking health alive: member=Node-c4f9016d-c596-510b-878f-c96bd6ae32ce +50:44.720 [INFO] TestAgent.server: federation state anti-entropy synced +50:44.909 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:54432 latency=1.039358ms +50:44.911 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:54432 latency=611.009µs +50:44.911 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:54432 latency=41.99µs +50:44.912 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=3378ce48-dee7-cbdd-5e00-41dd56050430&flags=3304740253564472344 from=127.0.0.1:54432 latency=425.855µs +50:44.912 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:54432 latency=178.134µs +50:44.916 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=3378ce48-dee7-cbdd-5e00-41dd56050430 from=127.0.0.1:54434 latency=402.961µs +50:44.916 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=13 from=127.0.0.1:54432 latency=2.757269ms +50:44.916 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:54434 latency=234.826µs +50:44.917 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/3378ce48-dee7-cbdd-5e00-41dd56050430 from=127.0.0.1:54432 latency=408.105µs +50:44.918 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=14 from=127.0.0.1:54434 latency=1.24999ms +50:44.918 [INFO] TestAgent: Requesting shutdown +50:44.918 [INFO] TestAgent.server: shutting down server +50:44.918 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:44.918 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:44.918 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:44.918 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:44.918 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:44.918 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:44.918 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:44.918 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:44.918 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:44.918 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:44.918 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:44.918 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:44.918Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:44.918Z [INFO] agent.router.manager: shutting down +50:44.919 [INFO] TestAgent: consul server down +50:44.919 [INFO] TestAgent: shutdown complete +50:44.919 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp +50:44.919 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp +50:44.919 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp +50:45.919 [WARN] TestAgent: Timeout stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp +50:45.919 [INFO] TestAgent: Waiting for endpoints to shut down +50:45.919 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand_MonitorRetry_Lock_Default (1.40s) +2022-12-04T05:50:44.570Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:44.606Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:44.608 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d8ceabab-d5f0-26a0-d261-3aa71698322e Address:127.0.0.1:13111}]" +50:44.608 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13111 [Follower]" leader= +50:44.609 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d8ceabab-d5f0-26a0-d261-3aa71698322e.dc1 127.0.0.1 +50:44.615 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d8ceabab-d5f0-26a0-d261-3aa71698322e 127.0.0.1 +2022-12-04T05:50:44.617Z [INFO] agent.router: Initializing LAN area manager +50:44.619 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d8ceabab-d5f0-26a0-d261-3aa71698322e.dc1 area=wan +50:44.619 [INFO] TestAgent.server: Adding LAN server: server="Node-d8ceabab-d5f0-26a0-d261-3aa71698322e (Addr: tcp/127.0.0.1:13111) (DC: dc1)" +50:44.620 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13106 network=udp +50:44.621 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13106 network=tcp +50:44.621 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13107 network=tcp +50:44.621 [INFO] TestAgent: started state syncer +50:44.621 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13112 network=tcp +50:44.678 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:44.678 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13111 [Candidate]" term=2 +50:44.678 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:44.678 [DEBUG] TestAgent.server.raft: vote granted: from=d8ceabab-d5f0-26a0-d261-3aa71698322e term=2 tally=1 +50:44.678 [INFO] TestAgent.server.raft: election won: tally=1 +50:44.678 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13111 [Leader]" +50:44.678 [INFO] TestAgent.server: cluster leadership acquired +50:44.678 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13111 +50:44.679 [INFO] TestAgent.server: New leader elected: payload=Node-d8ceabab-d5f0-26a0-d261-3aa71698322e +50:44.679 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:44.680 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:44.680 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:44.680 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:44.680 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:44.680 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:44.680 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d8ceabab-d5f0-26a0-d261-3aa71698322e +50:44.680 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d8ceabab-d5f0-26a0-d261-3aa71698322e +50:44.681 [INFO] TestAgent.server: federation state anti-entropy synced +50:44.692 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:44.692 [INFO] TestAgent: Synced node info +50:44.692 [DEBUG] TestAgent: Node info in sync +50:44.950 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:60728 latency=1.201167ms +50:44.953 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:60728 latency=561.741µs +50:44.953 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:60728 latency=42.212µs +50:44.953 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=85960ec8-e2d0-80f6-9072-61c78bb01107&flags=3304740253564472344 from=127.0.0.1:60728 latency=275.382µs +50:44.954 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:60728 latency=293.039µs +50:44.957 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=85960ec8-e2d0-80f6-9072-61c78bb01107 from=127.0.0.1:60730 latency=412.258µs +50:44.958 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:60728 latency=2.620403ms +50:44.958 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:60730 latency=128.832µs +50:44.958 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/85960ec8-e2d0-80f6-9072-61c78bb01107 from=127.0.0.1:60728 latency=345.61µs +50:44.958 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:60730 latency=252.932µs +50:44.959 [INFO] TestAgent: Requesting shutdown +50:44.959 [INFO] TestAgent.server: shutting down server +50:44.959 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:44.959 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:44.959 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:44.959 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:44.959 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:44.959 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:44.959 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:44.959 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:44.959 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:44.959 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:44.959 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:44.959Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:44.959Z [INFO] agent.router.manager: shutting down +50:44.959 [INFO] TestAgent: consul server down +50:44.960 [INFO] TestAgent: shutdown complete +50:44.960 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13106 network=tcp +50:44.960 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13106 network=udp +50:44.960 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13107 network=tcp +50:45.960 [WARN] TestAgent: Timeout stopping server: protocol=HTTP address=127.0.0.1:13107 network=tcp +50:45.960 [INFO] TestAgent: Waiting for endpoints to shut down +50:45.960 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand_NoShell (1.44s) +2022-12-04T05:50:44.588Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:44.623Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:44.659 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:efc2cea5-04ac-b250-eece-d7de84ed414d Address:127.0.0.1:13076}]" +50:44.659 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-efc2cea5-04ac-b250-eece-d7de84ed414d.dc1 127.0.0.1 +50:44.660 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-efc2cea5-04ac-b250-eece-d7de84ed414d 127.0.0.1 +2022-12-04T05:50:44.660Z [INFO] agent.router: Initializing LAN area manager +50:44.660 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=udp +50:44.661 [INFO] TestAgent.server: Adding LAN server: server="Node-efc2cea5-04ac-b250-eece-d7de84ed414d (Addr: tcp/127.0.0.1:13076) (DC: dc1)" +50:44.661 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=tcp +50:44.662 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13076 [Follower]" leader= +50:44.662 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-efc2cea5-04ac-b250-eece-d7de84ed414d.dc1 area=wan +50:44.665 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13072 network=tcp +50:44.665 [INFO] TestAgent: started state syncer +50:44.665 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13077 network=tcp +50:44.706 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:44.706 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13076 [Candidate]" term=2 +50:44.706 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:44.706 [DEBUG] TestAgent.server.raft: vote granted: from=efc2cea5-04ac-b250-eece-d7de84ed414d term=2 tally=1 +50:44.706 [INFO] TestAgent.server.raft: election won: tally=1 +50:44.706 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13076 [Leader]" +50:44.706 [INFO] TestAgent.server: cluster leadership acquired +50:44.707 [INFO] TestAgent.server: New leader elected: payload=Node-efc2cea5-04ac-b250-eece-d7de84ed414d +50:44.707 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13076 +50:44.707 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:44.709 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:44.709 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:44.709 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:44.709 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:44.709 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:44.709 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-efc2cea5-04ac-b250-eece-d7de84ed414d +50:44.709 [INFO] TestAgent.server: member joined, marking health alive: member=Node-efc2cea5-04ac-b250-eece-d7de84ed414d +50:44.709 [INFO] TestAgent.server: federation state anti-entropy synced +50:44.919 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:44.919 [INFO] TestAgent: Synced node info +50:45.004 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:59722 latency=1.160619ms +50:45.006 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:59722 latency=404.971µs +50:45.006 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=10000ms from=127.0.0.1:59722 latency=50.627µs +50:45.007 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=e11af45c-275e-5df9-1504-dce9370ef809&flags=3304740253564472344 from=127.0.0.1:59722 latency=473.114µs +50:45.008 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:59722 latency=216.207µs +50:45.011 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=e11af45c-275e-5df9-1504-dce9370ef809 from=127.0.0.1:59726 latency=549.391µs +50:45.011 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:59722 latency=2.120794ms +50:45.012 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:59726 latency=265.717µs +50:45.016 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:59726 latency=1.832971ms +50:45.017 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/e11af45c-275e-5df9-1504-dce9370ef809 from=127.0.0.1:59722 latency=2.081219ms +50:45.017 [INFO] TestAgent: Requesting shutdown +50:45.017 [INFO] TestAgent.server: shutting down server +50:45.017 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:45.017 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:45.017 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:45.017 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:45.017 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:45.017 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:45.017 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:45.017 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:45.017 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:45.017 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:45.017 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:45.017Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:45.017Z [INFO] agent.router.manager: shutting down +50:45.018 [INFO] TestAgent: consul server down +50:45.018 [INFO] TestAgent: shutdown complete +50:45.018 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=tcp +50:45.018 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=udp +50:45.018 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13072 network=tcp +50:46.018 [WARN] TestAgent: Timeout stopping server: protocol=HTTP address=127.0.0.1:13072 network=tcp +50:46.018 [INFO] TestAgent: Waiting for endpoints to shut down +50:46.018 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand_TryLock (1.50s) +2022-12-04T05:50:44.577Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:44.617Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:44.622 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f57babf3-27fa-ef6e-1172-eb2091a52304 Address:127.0.0.1:13090}]" +50:44.622 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13090 [Follower]" leader= +50:44.623 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f57babf3-27fa-ef6e-1172-eb2091a52304.dc1 127.0.0.1 +50:44.658 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f57babf3-27fa-ef6e-1172-eb2091a52304 127.0.0.1 +2022-12-04T05:50:44.658Z [INFO] agent.router: Initializing LAN area manager +50:44.658 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13085 network=udp +50:44.659 [INFO] TestAgent.server: Adding LAN server: server="Node-f57babf3-27fa-ef6e-1172-eb2091a52304 (Addr: tcp/127.0.0.1:13090) (DC: dc1)" +50:44.659 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f57babf3-27fa-ef6e-1172-eb2091a52304.dc1 area=wan +50:44.659 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13085 network=tcp +50:44.660 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13086 network=tcp +50:44.660 [INFO] TestAgent: started state syncer +50:44.660 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13091 network=tcp +50:44.676 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:44.676 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13090 [Candidate]" term=2 +50:44.676 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:44.676 [DEBUG] TestAgent.server.raft: vote granted: from=f57babf3-27fa-ef6e-1172-eb2091a52304 term=2 tally=1 +50:44.676 [INFO] TestAgent.server.raft: election won: tally=1 +50:44.676 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13090 [Leader]" +50:44.676 [INFO] TestAgent.server: cluster leadership acquired +50:44.676 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13090 +50:44.676 [INFO] TestAgent.server: New leader elected: payload=Node-f57babf3-27fa-ef6e-1172-eb2091a52304 +50:44.677 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:44.678 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:44.678 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:44.678 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:44.678 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:44.678 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:44.678 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f57babf3-27fa-ef6e-1172-eb2091a52304 +50:44.678 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f57babf3-27fa-ef6e-1172-eb2091a52304 +50:44.679 [INFO] TestAgent.server: federation state anti-entropy synced +50:44.888 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:44.888 [INFO] TestAgent: Synced node info +50:44.888 [DEBUG] TestAgent: Node info in sync +50:45.065 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:53822 latency=1.800935ms +50:45.068 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/create from=127.0.0.1:53822 latency=895.397µs +50:45.068 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?wait=15000ms from=127.0.0.1:53822 latency=46.033µs +50:45.069 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?acquire=97e6a51f-8e34-2be6-6351-8247d822730b&flags=3304740253564472344 from=127.0.0.1:53822 latency=304.042µs +50:45.070 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent= from=127.0.0.1:53822 latency=262.521µs +50:45.073 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/test/prefix/.lock?flags=3304740253564472344&release=97e6a51f-8e34-2be6-6351-8247d822730b from=127.0.0.1:53824 latency=615.419µs +50:45.073 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock?consistent=&index=14 from=127.0.0.1:53822 latency=2.682046ms +50:45.073 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/kv/test/prefix/.lock from=127.0.0.1:53824 latency=186.16µs +50:45.074 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/session/destroy/97e6a51f-8e34-2be6-6351-8247d822730b from=127.0.0.1:53822 latency=355.014µs +50:45.074 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/kv/test/prefix/.lock?cas=15 from=127.0.0.1:53822 latency=339.79µs +50:45.075 [INFO] TestAgent: Requesting shutdown +50:45.075 [INFO] TestAgent.server: shutting down server +50:45.075 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:45.075 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:45.075 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:45.075 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:45.075 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:45.075 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:45.075 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:45.075 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:45.075 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:45.075 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:45.075 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:45.075Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:45.075Z [INFO] agent.router.manager: shutting down +50:45.075 [INFO] TestAgent: consul server down +50:45.075 [INFO] TestAgent: shutdown complete +50:45.075 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13085 network=tcp +50:45.075 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13085 network=udp +50:45.075 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13086 network=tcp +50:46.075 [WARN] TestAgent: Timeout stopping server: protocol=HTTP address=127.0.0.1:13086 network=tcp +50:46.075 [INFO] TestAgent: Waiting for endpoints to shut down +50:46.075 [INFO] TestAgent: Endpoints down +--- PASS: TestLockCommand_MonitorRetry_Lock_Arg (1.56s) PASS -ok github.com/hashicorp/consul/command/lock 2.092s +ok github.com/hashicorp/consul/command/lock 1.605s === RUN TestLoginCommand_noTabs === PAUSE TestLoginCommand_noTabs === RUN TestLoginCommand @@ -88608,14 +88768,13 @@ === RUN TestLoginCommand_jwt === PAUSE TestLoginCommand_jwt === CONT TestLoginCommand_noTabs ---- PASS: TestLoginCommand_noTabs (0.00s) === CONT TestLoginCommand_jwt -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestLoginCommand_k8s === CONT TestLoginCommand +=== CONT TestLoginCommand_k8s +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestLoginCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -=== RUN TestLoginCommand_k8s/try_login_with_method_configured_and_binding_rules === RUN TestLoginCommand/method_is_required === RUN TestLoginCommand/token-sink-file_is_required === RUN TestLoginCommand/bearer-token-file_is_required @@ -88623,259 +88782,259 @@ === RUN TestLoginCommand/try_login_with_no_method_configured === RUN TestLoginCommand/try_login_with_method_configured_but_no_binding_rules === RUN TestLoginCommand/try_login_with_method_configured_and_binding_rules +=== RUN TestLoginCommand_jwt/success_-_jwt_static_keys === RUN TestLoginCommand_jwt/success_-_jwt_jwks === RUN TestLoginCommand_jwt/success_-_jwt_oidc_discovery -=== RUN TestLoginCommand_jwt/success_-_jwt_static_keys -2021-10-31T23:18:13.298Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:13.357Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:13.359 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:af91ed8f-38bb-73b6-099c-7152f049630a Address:127.0.0.1:22265}]" -18:13.359 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-af91ed8f-38bb-73b6-099c-7152f049630a.dc1 127.0.0.1 -18:13.360 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-af91ed8f-38bb-73b6-099c-7152f049630a 127.0.0.1 -2021-10-31T23:18:13.360Z [INFO] agent.router: Initializing LAN area manager -18:13.360 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=udp -18:13.360 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22265 [Follower]" leader= -18:13.361 [INFO] TestAgent.server: Adding LAN server: server="Node-af91ed8f-38bb-73b6-099c-7152f049630a (Addr: tcp/127.0.0.1:22265) (DC: dc1)" -18:13.361 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-af91ed8f-38bb-73b6-099c-7152f049630a.dc1 area=wan -18:13.361 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22260 network=tcp -18:13.366 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22261 network=tcp -18:13.366 [INFO] TestAgent: started state syncer -18:13.366 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22266 network=tcp -18:13.412 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -18:13.417 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:13.417 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22265 [Candidate]" term=2 -18:13.418 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:13.418 [DEBUG] TestAgent.server.raft: vote granted: from=af91ed8f-38bb-73b6-099c-7152f049630a term=2 tally=1 -18:13.418 [INFO] TestAgent.server.raft: election won: tally=1 -18:13.418 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22265 [Leader]" -18:13.418 [INFO] TestAgent.server: cluster leadership acquired -18:13.418 [INFO] TestAgent.server: initializing acls -18:13.418 [INFO] TestAgent.server: Created ACL 'global-management' policy -18:13.418 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -18:13.419 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -18:13.419 [INFO] TestAgent.server: Created ACL anonymous token from configuration -18:13.419 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -18:13.419 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -18:13.420 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-af91ed8f-38bb-73b6-099c-7152f049630a -18:13.420 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-af91ed8f-38bb-73b6-099c-7152f049630a.dc1 -18:13.421 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:13.422 [INFO] TestAgent.server: New leader elected: payload=Node-af91ed8f-38bb-73b6-099c-7152f049630a -18:13.422 [INFO] TestAgent.server: Updating LAN server: server="Node-af91ed8f-38bb-73b6-099c-7152f049630a (Addr: tcp/127.0.0.1:22265) (DC: dc1)" -18:13.422 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-af91ed8f-38bb-73b6-099c-7152f049630a.dc1 area=wan -18:13.430 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:13.431 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:13.431 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:13.431 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:13.431 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:13.431 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-af91ed8f-38bb-73b6-099c-7152f049630a -18:13.431 [INFO] TestAgent.server: member joined, marking health alive: member=Node-af91ed8f-38bb-73b6-099c-7152f049630a -18:13.431 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-af91ed8f-38bb-73b6-099c-7152f049630a -18:13.432 [INFO] TestAgent.server: federation state anti-entropy synced -18:13.474 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:60786 latency=4.164149ms -18:13.484 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:60786 latency=2.365358ms -18:13.508 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:60788 latency=14.510517ms -18:13.509 [INFO] TestAgent: Requesting shutdown -18:13.509 [INFO] TestAgent.server: shutting down server -18:13.509 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -18:13.509 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:13.509 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:13.509 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:13.509 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:13.509 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -18:13.509 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:13.509 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:13.509 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:13.509 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -18:13.509 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -18:13.509 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:13.509 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:13.510 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:13.510 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -18:13.510 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:13.510Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:13.510Z [INFO] agent.router.manager: shutting down -18:13.510 [INFO] TestAgent: consul server down -18:13.510 [INFO] TestAgent: shutdown complete -18:13.510 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=tcp -18:13.510 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22260 network=udp -18:13.510 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22261 network=tcp -18:14.011 [INFO] TestAgent: Waiting for endpoints to shut down -18:14.011 [INFO] TestAgent: Endpoints down ---- PASS: TestLoginCommand_k8s (0.87s) - --- PASS: TestLoginCommand_k8s/try_login_with_method_configured_and_binding_rules (0.02s) -2021-10-31T23:18:13.277Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:13.365Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:13.371 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:93ec7218-d29b-d9a1-90f5-d2e458b102a7 Address:127.0.0.1:22272}]" -18:13.372 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-93ec7218-d29b-d9a1-90f5-d2e458b102a7.dc1 127.0.0.1 -18:13.372 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-93ec7218-d29b-d9a1-90f5-d2e458b102a7 127.0.0.1 -2021-10-31T23:18:13.372Z [INFO] agent.router: Initializing LAN area manager -18:13.372 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22267 network=udp -18:13.372 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22272 [Follower]" leader= -18:13.373 [INFO] TestAgent.server: Adding LAN server: server="Node-93ec7218-d29b-d9a1-90f5-d2e458b102a7 (Addr: tcp/127.0.0.1:22272) (DC: dc1)" -18:13.373 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-93ec7218-d29b-d9a1-90f5-d2e458b102a7.dc1 area=wan -18:13.373 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22267 network=tcp -18:13.377 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22268 network=tcp -18:13.378 [INFO] TestAgent: started state syncer -18:13.378 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22273 network=tcp -18:13.426 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -18:13.441 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:13.441 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22272 [Candidate]" term=2 -18:13.442 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:13.442 [DEBUG] TestAgent.server.raft: vote granted: from=93ec7218-d29b-d9a1-90f5-d2e458b102a7 term=2 tally=1 -18:13.442 [INFO] TestAgent.server.raft: election won: tally=1 -18:13.442 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22272 [Leader]" -18:13.442 [INFO] TestAgent.server: cluster leadership acquired -18:13.442 [INFO] TestAgent.server: initializing acls -18:13.442 [INFO] TestAgent.server: Created ACL 'global-management' policy -18:13.442 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -18:13.443 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -18:13.443 [INFO] TestAgent.server: Created ACL anonymous token from configuration -18:13.443 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -18:13.443 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -18:13.443 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-93ec7218-d29b-d9a1-90f5-d2e458b102a7 -18:13.443 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-93ec7218-d29b-d9a1-90f5-d2e458b102a7.dc1 -18:13.444 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:13.445 [INFO] TestAgent.server: New leader elected: payload=Node-93ec7218-d29b-d9a1-90f5-d2e458b102a7 -18:13.446 [INFO] TestAgent.server: Updating LAN server: server="Node-93ec7218-d29b-d9a1-90f5-d2e458b102a7 (Addr: tcp/127.0.0.1:22272) (DC: dc1)" -18:13.446 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-93ec7218-d29b-d9a1-90f5-d2e458b102a7.dc1 area=wan -18:13.446 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:13.446 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:13.446 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:13.446 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:13.446 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:13.446 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-93ec7218-d29b-d9a1-90f5-d2e458b102a7 -18:13.446 [INFO] TestAgent.server: member joined, marking health alive: member=Node-93ec7218-d29b-d9a1-90f5-d2e458b102a7 -18:13.447 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-93ec7218-d29b-d9a1-90f5-d2e458b102a7 -18:13.447 [INFO] TestAgent.server: federation state anti-entropy synced -18:13.578 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/login from=127.0.0.1:42296 error="ACL not found" -18:13.587 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:42296 latency=9.438962ms -18:13.597 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:42298 latency=1.427057ms -18:13.599 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/login from=127.0.0.1:42300 error="Permission denied" -18:13.600 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:42300 latency=1.376226ms -18:13.615 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:42298 latency=1.174586ms -18:13.618 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:42302 latency=1.204296ms -18:13.618 [INFO] TestAgent: Requesting shutdown -18:13.618 [INFO] TestAgent.server: shutting down server -18:13.618 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:13.618 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -18:13.618 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -18:13.618 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:13.618 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:13.618 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:13.618 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:13.619 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:13.619 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:13.619 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -18:13.619 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:13.619 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -18:13.619 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -18:13.619 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:13.619 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:13.619 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:13.619Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:13.619Z [INFO] agent.router.manager: shutting down -18:13.619 [INFO] TestAgent: consul server down -18:13.619 [INFO] TestAgent: shutdown complete -18:13.619 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22267 network=tcp -18:13.619 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22267 network=udp -18:13.619 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22268 network=tcp -18:14.121 [INFO] TestAgent: Waiting for endpoints to shut down -18:14.121 [INFO] TestAgent: Endpoints down ---- PASS: TestLoginCommand (0.98s) +=== RUN TestLoginCommand_k8s/try_login_with_method_configured_and_binding_rules +2022-12-04T05:50:47.877Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:47.901Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:47.903 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:eb90171f-3579-d54c-5341-d44e1d97050f Address:127.0.0.1:27363}]" +50:47.903 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= +50:47.903 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-eb90171f-3579-d54c-5341-d44e1d97050f.dc1 127.0.0.1 +50:47.908 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-eb90171f-3579-d54c-5341-d44e1d97050f 127.0.0.1 +2022-12-04T05:50:47.908Z [INFO] agent.router: Initializing LAN area manager +50:47.909 [INFO] TestAgent.server: Adding LAN server: server="Node-eb90171f-3579-d54c-5341-d44e1d97050f (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +50:47.909 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-eb90171f-3579-d54c-5341-d44e1d97050f.dc1 area=wan +50:47.909 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp +50:47.909 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp +50:47.909 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp +50:47.909 [INFO] TestAgent: started state syncer +50:47.909 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp +50:47.959 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +50:47.973 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:47.973 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 +50:47.973 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:47.973 [DEBUG] TestAgent.server.raft: vote granted: from=eb90171f-3579-d54c-5341-d44e1d97050f term=2 tally=1 +50:47.973 [INFO] TestAgent.server.raft: election won: tally=1 +50:47.973 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" +50:47.973 [INFO] TestAgent.server: cluster leadership acquired +50:47.973 [INFO] TestAgent.server: initializing acls +50:47.973 [INFO] TestAgent.server: New leader elected: payload=Node-eb90171f-3579-d54c-5341-d44e1d97050f +50:47.973 [INFO] TestAgent.server: Created ACL 'global-management' policy +50:47.973 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +50:47.974 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +50:47.974 [INFO] TestAgent.server: Created ACL anonymous token from configuration +50:47.974 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +50:47.974 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +50:47.974 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-eb90171f-3579-d54c-5341-d44e1d97050f +50:47.974 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-eb90171f-3579-d54c-5341-d44e1d97050f.dc1 +50:47.975 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:47.976 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:47.976 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:47.976 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:47.976 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:47.976 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:47.976 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eb90171f-3579-d54c-5341-d44e1d97050f +50:47.976 [INFO] TestAgent.server: member joined, marking health alive: member=Node-eb90171f-3579-d54c-5341-d44e1d97050f +50:47.976 [INFO] TestAgent.server: Updating LAN server: server="Node-eb90171f-3579-d54c-5341-d44e1d97050f (Addr: tcp/127.0.0.1:27363) (DC: dc1)" +50:47.976 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-eb90171f-3579-d54c-5341-d44e1d97050f.dc1 area=wan +50:47.977 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-eb90171f-3579-d54c-5341-d44e1d97050f +50:47.977 [INFO] TestAgent.server: federation state anti-entropy synced +50:48.005 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/login from=127.0.0.1:59696 error="ACL not found" +50:48.005 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:59696 latency=309.395µs +50:48.007 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:59698 latency=968.562µs +50:48.009 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/login from=127.0.0.1:59700 error="Permission denied" +50:48.009 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:59700 latency=416.76µs +50:48.011 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:59698 latency=1.36276ms +50:48.014 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:59704 latency=2.178034ms +50:48.020 [INFO] TestAgent: Requesting shutdown +50:48.020 [INFO] TestAgent.server: shutting down server +50:48.020 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:48.020 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:48.020 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +50:48.020 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +50:48.020 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:48.020 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:48.020 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:48.020 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:48.020 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:48.020 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:48.020 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +50:48.020 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:48.020 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:48.020 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +50:48.020 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:48.020 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:48.020Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:48.020Z [INFO] agent.router.manager: shutting down +50:48.021 [INFO] TestAgent: consul server down +50:48.021 [INFO] TestAgent: shutdown complete +50:48.021 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp +50:48.021 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp +50:48.021 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp +50:48.521 [INFO] TestAgent: Waiting for endpoints to shut down +50:48.521 [INFO] TestAgent: Endpoints down +--- PASS: TestLoginCommand (0.69s) --- PASS: TestLoginCommand/method_is_required (0.00s) --- PASS: TestLoginCommand/token-sink-file_is_required (0.00s) --- PASS: TestLoginCommand/bearer-token-file_is_required (0.00s) --- PASS: TestLoginCommand/bearer-token-file_is_empty (0.00s) - --- PASS: TestLoginCommand/try_login_with_no_method_configured (0.02s) - --- PASS: TestLoginCommand/try_login_with_method_configured_but_no_binding_rules (0.01s) - --- PASS: TestLoginCommand/try_login_with_method_configured_and_binding_rules (0.00s) -2021-10-31T23:18:13.242Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:13.307Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:13.333 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:50680198-3f33-005b-0783-051fb6d52a2a Address:127.0.0.1:22258}]" -18:13.333 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:22258 [Follower]" leader= -18:13.334 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-50680198-3f33-005b-0783-051fb6d52a2a.dc1 127.0.0.1 -18:13.335 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-50680198-3f33-005b-0783-051fb6d52a2a 127.0.0.1 -2021-10-31T23:18:13.335Z [INFO] agent.router: Initializing LAN area manager -18:13.336 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=udp -18:13.359 [INFO] TestAgent.server: Adding LAN server: server="Node-50680198-3f33-005b-0783-051fb6d52a2a (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -18:13.359 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-50680198-3f33-005b-0783-051fb6d52a2a.dc1 area=wan -18:13.359 [INFO] TestAgent: Started DNS server: address=127.0.0.1:22253 network=tcp -18:13.360 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:22254 network=tcp -18:13.360 [INFO] TestAgent: started state syncer -18:13.360 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:22259 network=tcp -18:13.401 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:13.401 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:22258 [Candidate]" term=2 -18:13.402 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:13.402 [DEBUG] TestAgent.server.raft: vote granted: from=50680198-3f33-005b-0783-051fb6d52a2a term=2 tally=1 -18:13.402 [INFO] TestAgent.server.raft: election won: tally=1 -18:13.402 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:22258 [Leader]" -18:13.402 [INFO] TestAgent.server: cluster leadership acquired -18:13.402 [INFO] TestAgent.server: initializing acls -18:13.402 [INFO] TestAgent.server: Created ACL 'global-management' policy -18:13.402 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -18:13.403 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -18:13.403 [INFO] TestAgent.server: Created ACL anonymous token from configuration -18:13.403 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -18:13.403 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -18:13.403 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-50680198-3f33-005b-0783-051fb6d52a2a -18:13.403 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-50680198-3f33-005b-0783-051fb6d52a2a.dc1 -18:13.416 [INFO] TestAgent.server: New leader elected: payload=Node-50680198-3f33-005b-0783-051fb6d52a2a -18:13.416 [INFO] TestAgent.server: Updating LAN server: server="Node-50680198-3f33-005b-0783-051fb6d52a2a (Addr: tcp/127.0.0.1:22258) (DC: dc1)" -18:13.416 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-50680198-3f33-005b-0783-051fb6d52a2a.dc1 area=wan -18:13.416 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:13.422 [INFO] TestAgent: Synced node info -18:13.427 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:13.427 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:13.427 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:13.427 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:13.427 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:13.427 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-50680198-3f33-005b-0783-051fb6d52a2a -18:13.427 [INFO] TestAgent.server: member joined, marking health alive: member=Node-50680198-3f33-005b-0783-051fb6d52a2a -18:13.428 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-50680198-3f33-005b-0783-051fb6d52a2a -18:13.429 [INFO] TestAgent.server: federation state anti-entropy synced -18:13.474 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:13.474 [DEBUG] TestAgent: Node info in sync -18:13.474 [DEBUG] TestAgent: Node info in sync -18:13.780 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:50356 latency=8.783011ms -18:13.793 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50356 latency=12.550263ms -18:13.808 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:50362 latency=10.173783ms -18:13.830 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:50356 latency=20.728409ms -18:13.853 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50356 latency=18.777595ms -18:13.864 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:50374 latency=6.593766ms -18:13.871 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:50356 latency=877.459µs -18:13.873 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50356 latency=1.278817ms -18:13.899 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:50376 latency=21.211942ms -18:13.901 [INFO] TestAgent: Requesting shutdown -18:13.901 [INFO] TestAgent.server: shutting down server -18:13.901 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:13.901 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -18:13.901 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -18:13.901 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:13.901 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:13.901 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:13.901 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:13.901 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:13.901 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:13.901 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:13.901 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -18:13.901 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -18:13.901 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:13.901 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:13.901 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:13.901Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:13.902Z [INFO] agent.router.manager: shutting down -18:13.902 [INFO] TestAgent: consul server down -18:13.902 [INFO] TestAgent: shutdown complete -18:13.902 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=tcp -18:13.902 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:22253 network=udp -18:13.902 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:22254 network=tcp -18:14.405 [INFO] TestAgent: Waiting for endpoints to shut down -18:14.405 [INFO] TestAgent: Endpoints down ---- PASS: TestLoginCommand_jwt (1.27s) - --- PASS: TestLoginCommand_jwt/success_-_jwt_jwks (0.04s) - --- PASS: TestLoginCommand_jwt/success_-_jwt_oidc_discovery (0.06s) - --- PASS: TestLoginCommand_jwt/success_-_jwt_static_keys (0.03s) + --- PASS: TestLoginCommand/try_login_with_no_method_configured (0.00s) + --- PASS: TestLoginCommand/try_login_with_method_configured_but_no_binding_rules (0.00s) + --- PASS: TestLoginCommand/try_login_with_method_configured_and_binding_rules (0.01s) +2022-12-04T05:50:47.873Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:47.889Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:47.891 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7dc5d5be-9095-7ceb-c4ce-e5f01b699fe1 Address:127.0.0.1:27370}]" +50:47.892 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27370 [Follower]" leader= +50:47.892 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7dc5d5be-9095-7ceb-c4ce-e5f01b699fe1.dc1 127.0.0.1 +50:47.899 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7dc5d5be-9095-7ceb-c4ce-e5f01b699fe1 127.0.0.1 +2022-12-04T05:50:47.899Z [INFO] agent.router: Initializing LAN area manager +50:47.899 [INFO] TestAgent.server: Adding LAN server: server="Node-7dc5d5be-9095-7ceb-c4ce-e5f01b699fe1 (Addr: tcp/127.0.0.1:27370) (DC: dc1)" +50:47.899 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7dc5d5be-9095-7ceb-c4ce-e5f01b699fe1.dc1 area=wan +50:47.899 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27365 network=tcp +50:47.900 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27365 network=udp +50:47.901 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27366 network=tcp +50:47.902 [INFO] TestAgent: started state syncer +50:47.902 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27371 network=tcp +50:47.933 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:47.933 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27370 [Candidate]" term=2 +50:47.933 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:47.933 [DEBUG] TestAgent.server.raft: vote granted: from=7dc5d5be-9095-7ceb-c4ce-e5f01b699fe1 term=2 tally=1 +50:47.933 [INFO] TestAgent.server.raft: election won: tally=1 +50:47.933 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27370 [Leader]" +50:47.933 [INFO] TestAgent.server: cluster leadership acquired +50:47.933 [INFO] TestAgent.server: initializing acls +50:47.934 [INFO] TestAgent.server: Created ACL 'global-management' policy +50:47.934 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +50:47.934 [INFO] TestAgent.server: New leader elected: payload=Node-7dc5d5be-9095-7ceb-c4ce-e5f01b699fe1 +50:47.934 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +50:47.934 [INFO] TestAgent.server: Created ACL anonymous token from configuration +50:47.934 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +50:47.934 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +50:47.934 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-7dc5d5be-9095-7ceb-c4ce-e5f01b699fe1 +50:47.934 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-7dc5d5be-9095-7ceb-c4ce-e5f01b699fe1.dc1 +50:47.935 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:47.936 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-7dc5d5be-9095-7ceb-c4ce-e5f01b699fe1.dc1 area=wan +50:47.936 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:47.936 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:47.936 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:47.936 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:47.936 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:47.936 [INFO] TestAgent.server: Updating LAN server: server="Node-7dc5d5be-9095-7ceb-c4ce-e5f01b699fe1 (Addr: tcp/127.0.0.1:27370) (DC: dc1)" +50:47.936 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7dc5d5be-9095-7ceb-c4ce-e5f01b699fe1 +50:47.936 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7dc5d5be-9095-7ceb-c4ce-e5f01b699fe1 +50:47.937 [INFO] TestAgent.server: federation state anti-entropy synced +50:47.937 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7dc5d5be-9095-7ceb-c4ce-e5f01b699fe1 +50:48.026 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:37530 latency=1.447504ms +50:48.028 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37530 latency=1.187505ms +50:48.032 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:37532 latency=2.595624ms +50:48.038 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:37530 latency=5.018251ms +50:48.044 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37530 latency=4.861176ms +50:48.051 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:37534 latency=5.102535ms +50:48.058 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:37530 latency=6.784209ms +50:48.066 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:37530 latency=6.799224ms +50:48.070 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:37536 latency=2.667383ms +50:48.071 [INFO] TestAgent: Requesting shutdown +50:48.071 [INFO] TestAgent.server: shutting down server +50:48.071 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +50:48.071 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +50:48.071 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:48.071 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:48.071 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:48.071 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:48.071 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:48.071 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:48.071 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:48.071 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:48.071 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +50:48.071 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +50:48.071 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:48.071 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +50:48.071 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:48.071 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:50:48.071Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:48.071Z [INFO] agent.router.manager: shutting down +50:48.071 [INFO] TestAgent: consul server down +50:48.071 [INFO] TestAgent: shutdown complete +50:48.071 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27365 network=tcp +50:48.071 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27365 network=udp +50:48.071 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27366 network=tcp +50:48.571 [INFO] TestAgent: Waiting for endpoints to shut down +50:48.571 [INFO] TestAgent: Endpoints down +--- PASS: TestLoginCommand_jwt (0.74s) + --- PASS: TestLoginCommand_jwt/success_-_jwt_static_keys (0.01s) + --- PASS: TestLoginCommand_jwt/success_-_jwt_jwks (0.02s) + --- PASS: TestLoginCommand_jwt/success_-_jwt_oidc_discovery (0.02s) +2022-12-04T05:50:47.873Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:47.890Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:47.892 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:87eb017e-66d4-f004-aa87-d0cabd22d05f Address:127.0.0.1:27377}]" +50:47.892 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27377 [Follower]" leader= +50:47.898 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-87eb017e-66d4-f004-aa87-d0cabd22d05f.dc1 127.0.0.1 +50:47.900 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-87eb017e-66d4-f004-aa87-d0cabd22d05f 127.0.0.1 +2022-12-04T05:50:47.900Z [INFO] agent.router: Initializing LAN area manager +50:47.900 [INFO] TestAgent.server: Adding LAN server: server="Node-87eb017e-66d4-f004-aa87-d0cabd22d05f (Addr: tcp/127.0.0.1:27377) (DC: dc1)" +50:47.900 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-87eb017e-66d4-f004-aa87-d0cabd22d05f.dc1 area=wan +50:47.901 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27372 network=udp +50:47.901 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27372 network=tcp +50:47.901 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27373 network=tcp +50:47.901 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27378 network=tcp +50:47.901 [INFO] TestAgent: started state syncer +50:47.951 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +50:47.951 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:47.951 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27377 [Candidate]" term=2 +50:47.951 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:47.951 [DEBUG] TestAgent.server.raft: vote granted: from=87eb017e-66d4-f004-aa87-d0cabd22d05f term=2 tally=1 +50:47.951 [INFO] TestAgent.server.raft: election won: tally=1 +50:47.951 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27377 [Leader]" +50:47.951 [INFO] TestAgent.server: cluster leadership acquired +50:47.951 [INFO] TestAgent.server: initializing acls +50:47.951 [INFO] TestAgent.server: New leader elected: payload=Node-87eb017e-66d4-f004-aa87-d0cabd22d05f +50:47.951 [INFO] TestAgent.server: Created ACL 'global-management' policy +50:47.951 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +50:47.952 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +50:47.952 [INFO] TestAgent.server: Created ACL anonymous token from configuration +50:47.952 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +50:47.952 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +50:47.952 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-87eb017e-66d4-f004-aa87-d0cabd22d05f +50:47.952 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-87eb017e-66d4-f004-aa87-d0cabd22d05f.dc1 +50:47.952 [INFO] TestAgent.server: Updating LAN server: server="Node-87eb017e-66d4-f004-aa87-d0cabd22d05f (Addr: tcp/127.0.0.1:27377) (DC: dc1)" +50:47.952 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-87eb017e-66d4-f004-aa87-d0cabd22d05f.dc1 area=wan +50:47.953 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:47.956 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:47.956 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:47.956 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:47.956 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:47.956 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:47.956 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-87eb017e-66d4-f004-aa87-d0cabd22d05f +50:47.956 [INFO] TestAgent.server: member joined, marking health alive: member=Node-87eb017e-66d4-f004-aa87-d0cabd22d05f +50:47.957 [INFO] TestAgent.server: federation state anti-entropy synced +50:47.957 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-87eb017e-66d4-f004-aa87-d0cabd22d05f +50:48.268 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:48.269 [INFO] TestAgent: Synced node info +50:48.269 [DEBUG] TestAgent: Node info in sync +50:48.271 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:43596 latency=1.92924ms +50:48.273 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:43596 latency=1.228022ms +50:48.282 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:43598 latency=7.394792ms +50:48.282 [INFO] TestAgent: Requesting shutdown +50:48.282 [INFO] TestAgent.server: shutting down server +50:48.282 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:48.282 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:48.282 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:48.282 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:48.282 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +50:48.282 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +50:48.282 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:48.282 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +50:48.283 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +50:48.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:48.283 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:48.283 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +50:48.283 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:48.283 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:48.283 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:48.283 [INFO] TestAgent: consul server down +50:48.283 [INFO] TestAgent: shutdown complete +50:48.283 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27372 network=tcp +2022-12-04T05:50:48.283Z [INFO] agent.router.manager: shutting down +50:48.283 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27372 network=udp +2022-12-04T05:50:48.283Z [INFO] agent.router.manager: shutting down +50:48.283 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27373 network=tcp +50:48.783 [INFO] TestAgent: Waiting for endpoints to shut down +50:48.783 [INFO] TestAgent: Endpoints down +--- PASS: TestLoginCommand_k8s (0.96s) + --- PASS: TestLoginCommand_k8s/try_login_with_method_configured_and_binding_rules (0.01s) PASS -ok github.com/hashicorp/consul/command/login 1.431s +ok github.com/hashicorp/consul/command/login 1.021s === RUN TestLogout_noTabs === PAUSE TestLogout_noTabs === RUN TestLogoutCommand @@ -88883,199 +89042,197 @@ === RUN TestLogoutCommand_k8s === PAUSE TestLogoutCommand_k8s === CONT TestLogout_noTabs ---- PASS: TestLogout_noTabs (0.00s) === CONT TestLogoutCommand_k8s -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestLogoutCommand +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestLogout_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -=== RUN TestLogoutCommand_k8s/no_token_specified -=== RUN TestLogoutCommand_k8s/logout_of_deleted_token -=== RUN TestLogoutCommand_k8s/logout_of_ordinary_token -=== RUN TestLogoutCommand_k8s/logout_of_login_token === RUN TestLogoutCommand/no_token_specified === RUN TestLogoutCommand/logout_of_deleted_token === RUN TestLogoutCommand/logout_of_ordinary_token === RUN TestLogoutCommand/logout_of_login_token -2021-10-31T23:18:15.060Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:15.111Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:15.136 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:725fc05b-90d2-2d6e-14e7-a3a8056d6da6 Address:127.0.0.1:16132}]" -18:15.136 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-725fc05b-90d2-2d6e-14e7-a3a8056d6da6.dc1 127.0.0.1 -18:15.137 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16132 [Follower]" leader= -18:15.137 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-725fc05b-90d2-2d6e-14e7-a3a8056d6da6 127.0.0.1 -2021-10-31T23:18:15.137Z [INFO] agent.router: Initializing LAN area manager -18:15.138 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=udp -18:15.138 [INFO] TestAgent.server: Adding LAN server: server="Node-725fc05b-90d2-2d6e-14e7-a3a8056d6da6 (Addr: tcp/127.0.0.1:16132) (DC: dc1)" -18:15.138 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-725fc05b-90d2-2d6e-14e7-a3a8056d6da6.dc1 area=wan -18:15.138 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=tcp -18:15.139 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16128 network=tcp -18:15.139 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16133 network=tcp -18:15.139 [INFO] TestAgent: started state syncer -18:15.194 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= -18:15.197 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:15.197 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16132 [Candidate]" term=2 -18:15.198 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:15.198 [DEBUG] TestAgent.server.raft: vote granted: from=725fc05b-90d2-2d6e-14e7-a3a8056d6da6 term=2 tally=1 -18:15.198 [INFO] TestAgent.server.raft: election won: tally=1 -18:15.198 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16132 [Leader]" -18:15.198 [INFO] TestAgent.server: cluster leadership acquired -18:15.198 [INFO] TestAgent.server: initializing acls -18:15.198 [INFO] TestAgent.server: Created ACL 'global-management' policy -18:15.198 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -18:15.199 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -18:15.199 [INFO] TestAgent.server: Created ACL anonymous token from configuration -18:15.199 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -18:15.199 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -18:15.199 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-725fc05b-90d2-2d6e-14e7-a3a8056d6da6 -18:15.199 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-725fc05b-90d2-2d6e-14e7-a3a8056d6da6.dc1 -18:15.200 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:15.200 [INFO] TestAgent.server: New leader elected: payload=Node-725fc05b-90d2-2d6e-14e7-a3a8056d6da6 -18:15.200 [INFO] TestAgent.server: Updating LAN server: server="Node-725fc05b-90d2-2d6e-14e7-a3a8056d6da6 (Addr: tcp/127.0.0.1:16132) (DC: dc1)" -18:15.200 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-725fc05b-90d2-2d6e-14e7-a3a8056d6da6.dc1 area=wan -18:15.247 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:15.247 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:15.247 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:15.247 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:15.247 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:15.247 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-725fc05b-90d2-2d6e-14e7-a3a8056d6da6 -18:15.247 [INFO] TestAgent.server: member joined, marking health alive: member=Node-725fc05b-90d2-2d6e-14e7-a3a8056d6da6 -18:15.248 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-725fc05b-90d2-2d6e-14e7-a3a8056d6da6 -18:15.248 [INFO] TestAgent.server: federation state anti-entropy synced -18:15.298 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:15.298 [INFO] TestAgent: Synced node info -18:15.298 [DEBUG] TestAgent: Node info in sync -18:15.335 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:43854 error="ACL not found" -18:15.335 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:43854 latency=349.322µs -18:15.342 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:43856 error="ACL not found" -18:15.342 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:43856 latency=310.732µs -18:15.353 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:43858 latency=2.6443ms -18:15.358 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:43860 error="Permission denied" -18:15.358 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:43860 latency=221.985µs -18:15.363 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:43858 latency=2.575041ms -18:15.367 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:43858 latency=3.138617ms -18:15.396 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:43858 latency=28.650473ms -18:15.414 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:43864 latency=8.183195ms -18:15.420 [INFO] TestAgent: Requesting shutdown -18:15.420 [INFO] TestAgent.server: shutting down server -18:15.420 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:15.420 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -18:15.420 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -18:15.420 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:15.420 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:15.420 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:15.420 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:15.420 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:15.420 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:15.420 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:15.420 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -18:15.420 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -18:15.420 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:15.420 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:15.420 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:15.420Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:15.420Z [INFO] agent.router.manager: shutting down -18:15.420 [INFO] TestAgent: consul server down -18:15.420 [INFO] TestAgent: shutdown complete -18:15.420 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=tcp -18:15.420 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=udp -18:15.420 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16128 network=tcp -18:15.921 [INFO] TestAgent: Waiting for endpoints to shut down -18:15.921 [INFO] TestAgent: Endpoints down ---- PASS: TestLogoutCommand_k8s (0.94s) - --- PASS: TestLogoutCommand_k8s/no_token_specified (0.01s) - --- PASS: TestLogoutCommand_k8s/logout_of_deleted_token (0.01s) - --- PASS: TestLogoutCommand_k8s/logout_of_ordinary_token (0.00s) - --- PASS: TestLogoutCommand_k8s/logout_of_login_token (0.02s) -2021-10-31T23:18:15.070Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:15.119Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:15.125 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:07d316ee-b12f-8265-d49e-11b661998b9c Address:127.0.0.1:16139}]" -18:15.126 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16139 [Follower]" leader= -18:15.127 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-07d316ee-b12f-8265-d49e-11b661998b9c.dc1 127.0.0.1 -18:15.138 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-07d316ee-b12f-8265-d49e-11b661998b9c 127.0.0.1 -2021-10-31T23:18:15.138Z [INFO] agent.router: Initializing LAN area manager -18:15.139 [INFO] TestAgent.server: Adding LAN server: server="Node-07d316ee-b12f-8265-d49e-11b661998b9c (Addr: tcp/127.0.0.1:16139) (DC: dc1)" -18:15.139 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-07d316ee-b12f-8265-d49e-11b661998b9c.dc1 area=wan -18:15.139 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=udp -18:15.139 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=tcp -18:15.140 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16135 network=tcp -18:15.140 [INFO] TestAgent: started state syncer -18:15.140 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16140 network=tcp -18:15.189 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:15.189 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16139 [Candidate]" term=2 -18:15.190 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:15.190 [DEBUG] TestAgent.server.raft: vote granted: from=07d316ee-b12f-8265-d49e-11b661998b9c term=2 tally=1 -18:15.190 [INFO] TestAgent.server.raft: election won: tally=1 -18:15.190 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16139 [Leader]" -18:15.190 [INFO] TestAgent.server: cluster leadership acquired -18:15.190 [INFO] TestAgent.server: initializing acls -18:15.190 [INFO] TestAgent.server: Created ACL 'global-management' policy -18:15.190 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated -18:15.191 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration -18:15.191 [INFO] TestAgent.server: Created ACL anonymous token from configuration -18:15.191 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" -18:15.191 [INFO] TestAgent.leader: started routine: routine="acl token reaping" -18:15.191 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-07d316ee-b12f-8265-d49e-11b661998b9c -18:15.191 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-07d316ee-b12f-8265-d49e-11b661998b9c.dc1 -18:15.192 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:15.193 [INFO] TestAgent.server: New leader elected: payload=Node-07d316ee-b12f-8265-d49e-11b661998b9c -18:15.194 [INFO] TestAgent.server: Updating LAN server: server="Node-07d316ee-b12f-8265-d49e-11b661998b9c (Addr: tcp/127.0.0.1:16139) (DC: dc1)" -18:15.194 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-07d316ee-b12f-8265-d49e-11b661998b9c.dc1 area=wan -18:15.244 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:15.244 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:15.244 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:15.244 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:15.244 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:15.244 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-07d316ee-b12f-8265-d49e-11b661998b9c -18:15.244 [INFO] TestAgent.server: member joined, marking health alive: member=Node-07d316ee-b12f-8265-d49e-11b661998b9c -18:15.246 [INFO] TestAgent.server: federation state anti-entropy synced -18:15.253 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-07d316ee-b12f-8265-d49e-11b661998b9c -18:15.389 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:15.402 [INFO] TestAgent: Synced node info -18:15.402 [DEBUG] TestAgent: Node info in sync -18:15.519 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:51144 error="ACL not found" -18:15.520 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:51144 latency=259.413µs -18:15.525 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:51146 error="ACL not found" -18:15.525 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:51146 latency=323.028µs -18:15.533 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:51148 latency=2.223719ms -18:15.540 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:51150 error="Permission denied" -18:15.540 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:51150 latency=422.207µs -18:15.541 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:51148 latency=676.652µs -18:15.547 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:51148 latency=995.216µs -18:15.551 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:51148 latency=979.596µs -18:15.554 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:51152 latency=1.468528ms -18:15.555 [INFO] TestAgent: Requesting shutdown -18:15.555 [INFO] TestAgent.server: shutting down server -18:15.555 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" -18:15.555 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" -18:15.555 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:15.555 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:15.555 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:15.555 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:15.555 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:15.555 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:15.555 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:15.555 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" -18:15.555 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" -18:15.555 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:15.555 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:15.555 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:15.555 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:15.555Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:15.555Z [INFO] agent.router.manager: shutting down -18:15.555 [INFO] TestAgent: consul server down -18:15.555 [INFO] TestAgent: shutdown complete -18:15.555 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=tcp -18:15.555 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=udp -18:15.555 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16135 network=tcp -18:16.058 [INFO] TestAgent: Waiting for endpoints to shut down -18:16.058 [INFO] TestAgent: Endpoints down ---- PASS: TestLogoutCommand (1.08s) - --- PASS: TestLogoutCommand/no_token_specified (0.01s) - --- PASS: TestLogoutCommand/logout_of_deleted_token (0.01s) +=== RUN TestLogoutCommand_k8s/no_token_specified +=== RUN TestLogoutCommand_k8s/logout_of_deleted_token +=== RUN TestLogoutCommand_k8s/logout_of_ordinary_token +=== RUN TestLogoutCommand_k8s/logout_of_login_token +2022-12-04T05:50:48.542Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:48.557Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:48.559 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:6ac8b43b-e370-7108-56e7-5692fb6805f5 Address:127.0.0.1:30426}]" +50:48.560 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= +50:48.562 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-6ac8b43b-e370-7108-56e7-5692fb6805f5.dc1 127.0.0.1 +50:48.564 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-6ac8b43b-e370-7108-56e7-5692fb6805f5 127.0.0.1 +2022-12-04T05:50:48.564Z [INFO] agent.router: Initializing LAN area manager +50:48.564 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-6ac8b43b-e370-7108-56e7-5692fb6805f5.dc1 area=wan +50:48.564 [INFO] TestAgent.server: Adding LAN server: server="Node-6ac8b43b-e370-7108-56e7-5692fb6805f5 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +50:48.564 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp +50:48.564 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp +50:48.565 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp +50:48.565 [INFO] TestAgent: started state syncer +50:48.565 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp +50:48.614 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +50:48.616 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:48.616 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 +50:48.617 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:48.617 [DEBUG] TestAgent.server.raft: vote granted: from=6ac8b43b-e370-7108-56e7-5692fb6805f5 term=2 tally=1 +50:48.617 [INFO] TestAgent.server.raft: election won: tally=1 +50:48.617 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" +50:48.617 [INFO] TestAgent.server: cluster leadership acquired +50:48.617 [INFO] TestAgent.server: New leader elected: payload=Node-6ac8b43b-e370-7108-56e7-5692fb6805f5 +50:48.617 [INFO] TestAgent.server: initializing acls +50:48.617 [INFO] TestAgent.server: Created ACL 'global-management' policy +50:48.617 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +50:48.618 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +50:48.618 [INFO] TestAgent.server: Created ACL anonymous token from configuration +50:48.618 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +50:48.618 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +50:48.619 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-6ac8b43b-e370-7108-56e7-5692fb6805f5 +50:48.619 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-6ac8b43b-e370-7108-56e7-5692fb6805f5.dc1 +50:48.619 [INFO] TestAgent.server: Updating LAN server: server="Node-6ac8b43b-e370-7108-56e7-5692fb6805f5 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +50:48.619 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-6ac8b43b-e370-7108-56e7-5692fb6805f5.dc1 area=wan +50:48.619 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:48.635 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:48.635 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:48.635 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:48.635 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:48.635 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:48.635 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6ac8b43b-e370-7108-56e7-5692fb6805f5 +50:48.635 [INFO] TestAgent.server: member joined, marking health alive: member=Node-6ac8b43b-e370-7108-56e7-5692fb6805f5 +50:48.635 [INFO] TestAgent.server: federation state anti-entropy synced +50:48.635 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-6ac8b43b-e370-7108-56e7-5692fb6805f5 +50:48.884 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:50388 error="ACL not found" +50:48.885 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:50388 latency=999.462µs +50:48.888 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:50390 error="ACL not found" +50:48.889 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:50390 latency=1.22318ms +50:48.892 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:50392 latency=1.316716ms +50:48.893 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:50394 error="Permission denied" +50:48.893 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:50394 latency=178.071µs +50:48.895 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:50392 latency=600.236µs +50:48.896 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:50392 latency=530.756µs +50:48.897 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:50392 latency=546.017µs +50:48.898 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:50396 latency=576.501µs +50:48.898 [INFO] TestAgent: Requesting shutdown +50:48.899 [INFO] TestAgent.server: shutting down server +50:48.899 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +50:48.899 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +50:48.899 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:48.899 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:48.899 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:48.899 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:48.899 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +50:48.899 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:48.899 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +50:48.899 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:48.899 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:48.899 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:48.899 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:48.899 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:48.899 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:48.899 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:48.899Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:48.899Z [INFO] agent.router.manager: shutting down +50:48.899 [INFO] TestAgent: consul server down +50:48.899 [INFO] TestAgent: shutdown complete +50:48.899 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp +50:48.899 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp +50:48.899 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp +50:49.399 [INFO] TestAgent: Waiting for endpoints to shut down +50:49.399 [INFO] TestAgent: Endpoints down +--- PASS: TestLogoutCommand (0.89s) + --- PASS: TestLogoutCommand/no_token_specified (0.00s) + --- PASS: TestLogoutCommand/logout_of_deleted_token (0.00s) --- PASS: TestLogoutCommand/logout_of_ordinary_token (0.00s) --- PASS: TestLogoutCommand/logout_of_login_token (0.00s) +2022-12-04T05:50:48.544Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:48.558Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:48.560 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8356c51b-6a0e-b8be-5db5-898712534706 Address:127.0.0.1:30433}]" +50:48.560 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30433 [Follower]" leader= +50:48.562 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8356c51b-6a0e-b8be-5db5-898712534706.dc1 127.0.0.1 +50:48.564 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8356c51b-6a0e-b8be-5db5-898712534706 127.0.0.1 +2022-12-04T05:50:48.564Z [INFO] agent.router: Initializing LAN area manager +50:48.564 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8356c51b-6a0e-b8be-5db5-898712534706.dc1 area=wan +50:48.564 [INFO] TestAgent.server: Adding LAN server: server="Node-8356c51b-6a0e-b8be-5db5-898712534706 (Addr: tcp/127.0.0.1:30433) (DC: dc1)" +50:48.564 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=tcp +50:48.564 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=udp +50:48.565 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30429 network=tcp +50:48.565 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30434 network=tcp +50:48.565 [INFO] TestAgent: started state syncer +50:48.614 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=3 mode=2 found=true leader= +50:48.616 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:48.616 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30433 [Candidate]" term=2 +50:48.616 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:48.616 [DEBUG] TestAgent.server.raft: vote granted: from=8356c51b-6a0e-b8be-5db5-898712534706 term=2 tally=1 +50:48.616 [INFO] TestAgent.server.raft: election won: tally=1 +50:48.616 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30433 [Leader]" +50:48.616 [INFO] TestAgent.server: cluster leadership acquired +50:48.617 [INFO] TestAgent.server: New leader elected: payload=Node-8356c51b-6a0e-b8be-5db5-898712534706 +50:48.617 [INFO] TestAgent.server: initializing acls +50:48.617 [INFO] TestAgent.server: Created ACL 'global-management' policy +50:48.617 [WARN] TestAgent.server: Configuring a non-UUID master token is deprecated +50:48.617 [INFO] TestAgent.server: Bootstrapped ACL master token from configuration +50:48.618 [INFO] TestAgent.server: Created ACL anonymous token from configuration +50:48.618 [INFO] TestAgent.leader: started routine: routine="legacy ACL token upgrade" +50:48.618 [INFO] TestAgent.leader: started routine: routine="acl token reaping" +50:48.618 [INFO] TestAgent.server.serf.lan: serf: EventMemberUpdate: Node-8356c51b-6a0e-b8be-5db5-898712534706 +50:48.618 [INFO] TestAgent.server.serf.wan: serf: EventMemberUpdate: Node-8356c51b-6a0e-b8be-5db5-898712534706.dc1 +50:48.618 [INFO] TestAgent.server: Handled event for server in area: event=member-update server=Node-8356c51b-6a0e-b8be-5db5-898712534706.dc1 area=wan +50:48.618 [INFO] TestAgent.server: Updating LAN server: server="Node-8356c51b-6a0e-b8be-5db5-898712534706 (Addr: tcp/127.0.0.1:30433) (DC: dc1)" +50:48.619 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:48.628 [INFO] TestAgent: Synced node info +50:48.628 [DEBUG] TestAgent: Node info in sync +50:48.634 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:48.634 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:48.634 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:48.634 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:48.634 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:48.634 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8356c51b-6a0e-b8be-5db5-898712534706 +50:48.634 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8356c51b-6a0e-b8be-5db5-898712534706 +50:48.634 [INFO] TestAgent.server: federation state anti-entropy synced +50:48.635 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8356c51b-6a0e-b8be-5db5-898712534706 +50:48.964 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:33400 error="ACL not found" +50:48.965 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:33400 latency=133.784µs +50:48.966 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:33402 error="ACL not found" +50:48.967 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:33402 latency=878.652µs +50:48.969 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/token from=127.0.0.1:33404 latency=1.024236ms +50:48.970 [ERROR] TestAgent.http: Request error: method=POST url=/v1/acl/logout from=127.0.0.1:33406 error="Permission denied" +50:48.970 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:33406 latency=139.569µs +50:48.977 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/auth-method from=127.0.0.1:33404 latency=6.330907ms +50:48.980 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/acl/binding-rule from=127.0.0.1:33404 latency=1.325619ms +50:48.988 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/login from=127.0.0.1:33404 latency=7.472436ms +50:48.990 [DEBUG] TestAgent.http: Request finished: method=POST url=/v1/acl/logout from=127.0.0.1:33408 latency=476.751µs +50:48.990 [INFO] TestAgent: Requesting shutdown +50:48.990 [INFO] TestAgent.server: shutting down server +50:48.990 [DEBUG] TestAgent.leader: stopping routine: routine="acl token reaping" +50:48.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:48.990 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:48.990 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:48.990 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:48.990 [DEBUG] TestAgent.leader: stopping routine: routine="legacy ACL token upgrade" +50:48.990 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:48.990 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:48.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:48.990 [DEBUG] TestAgent.leader: stopped routine: routine="legacy ACL token upgrade" +50:48.990 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:48.990 [DEBUG] TestAgent.leader: stopped routine: routine="acl token reaping" +50:48.990 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:48.990 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:48.990 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:48.990Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:48.990Z [INFO] agent.router.manager: shutting down +50:48.990 [INFO] TestAgent: consul server down +50:48.990 [INFO] TestAgent: shutdown complete +50:48.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=tcp +50:48.990 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=udp +50:48.991 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30429 network=tcp +50:49.491 [INFO] TestAgent: Waiting for endpoints to shut down +50:49.491 [INFO] TestAgent: Endpoints down +--- PASS: TestLogoutCommand_k8s (0.98s) + --- PASS: TestLogoutCommand_k8s/no_token_specified (0.00s) + --- PASS: TestLogoutCommand_k8s/logout_of_deleted_token (0.00s) + --- PASS: TestLogoutCommand_k8s/logout_of_ordinary_token (0.00s) + --- PASS: TestLogoutCommand_k8s/logout_of_login_token (0.00s) PASS -ok github.com/hashicorp/consul/command/logout 1.194s +ok github.com/hashicorp/consul/command/logout 1.032s === RUN TestMaintCommand_noTabs === PAUSE TestMaintCommand_noTabs === RUN TestMaintCommand_ConflictingArgs @@ -89092,372 +89249,379 @@ === PAUSE TestMaintCommand_DisableServiceMaintenance === RUN TestMaintCommand_ServiceMaintenance_NoService === PAUSE TestMaintCommand_ServiceMaintenance_NoService -=== CONT TestMaintCommand_noTabs ---- PASS: TestMaintCommand_noTabs (0.00s) +=== CONT TestMaintCommand_DisableServiceMaintenance +=== CONT TestMaintCommand_NoArgs === CONT TestMaintCommand_ServiceMaintenance_NoService +=== CONT TestMaintCommand_ConflictingArgs [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestMaintCommand_EnableNodeMaintenance -=== CONT TestMaintCommand_DisableServiceMaintenance +=== CONT TestMaintCommand_noTabs === CONT TestMaintCommand_EnableServiceMaintenance === CONT TestMaintCommand_DisableNodeMaintenance -=== CONT TestMaintCommand_ConflictingArgs ---- PASS: TestMaintCommand_ConflictingArgs (0.00s) -=== CONT TestMaintCommand_NoArgs +=== CONT TestMaintCommand_EnableNodeMaintenance +--- PASS: TestMaintCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:18:20.794Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:20.860Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:20.866 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:19719b20-e35b-ab55-e900-100e4dffa202 Address:127.0.0.1:30426}]" -18:20.867 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-19719b20-e35b-ab55-e900-100e4dffa202.dc1 127.0.0.1 -18:20.867 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-19719b20-e35b-ab55-e900-100e4dffa202 127.0.0.1 -2021-10-31T23:18:20.867Z [INFO] agent.router: Initializing LAN area manager -18:20.868 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp -18:20.868 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= -18:20.888 [INFO] TestAgent.server: Adding LAN server: server="Node-19719b20-e35b-ab55-e900-100e4dffa202 (Addr: tcp/127.0.0.1:30426) (DC: dc1)" -18:20.889 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-19719b20-e35b-ab55-e900-100e4dffa202.dc1 area=wan -18:20.889 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp -18:20.918 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:20.918 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 -18:20.918 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:20.918 [DEBUG] TestAgent.server.raft: vote granted: from=19719b20-e35b-ab55-e900-100e4dffa202 term=2 tally=1 -18:20.918 [INFO] TestAgent.server.raft: election won: tally=1 -18:20.918 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" -18:20.918 [INFO] TestAgent.server: cluster leadership acquired -18:20.919 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30426 -18:20.920 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:20.941 [INFO] TestAgent.server: New leader elected: payload=Node-19719b20-e35b-ab55-e900-100e4dffa202 -18:20.942 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp -18:20.942 [INFO] TestAgent: started state syncer -18:20.942 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp -18:20.954 [INFO] TestAgent: Synced node info -18:20.954 [DEBUG] TestAgent: Node info in sync -18:20.977 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/maintenance/redis?enable=true&reason=broken from=127.0.0.1:45236 latency=20.419644ms -18:20.982 [INFO] TestAgent: Requesting shutdown -18:20.982 [INFO] TestAgent.server: shutting down server -18:20.982 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:20.982 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:20.982Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:20.982Z [INFO] agent.router.manager: shutting down -18:20.982 [INFO] TestAgent: consul server down -18:20.982 [INFO] TestAgent: shutdown complete -18:20.982 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp -18:20.982 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp -18:20.982 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp -18:20.989 [ERROR] TestAgent.server: failed to establish leadership: error="error generating CA root certificate: error computing next serial number: raft is already shutdown" -18:20.989 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=0 retry_limit=3 error="raft is already shutdown" -18:20.989 [ERROR] TestAgent.server: failed to transfer leadership attempt, will retry: attempt=1 retry_limit=3 error="raft is already shutdown" -18:21.485 [INFO] TestAgent: Waiting for endpoints to shut down -18:21.485 [INFO] TestAgent: Endpoints down ---- PASS: TestMaintCommand_ServiceMaintenance_NoService (0.78s) -2021-10-31T23:18:20.867Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:20.999Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:21.001 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:51bf17f3-ab1e-2a28-0ec8-086415dc9f33 Address:127.0.0.1:30447}]" -18:21.001 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30447 [Follower]" leader= -18:21.006 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-51bf17f3-ab1e-2a28-0ec8-086415dc9f33.dc1 127.0.0.1 -18:21.006 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-51bf17f3-ab1e-2a28-0ec8-086415dc9f33 127.0.0.1 -2021-10-31T23:18:21.006Z [INFO] agent.router: Initializing LAN area manager -18:21.007 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30442 network=udp -18:21.007 [INFO] TestAgent.server: Adding LAN server: server="Node-51bf17f3-ab1e-2a28-0ec8-086415dc9f33 (Addr: tcp/127.0.0.1:30447) (DC: dc1)" -18:21.007 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-51bf17f3-ab1e-2a28-0ec8-086415dc9f33.dc1 area=wan -18:21.037 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:21.037 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30447 [Candidate]" term=2 -18:21.037 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:21.037 [DEBUG] TestAgent.server.raft: vote granted: from=51bf17f3-ab1e-2a28-0ec8-086415dc9f33 term=2 tally=1 -18:21.037 [INFO] TestAgent.server.raft: election won: tally=1 -18:21.037 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30447 [Leader]" -18:21.037 [INFO] TestAgent.server: cluster leadership acquired -18:21.038 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30447 -18:21.038 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:21.038 [INFO] TestAgent.server: New leader elected: payload=Node-51bf17f3-ab1e-2a28-0ec8-086415dc9f33 -18:21.074 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30442 network=tcp -18:21.075 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30443 network=tcp -18:21.075 [INFO] TestAgent: started state syncer -18:21.076 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:21.076 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:21.076 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:21.076 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:21.076 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:21.076 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30448 network=tcp -18:21.076 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-51bf17f3-ab1e-2a28-0ec8-086415dc9f33 -18:21.076 [INFO] TestAgent.server: member joined, marking health alive: member=Node-51bf17f3-ab1e-2a28-0ec8-086415dc9f33 -18:21.082 [INFO] TestAgent.server: federation state anti-entropy synced -18:21.082 [INFO] TestAgent: Synced node info -18:21.083 [INFO] TestAgent: Synced service: service=test -18:21.083 [DEBUG] TestAgent: Node info in sync -18:21.083 [DEBUG] TestAgent: Service in sync: service=test -18:21.084 [INFO] TestAgent: Service entered maintenance mode: service=test -18:21.084 [DEBUG] TestAgent: Node info in sync -18:21.084 [DEBUG] TestAgent: Service in sync: service=test -18:21.084 [INFO] TestAgent: Synced check: check=_service_maintenance:test -18:21.084 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/maintenance/test?enable=true&reason=broken from=127.0.0.1:41122 latency=6.465729ms -18:21.085 [INFO] TestAgent: Requesting shutdown -18:21.085 [INFO] TestAgent.server: shutting down server -18:21.085 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:21.085 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:21.085 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:21.085 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:21.085 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:21.085 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -18:21.085 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:21.086 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:21.086 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:21.086 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:21.086 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -2021-10-31T23:18:21.086Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:21.086Z [INFO] agent.router.manager: shutting down -18:21.086 [INFO] TestAgent: consul server down -18:21.086 [INFO] TestAgent: shutdown complete -18:21.086 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30442 network=tcp -18:21.086 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30442 network=udp -18:21.086 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30443 network=tcp -18:21.589 [INFO] TestAgent: Waiting for endpoints to shut down -18:21.589 [INFO] TestAgent: Endpoints down ---- PASS: TestMaintCommand_EnableServiceMaintenance (0.89s) -2021-10-31T23:18:20.815Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:20.959Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:20.966 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3c85380e-191c-8cae-37ec-153215f57abe Address:127.0.0.1:30461}]" -18:20.967 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3c85380e-191c-8cae-37ec-153215f57abe.dc1 127.0.0.1 -18:20.968 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3c85380e-191c-8cae-37ec-153215f57abe 127.0.0.1 -2021-10-31T23:18:20.968Z [INFO] agent.router: Initializing LAN area manager -18:20.987 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30456 network=udp -18:20.987 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30461 [Follower]" leader= -18:20.988 [INFO] TestAgent.server: Adding LAN server: server="Node-3c85380e-191c-8cae-37ec-153215f57abe (Addr: tcp/127.0.0.1:30461) (DC: dc1)" -18:20.988 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3c85380e-191c-8cae-37ec-153215f57abe.dc1 area=wan -18:20.988 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30456 network=tcp -18:20.988 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30457 network=tcp -18:20.988 [INFO] TestAgent: started state syncer -18:20.989 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30462 network=tcp -18:21.070 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:21.070 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30461 [Candidate]" term=2 -18:21.070 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:21.070 [DEBUG] TestAgent.server.raft: vote granted: from=3c85380e-191c-8cae-37ec-153215f57abe term=2 tally=1 -18:21.070 [INFO] TestAgent.server.raft: election won: tally=1 -18:21.070 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30461 [Leader]" -18:21.077 [INFO] TestAgent.server: cluster leadership acquired -18:21.077 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30461 -18:21.078 [INFO] TestAgent.server: New leader elected: payload=Node-3c85380e-191c-8cae-37ec-153215f57abe -18:21.086 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:21.112 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:21.112 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:21.112 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:21.112 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:21.112 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:21.112 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3c85380e-191c-8cae-37ec-153215f57abe -18:21.112 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3c85380e-191c-8cae-37ec-153215f57abe -18:21.113 [INFO] TestAgent.server: federation state anti-entropy synced -18:21.226 [INFO] TestAgent: Service entered maintenance mode: service=test -18:21.226 [INFO] TestAgent: Node entered maintenance mode -18:21.229 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:40994 latency=1.99119ms -18:21.246 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:40994 latency=355.349µs -18:21.247 [INFO] TestAgent: Requesting shutdown -18:21.247 [INFO] TestAgent.server: shutting down server -18:21.247 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:21.247 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:21.247 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:21.247 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:21.247 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:21.247 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:21.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:21.247 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -18:21.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:21.247 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:21.247 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:21.247 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:21.247Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:21.247Z [INFO] agent.router.manager: shutting down -18:21.247 [INFO] TestAgent: consul server down -18:21.247 [INFO] TestAgent: shutdown complete -18:21.247 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30456 network=tcp -18:21.247 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30456 network=udp -18:21.247 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30457 network=tcp -18:21.748 [INFO] TestAgent: Waiting for endpoints to shut down -18:21.748 [INFO] TestAgent: Endpoints down ---- PASS: TestMaintCommand_NoArgs (1.04s) -2021-10-31T23:18:20.867Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:20.984Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:20.987 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ee035a5f-a447-7c1e-1c0c-b4fbd88cbf2a Address:127.0.0.1:30433}]" -18:20.987 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ee035a5f-a447-7c1e-1c0c-b4fbd88cbf2a.dc1 127.0.0.1 -18:20.988 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ee035a5f-a447-7c1e-1c0c-b4fbd88cbf2a 127.0.0.1 -2021-10-31T23:18:20.988Z [INFO] agent.router: Initializing LAN area manager -18:20.988 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=udp -18:20.988 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30433 [Follower]" leader= -18:20.989 [INFO] TestAgent.server: Adding LAN server: server="Node-ee035a5f-a447-7c1e-1c0c-b4fbd88cbf2a (Addr: tcp/127.0.0.1:30433) (DC: dc1)" -18:20.989 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ee035a5f-a447-7c1e-1c0c-b4fbd88cbf2a.dc1 area=wan -18:20.989 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=tcp -18:21.002 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30429 network=tcp -18:21.002 [INFO] TestAgent: started state syncer -18:21.002 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30434 network=tcp -18:21.078 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:21.078 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30433 [Candidate]" term=2 -18:21.078 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:21.078 [DEBUG] TestAgent.server.raft: vote granted: from=ee035a5f-a447-7c1e-1c0c-b4fbd88cbf2a term=2 tally=1 -18:21.078 [INFO] TestAgent.server.raft: election won: tally=1 -18:21.078 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30433 [Leader]" -18:21.079 [INFO] TestAgent.server: cluster leadership acquired -18:21.079 [INFO] TestAgent.server: New leader elected: payload=Node-ee035a5f-a447-7c1e-1c0c-b4fbd88cbf2a -18:21.079 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30433 -18:21.080 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:21.094 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:21.094 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:21.094 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:21.094 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:21.094 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:21.094 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ee035a5f-a447-7c1e-1c0c-b4fbd88cbf2a -18:21.094 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ee035a5f-a447-7c1e-1c0c-b4fbd88cbf2a -18:21.110 [INFO] TestAgent.server: federation state anti-entropy synced -18:21.113 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:21.116 [INFO] TestAgent: Synced node info -18:21.116 [DEBUG] TestAgent: Node info in sync -18:21.245 [INFO] TestAgent: Node entered maintenance mode -18:21.245 [DEBUG] TestAgent: Node info in sync -18:21.246 [INFO] TestAgent: Synced check: check=_node_maintenance -18:21.246 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/maintenance?enable=true&reason=broken from=127.0.0.1:38458 latency=895.789µs -18:21.246 [INFO] TestAgent: Requesting shutdown -18:21.246 [INFO] TestAgent.server: shutting down server -18:21.246 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:21.246 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:21.246 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:21.246 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:21.246 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:21.246 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:21.246 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:21.247 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:21.247 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:21.247 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:21.247 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:21.247Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:21.247Z [INFO] agent.router.manager: shutting down -18:21.247 [INFO] TestAgent: consul server down -18:21.247 [INFO] TestAgent: shutdown complete -18:21.247 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=tcp -18:21.247 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=udp -18:21.247 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30429 network=tcp -18:21.749 [INFO] TestAgent: Waiting for endpoints to shut down -18:21.749 [INFO] TestAgent: Endpoints down ---- PASS: TestMaintCommand_EnableNodeMaintenance (1.04s) -2021-10-31T23:18:20.810Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:20.846Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:20.850 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1a2f667e-e05d-a290-084e-1c935012f31e Address:127.0.0.1:30440}]" -18:20.851 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1a2f667e-e05d-a290-084e-1c935012f31e.dc1 127.0.0.1 -18:20.852 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1a2f667e-e05d-a290-084e-1c935012f31e 127.0.0.1 -2021-10-31T23:18:20.852Z [INFO] agent.router: Initializing LAN area manager -18:20.889 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30435 network=udp -18:20.889 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30440 [Follower]" leader= -18:20.891 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1a2f667e-e05d-a290-084e-1c935012f31e.dc1 area=wan -18:20.891 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30435 network=tcp -18:20.892 [INFO] TestAgent.server: Adding LAN server: server="Node-1a2f667e-e05d-a290-084e-1c935012f31e (Addr: tcp/127.0.0.1:30440) (DC: dc1)" -18:20.915 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30436 network=tcp -18:20.915 [INFO] TestAgent: started state syncer -18:20.918 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30441 network=tcp -18:20.949 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:20.953 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30440 [Candidate]" term=2 -18:20.954 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:20.954 [DEBUG] TestAgent.server.raft: vote granted: from=1a2f667e-e05d-a290-084e-1c935012f31e term=2 tally=1 -18:20.954 [INFO] TestAgent.server.raft: election won: tally=1 -18:20.954 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30440 [Leader]" -18:20.954 [INFO] TestAgent.server: cluster leadership acquired -18:20.954 [INFO] TestAgent.server: New leader elected: payload=Node-1a2f667e-e05d-a290-084e-1c935012f31e -18:20.954 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30440 -18:20.955 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:20.995 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:20.995 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:20.995 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:20.995 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:20.995 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:20.995 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1a2f667e-e05d-a290-084e-1c935012f31e -18:20.995 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1a2f667e-e05d-a290-084e-1c935012f31e -18:20.996 [INFO] TestAgent.server: federation state anti-entropy synced -18:21.190 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:21.190 [INFO] TestAgent: Synced node info -18:21.190 [DEBUG] TestAgent: Node info in sync -18:21.310 [DEBUG] TestAgent: Node info in sync -18:21.310 [INFO] TestAgent: Synced service: service=test -18:21.310 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/maintenance/test?enable=false from=127.0.0.1:53366 latency=553.289µs -18:21.314 [INFO] TestAgent: Requesting shutdown -18:21.314 [INFO] TestAgent.server: shutting down server -18:21.314 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:21.314 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:21.314 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:21.314 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:21.314 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:21.314 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:21.314 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:21.314 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:21.314 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:21.314 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:21.314 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:21.314Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:21.314Z [INFO] agent.router.manager: shutting down -18:21.314 [INFO] TestAgent: consul server down -18:21.314 [INFO] TestAgent: shutdown complete -18:21.314 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30435 network=tcp -18:21.314 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30435 network=udp -18:21.314 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30436 network=tcp -18:21.818 [INFO] TestAgent: Waiting for endpoints to shut down -18:21.818 [INFO] TestAgent: Endpoints down ---- PASS: TestMaintCommand_DisableServiceMaintenance (1.11s) -2021-10-31T23:18:20.890Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:20.956Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:20.958 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d036a236-6041-cc50-0847-c1a2c8ed9c7c Address:127.0.0.1:30454}]" -18:20.959 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d036a236-6041-cc50-0847-c1a2c8ed9c7c.dc1 127.0.0.1 -18:20.960 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d036a236-6041-cc50-0847-c1a2c8ed9c7c 127.0.0.1 -2021-10-31T23:18:20.960Z [INFO] agent.router: Initializing LAN area manager -18:20.986 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30454 [Follower]" leader= -18:20.986 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30449 network=udp -18:20.986 [INFO] TestAgent.server: Adding LAN server: server="Node-d036a236-6041-cc50-0847-c1a2c8ed9c7c (Addr: tcp/127.0.0.1:30454) (DC: dc1)" -18:20.986 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d036a236-6041-cc50-0847-c1a2c8ed9c7c.dc1 area=wan -18:20.986 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30449 network=tcp -18:20.987 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30450 network=tcp -18:20.987 [INFO] TestAgent: started state syncer -18:20.987 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30455 network=tcp -18:21.074 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:21.074 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30454 [Candidate]" term=2 -18:21.074 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:21.074 [DEBUG] TestAgent.server.raft: vote granted: from=d036a236-6041-cc50-0847-c1a2c8ed9c7c term=2 tally=1 -18:21.074 [INFO] TestAgent.server.raft: election won: tally=1 -18:21.074 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30454 [Leader]" -18:21.075 [INFO] TestAgent.server: cluster leadership acquired -18:21.075 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30454 -18:21.076 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:21.078 [INFO] TestAgent.server: New leader elected: payload=Node-d036a236-6041-cc50-0847-c1a2c8ed9c7c -18:21.079 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:21.079 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:21.079 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:21.079 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:21.079 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:21.080 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d036a236-6041-cc50-0847-c1a2c8ed9c7c -18:21.080 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d036a236-6041-cc50-0847-c1a2c8ed9c7c -18:21.081 [INFO] TestAgent.server: federation state anti-entropy synced -18:21.268 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:21.268 [INFO] TestAgent: Synced node info -18:21.268 [DEBUG] TestAgent: Node info in sync -18:21.390 [DEBUG] TestAgent: Node info in sync -18:21.390 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/maintenance?enable=false from=127.0.0.1:49652 latency=26.178µs -18:21.393 [INFO] TestAgent: Requesting shutdown -18:21.393 [INFO] TestAgent.server: shutting down server -18:21.393 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:21.393 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:21.393 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:21.393 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:21.393 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:21.393 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:21.393 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:21.393 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:21.393 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:21.393 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:21.393 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:21.393Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:21.393Z [INFO] agent.router.manager: shutting down -18:21.393 [INFO] TestAgent: consul server down -18:21.393 [INFO] TestAgent: shutdown complete -18:21.393 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30449 network=tcp -18:21.393 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30449 network=udp -18:21.393 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30450 network=tcp -18:21.893 [INFO] TestAgent: Waiting for endpoints to shut down -18:21.893 [INFO] TestAgent: Endpoints down ---- PASS: TestMaintCommand_DisableNodeMaintenance (1.19s) +--- PASS: TestMaintCommand_ConflictingArgs (0.00s) +2022-12-04T05:50:51.495Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:51.541Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:51.542 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cfb08635-5b12-9298-e256-471eb345236e Address:127.0.0.1:13083}]" +50:51.542 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13083 [Follower]" leader= +50:51.543 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cfb08635-5b12-9298-e256-471eb345236e.dc1 127.0.0.1 +50:51.568 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cfb08635-5b12-9298-e256-471eb345236e 127.0.0.1 +2022-12-04T05:50:51.568Z [INFO] agent.router: Initializing LAN area manager +50:51.569 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13078 network=udp +50:51.569 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cfb08635-5b12-9298-e256-471eb345236e.dc1 area=wan +50:51.569 [INFO] TestAgent.server: Adding LAN server: server="Node-cfb08635-5b12-9298-e256-471eb345236e (Addr: tcp/127.0.0.1:13083) (DC: dc1)" +50:51.569 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13078 network=tcp +50:51.569 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13079 network=tcp +50:51.569 [INFO] TestAgent: started state syncer +50:51.569 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13084 network=tcp +50:51.583 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:51.583 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13083 [Candidate]" term=2 +50:51.583 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:51.583 [DEBUG] TestAgent.server.raft: vote granted: from=cfb08635-5b12-9298-e256-471eb345236e term=2 tally=1 +50:51.583 [INFO] TestAgent.server.raft: election won: tally=1 +50:51.583 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13083 [Leader]" +50:51.583 [INFO] TestAgent.server: cluster leadership acquired +50:51.583 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13083 +50:51.583 [INFO] TestAgent.server: New leader elected: payload=Node-cfb08635-5b12-9298-e256-471eb345236e +50:51.583 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:51.586 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:51.586 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:51.586 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:51.586 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:51.586 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:51.586 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cfb08635-5b12-9298-e256-471eb345236e +50:51.586 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cfb08635-5b12-9298-e256-471eb345236e +50:51.675 [INFO] TestAgent: Synced node info +50:51.675 [INFO] TestAgent: Synced service: service=test +50:51.675 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/maintenance/test?enable=false from=127.0.0.1:42364 latency=577.617µs +50:51.676 [INFO] TestAgent: Requesting shutdown +50:51.676 [INFO] TestAgent.server: shutting down server +50:51.676 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:51.676 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:51.676 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:51.676 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:51.676 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:51.676 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:51.676 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:51.676 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:51.676 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="shutdown waiting for leader" +50:51.676 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:51.676 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:51.676 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:51.676Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:51.676Z [INFO] agent.router.manager: shutting down +50:51.676 [INFO] TestAgent: consul server down +50:51.676 [INFO] TestAgent: shutdown complete +50:51.676 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13078 network=tcp +50:51.676 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13078 network=udp +50:51.676 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13079 network=tcp +50:52.177 [INFO] TestAgent: Waiting for endpoints to shut down +50:52.177 [INFO] TestAgent: Endpoints down +--- PASS: TestMaintCommand_DisableServiceMaintenance (0.74s) +2022-12-04T05:50:51.489Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:51.518Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:51.526 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:057e33b6-f347-bc7c-c266-79e72aac393e Address:127.0.0.1:13090}]" +50:51.526 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-057e33b6-f347-bc7c-c266-79e72aac393e.dc1 127.0.0.1 +50:51.535 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-057e33b6-f347-bc7c-c266-79e72aac393e 127.0.0.1 +2022-12-04T05:50:51.535Z [INFO] agent.router: Initializing LAN area manager +50:51.535 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13090 [Follower]" leader= +50:51.535 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-057e33b6-f347-bc7c-c266-79e72aac393e.dc1 area=wan +50:51.535 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13085 network=udp +50:51.535 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13085 network=tcp +50:51.536 [INFO] TestAgent.server: Adding LAN server: server="Node-057e33b6-f347-bc7c-c266-79e72aac393e (Addr: tcp/127.0.0.1:13090) (DC: dc1)" +50:51.537 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13086 network=tcp +50:51.537 [INFO] TestAgent: started state syncer +50:51.537 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13091 network=tcp +50:51.574 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:51.574 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13090 [Candidate]" term=2 +50:51.574 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:51.574 [DEBUG] TestAgent.server.raft: vote granted: from=057e33b6-f347-bc7c-c266-79e72aac393e term=2 tally=1 +50:51.574 [INFO] TestAgent.server.raft: election won: tally=1 +50:51.574 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13090 [Leader]" +50:51.574 [INFO] TestAgent.server: cluster leadership acquired +50:51.574 [INFO] TestAgent.server: New leader elected: payload=Node-057e33b6-f347-bc7c-c266-79e72aac393e +50:51.574 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13090 +50:51.575 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:51.576 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:51.576 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:51.576 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:51.576 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:51.576 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:51.576 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-057e33b6-f347-bc7c-c266-79e72aac393e +50:51.576 [INFO] TestAgent.server: member joined, marking health alive: member=Node-057e33b6-f347-bc7c-c266-79e72aac393e +50:51.577 [INFO] TestAgent.server: federation state anti-entropy synced +50:51.683 [INFO] TestAgent: Service entered maintenance mode: service=test +50:51.684 [INFO] TestAgent: Synced node info +50:51.684 [INFO] TestAgent: Synced service: service=test +50:51.684 [DEBUG] TestAgent: Check in sync: check=_service_maintenance:test +50:51.684 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/maintenance/test?enable=true&reason=broken from=127.0.0.1:53828 latency=1.109598ms +50:51.684 [INFO] TestAgent: Requesting shutdown +50:51.684 [INFO] TestAgent.server: shutting down server +50:51.684 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:51.684 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:51.684 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:51.684 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:51.684 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:51.684 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:51.684 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:51.684 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:51.684 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:51.684 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:51.684 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:51.684 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:51.684Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:51.684Z [INFO] agent.router.manager: shutting down +50:51.685 [INFO] TestAgent: consul server down +50:51.685 [INFO] TestAgent: shutdown complete +50:51.685 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13085 network=tcp +50:51.685 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13085 network=udp +50:51.685 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13086 network=tcp +50:52.185 [INFO] TestAgent: Waiting for endpoints to shut down +50:52.185 [INFO] TestAgent: Endpoints down +--- PASS: TestMaintCommand_EnableServiceMaintenance (0.75s) +2022-12-04T05:50:51.489Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:51.509Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:51.511 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:a0e07425-be00-3c3c-9f53-c1bcf84c1735 Address:127.0.0.1:13069}]" +50:51.514 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= +50:51.515 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-a0e07425-be00-3c3c-9f53-c1bcf84c1735.dc1 127.0.0.1 +50:51.516 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-a0e07425-be00-3c3c-9f53-c1bcf84c1735 127.0.0.1 +2022-12-04T05:50:51.516Z [INFO] agent.router: Initializing LAN area manager +50:51.516 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-a0e07425-be00-3c3c-9f53-c1bcf84c1735.dc1 area=wan +50:51.516 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp +50:51.516 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp +50:51.517 [INFO] TestAgent.server: Adding LAN server: server="Node-a0e07425-be00-3c3c-9f53-c1bcf84c1735 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" +50:51.517 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp +50:51.518 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp +50:51.518 [INFO] TestAgent: started state syncer +50:51.560 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:51.560 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 +50:51.560 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:51.560 [DEBUG] TestAgent.server.raft: vote granted: from=a0e07425-be00-3c3c-9f53-c1bcf84c1735 term=2 tally=1 +50:51.560 [INFO] TestAgent.server.raft: election won: tally=1 +50:51.560 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" +50:51.560 [INFO] TestAgent.server: cluster leadership acquired +50:51.560 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13069 +50:51.560 [INFO] TestAgent.server: New leader elected: payload=Node-a0e07425-be00-3c3c-9f53-c1bcf84c1735 +50:51.561 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:51.575 [INFO] TestAgent: Synced node info +50:51.575 [DEBUG] TestAgent: Node info in sync +50:51.576 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:51.576 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:51.576 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:51.576 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:51.576 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:51.576 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-a0e07425-be00-3c3c-9f53-c1bcf84c1735 +50:51.576 [INFO] TestAgent.server: member joined, marking health alive: member=Node-a0e07425-be00-3c3c-9f53-c1bcf84c1735 +50:51.577 [INFO] TestAgent.server: federation state anti-entropy synced +50:51.789 [INFO] TestAgent: Service entered maintenance mode: service=test +50:51.789 [INFO] TestAgent: Node entered maintenance mode +50:51.791 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:54438 latency=1.25582ms +50:51.794 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:54438 latency=199.74µs +50:51.794 [INFO] TestAgent: Requesting shutdown +50:51.794 [INFO] TestAgent.server: shutting down server +50:51.794 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:51.794 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:51.794 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:51.794 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:51.794 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:51.794 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:51.794 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:51.794 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:51.794 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:51.794 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:51.794 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:51.794Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:51.794Z [INFO] agent.router.manager: shutting down +50:51.795 [INFO] TestAgent: consul server down +50:51.795 [INFO] TestAgent: shutdown complete +50:51.795 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp +50:51.795 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp +50:51.795 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp +50:52.295 [INFO] TestAgent: Waiting for endpoints to shut down +50:52.295 [INFO] TestAgent: Endpoints down +--- PASS: TestMaintCommand_NoArgs (0.86s) +2022-12-04T05:50:51.489Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:51.523Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:51.525 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b92f46d2-412b-b938-6a9d-06c8e68a844f Address:127.0.0.1:13076}]" +50:51.525 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b92f46d2-412b-b938-6a9d-06c8e68a844f.dc1 127.0.0.1 +50:51.526 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b92f46d2-412b-b938-6a9d-06c8e68a844f 127.0.0.1 +2022-12-04T05:50:51.526Z [INFO] agent.router: Initializing LAN area manager +50:51.526 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=udp +50:51.526 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13076 [Follower]" leader= +50:51.526 [INFO] TestAgent.server: Adding LAN server: server="Node-b92f46d2-412b-b938-6a9d-06c8e68a844f (Addr: tcp/127.0.0.1:13076) (DC: dc1)" +50:51.526 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b92f46d2-412b-b938-6a9d-06c8e68a844f.dc1 area=wan +50:51.526 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=tcp +50:51.536 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13072 network=tcp +50:51.537 [INFO] TestAgent: started state syncer +50:51.537 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13077 network=tcp +50:51.588 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:51.588 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13076 [Candidate]" term=2 +50:51.588 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:51.588 [DEBUG] TestAgent.server.raft: vote granted: from=b92f46d2-412b-b938-6a9d-06c8e68a844f term=2 tally=1 +50:51.588 [INFO] TestAgent.server.raft: election won: tally=1 +50:51.588 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13076 [Leader]" +50:51.589 [INFO] TestAgent.server: cluster leadership acquired +50:51.589 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13076 +50:51.589 [INFO] TestAgent.server: New leader elected: payload=Node-b92f46d2-412b-b938-6a9d-06c8e68a844f +50:51.589 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:51.591 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:51.591 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:51.591 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:51.591 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:51.591 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:51.591 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b92f46d2-412b-b938-6a9d-06c8e68a844f +50:51.591 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b92f46d2-412b-b938-6a9d-06c8e68a844f +50:51.592 [INFO] TestAgent.server: federation state anti-entropy synced +50:51.794 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:51.794 [INFO] TestAgent: Synced node info +50:51.794 [DEBUG] TestAgent: Node info in sync +50:51.818 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/maintenance/redis?enable=true&reason=broken from=127.0.0.1:59730 latency=158.288µs +50:51.819 [INFO] TestAgent: Requesting shutdown +50:51.819 [INFO] TestAgent.server: shutting down server +50:51.819 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:51.819 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:51.819 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:51.819 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:51.819 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:51.819 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:51.819 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:51.819 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:51.819 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:51.819 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:51.819 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:51.819Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:51.819Z [INFO] agent.router.manager: shutting down +50:51.819 [INFO] TestAgent: consul server down +50:51.819 [INFO] TestAgent: shutdown complete +50:51.819 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=tcp +50:51.819 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=udp +50:51.819 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13072 network=tcp +50:52.319 [INFO] TestAgent: Waiting for endpoints to shut down +50:52.319 [INFO] TestAgent: Endpoints down +--- PASS: TestMaintCommand_ServiceMaintenance_NoService (0.88s) +2022-12-04T05:50:51.494Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:51.509Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:51.521 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3a9fd30c-6d82-998c-b774-997f30d35158 Address:127.0.0.1:13104}]" +50:51.522 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3a9fd30c-6d82-998c-b774-997f30d35158.dc1 127.0.0.1 +50:51.522 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3a9fd30c-6d82-998c-b774-997f30d35158 127.0.0.1 +2022-12-04T05:50:51.522Z [INFO] agent.router: Initializing LAN area manager +50:51.522 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13099 network=udp +50:51.522 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13104 [Follower]" leader= +50:51.523 [INFO] TestAgent.server: Adding LAN server: server="Node-3a9fd30c-6d82-998c-b774-997f30d35158 (Addr: tcp/127.0.0.1:13104) (DC: dc1)" +50:51.523 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3a9fd30c-6d82-998c-b774-997f30d35158.dc1 area=wan +50:51.523 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13099 network=tcp +50:51.524 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13100 network=tcp +50:51.524 [INFO] TestAgent: started state syncer +50:51.524 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13105 network=tcp +50:51.590 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:51.590 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13104 [Candidate]" term=2 +50:51.590 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:51.590 [DEBUG] TestAgent.server.raft: vote granted: from=3a9fd30c-6d82-998c-b774-997f30d35158 term=2 tally=1 +50:51.590 [INFO] TestAgent.server.raft: election won: tally=1 +50:51.590 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13104 [Leader]" +50:51.590 [INFO] TestAgent.server: cluster leadership acquired +50:51.590 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13104 +50:51.590 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:51.591 [INFO] TestAgent.server: New leader elected: payload=Node-3a9fd30c-6d82-998c-b774-997f30d35158 +50:51.591 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:51.591 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:51.591 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:51.591 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:51.591 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:51.592 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3a9fd30c-6d82-998c-b774-997f30d35158 +50:51.592 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3a9fd30c-6d82-998c-b774-997f30d35158 +50:51.592 [INFO] TestAgent.server: federation state anti-entropy synced +50:51.705 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:51.705 [INFO] TestAgent: Synced node info +50:51.939 [INFO] TestAgent: Node entered maintenance mode +50:51.939 [DEBUG] TestAgent: Node info in sync +50:51.940 [INFO] TestAgent: Synced check: check=_node_maintenance +50:51.940 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/maintenance?enable=true&reason=broken from=127.0.0.1:58268 latency=507.858µs +50:51.940 [INFO] TestAgent: Requesting shutdown +50:51.940 [INFO] TestAgent.server: shutting down server +50:51.940 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:51.940 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:51.940 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:51.940 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:51.940 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:51.940 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:51.940 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:51.940 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:51.940 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:51.940 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:51.940 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:51.940Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:51.940Z [INFO] agent.router.manager: shutting down +50:51.940 [INFO] TestAgent: consul server down +50:51.940 [INFO] TestAgent: shutdown complete +50:51.940 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13099 network=tcp +50:51.940 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13099 network=udp +50:51.940 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13100 network=tcp +50:52.441 [INFO] TestAgent: Waiting for endpoints to shut down +50:52.441 [INFO] TestAgent: Endpoints down +--- PASS: TestMaintCommand_EnableNodeMaintenance (1.00s) +2022-12-04T05:50:51.496Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:51.541Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:51.542 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9d490401-d5a4-96f4-1a2f-c06f78c352eb Address:127.0.0.1:13097}]" +50:51.542 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13097 [Follower]" leader= +50:51.568 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9d490401-d5a4-96f4-1a2f-c06f78c352eb.dc1 127.0.0.1 +50:51.569 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9d490401-d5a4-96f4-1a2f-c06f78c352eb 127.0.0.1 +2022-12-04T05:50:51.569Z [INFO] agent.router: Initializing LAN area manager +50:51.569 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9d490401-d5a4-96f4-1a2f-c06f78c352eb.dc1 area=wan +50:51.569 [INFO] TestAgent.server: Adding LAN server: server="Node-9d490401-d5a4-96f4-1a2f-c06f78c352eb (Addr: tcp/127.0.0.1:13097) (DC: dc1)" +50:51.569 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13092 network=udp +50:51.569 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13092 network=tcp +50:51.569 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13093 network=tcp +50:51.569 [INFO] TestAgent: started state syncer +50:51.569 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13098 network=tcp +50:51.602 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:51.603 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13097 [Candidate]" term=2 +50:51.603 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:51.603 [DEBUG] TestAgent.server.raft: vote granted: from=9d490401-d5a4-96f4-1a2f-c06f78c352eb term=2 tally=1 +50:51.603 [INFO] TestAgent.server.raft: election won: tally=1 +50:51.603 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13097 [Leader]" +50:51.603 [INFO] TestAgent.server: cluster leadership acquired +50:51.603 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13097 +50:51.603 [INFO] TestAgent.server: New leader elected: payload=Node-9d490401-d5a4-96f4-1a2f-c06f78c352eb +50:51.604 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:51.605 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:51.605 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:51.605 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:51.605 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:51.605 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:51.605 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9d490401-d5a4-96f4-1a2f-c06f78c352eb +50:51.605 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9d490401-d5a4-96f4-1a2f-c06f78c352eb +50:51.606 [INFO] TestAgent.server: federation state anti-entropy synced +50:51.981 [INFO] TestAgent: Synced node info +50:51.981 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/maintenance?enable=false from=127.0.0.1:57028 latency=279.997µs +50:51.981 [INFO] TestAgent: Requesting shutdown +50:51.982 [INFO] TestAgent.server: shutting down server +50:51.982 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:51.982 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:51.982 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:51.982 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:51.982 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:51.982 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:51.982 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:51.982 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:51.982 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:51.982 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:51.982 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:51.982 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:51.982Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:51.982Z [INFO] agent.router.manager: shutting down +50:51.982 [INFO] TestAgent: consul server down +50:51.982 [INFO] TestAgent: shutdown complete +50:51.982 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13092 network=tcp +50:51.982 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13092 network=udp +50:51.982 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13093 network=tcp +50:52.482 [INFO] TestAgent: Waiting for endpoints to shut down +50:52.482 [INFO] TestAgent: Endpoints down +--- PASS: TestMaintCommand_DisableNodeMaintenance (1.05s) PASS -ok github.com/hashicorp/consul/command/maint 1.884s +ok github.com/hashicorp/consul/command/maint 1.094s === RUN TestMembersCommand_noTabs === PAUSE TestMembersCommand_noTabs === RUN TestMembersCommand @@ -89472,466 +89636,466 @@ === PAUSE TestMembersCommand_verticalBar === CONT TestMembersCommand_noTabs === CONT TestMembersCommand_statusFilter_failed -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestMembersCommand_verticalBar === CONT TestMembersCommand_WAN -=== CONT TestMembersCommand_statusFilter ---- PASS: TestMembersCommand_noTabs (0.01s) +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestMembersCommand +--- PASS: TestMembersCommand_noTabs (0.00s) +=== CONT TestMembersCommand_statusFilter [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:18:28.051Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:28.111Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:28.113 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:78b82f1b-2813-a627-8e5a-6f9bd9f199f5 Address:127.0.0.1:13097}]" -18:28.114 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13097 [Follower]" leader= -18:28.115 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-78b82f1b-2813-a627-8e5a-6f9bd9f199f5.dc1 127.0.0.1 -18:28.116 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-78b82f1b-2813-a627-8e5a-6f9bd9f199f5 127.0.0.1 -2021-10-31T23:18:28.116Z [INFO] agent.router: Initializing LAN area manager -18:28.116 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13092 network=udp -18:28.116 [INFO] TestAgent.server: Adding LAN server: server="Node-78b82f1b-2813-a627-8e5a-6f9bd9f199f5 (Addr: tcp/127.0.0.1:13097) (DC: dc1)" -18:28.116 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-78b82f1b-2813-a627-8e5a-6f9bd9f199f5.dc1 area=wan -18:28.116 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13092 network=tcp -18:28.117 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13093 network=tcp -18:28.117 [INFO] TestAgent: started state syncer -18:28.117 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13098 network=tcp -18:28.159 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:28.159 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13097 [Candidate]" term=2 -18:28.159 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:28.159 [DEBUG] TestAgent.server.raft: vote granted: from=78b82f1b-2813-a627-8e5a-6f9bd9f199f5 term=2 tally=1 -18:28.159 [INFO] TestAgent.server.raft: election won: tally=1 -18:28.159 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13097 [Leader]" -18:28.159 [INFO] TestAgent.server: cluster leadership acquired -18:28.159 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13097 -18:28.160 [INFO] TestAgent.server: New leader elected: payload=Node-78b82f1b-2813-a627-8e5a-6f9bd9f199f5 -18:28.160 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:28.238 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:28.238 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:28.238 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:28.238 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:28.238 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:28.238 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-78b82f1b-2813-a627-8e5a-6f9bd9f199f5 -18:28.238 [INFO] TestAgent.server: member joined, marking health alive: member=Node-78b82f1b-2813-a627-8e5a-6f9bd9f199f5 -18:28.239 [INFO] TestAgent.server: federation state anti-entropy synced -18:28.282 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:28.282 [INFO] TestAgent: Synced node info -18:28.427 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all from=127.0.0.1:41524 latency=357.372µs -18:28.444 [INFO] TestAgent: Requesting shutdown -18:28.445 [INFO] TestAgent.server: shutting down server -18:28.445 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:28.445 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:28.445 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:28.445 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:28.445 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:28.445 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:28.445 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:28.445 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:28.445 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:28.445 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:28.445 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:28.445Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:28.445Z [INFO] agent.router.manager: shutting down -18:28.445 [INFO] TestAgent: consul server down -18:28.445 [INFO] TestAgent: shutdown complete -18:28.445 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13092 network=tcp -18:28.445 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13092 network=udp -18:28.445 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13093 network=tcp -18:28.946 [INFO] TestAgent: Waiting for endpoints to shut down -18:28.946 [INFO] TestAgent: Endpoints down ---- PASS: TestMembersCommand (1.07s) -2021-10-31T23:18:28.014Z [WARN] agent: Node name "name|with|bars" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2021-10-31T23:18:28.014Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:28.117Z [WARN] agent.auto_config: Node name "name|with|bars" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2021-10-31T23:18:28.117Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:28.144 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e1ecad03-21ec-e086-8f7a-915de5f624da Address:127.0.0.1:13076}]" -18:28.144 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: name|with|bars.dc1 127.0.0.1 -18:28.145 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: name|with|bars 127.0.0.1 -2021-10-31T23:18:28.145Z [INFO] agent.router: Initializing LAN area manager -18:28.145 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13076 [Follower]" leader= -18:28.146 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=udp -18:28.156 [INFO] TestAgent.server: Adding LAN server: server="name|with|bars (Addr: tcp/127.0.0.1:13076) (DC: dc1)" -18:28.165 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=name|with|bars.dc1 area=wan -18:28.166 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13071 network=tcp -18:28.166 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13072 network=tcp -18:28.166 [INFO] TestAgent: started state syncer -18:28.166 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13077 network=tcp -18:28.193 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:28.193 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13076 [Candidate]" term=2 -18:28.194 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:28.194 [DEBUG] TestAgent.server.raft: vote granted: from=e1ecad03-21ec-e086-8f7a-915de5f624da term=2 tally=1 -18:28.194 [INFO] TestAgent.server.raft: election won: tally=1 -18:28.194 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13076 [Leader]" -18:28.194 [INFO] TestAgent.server: cluster leadership acquired -18:28.194 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13076 -18:28.195 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:28.195 [INFO] TestAgent.server: New leader elected: payload=name|with|bars -18:28.236 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:28.236 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:28.236 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:28.236 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:28.236 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:28.236 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=name|with|bars -18:28.236 [INFO] TestAgent.server: member joined, marking health alive: member=name|with|bars -18:28.237 [INFO] TestAgent.server: federation state anti-entropy synced -18:28.266 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:28.266 [INFO] TestAgent: Synced node info -18:28.469 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all from=127.0.0.1:43020 latency=1.483319ms -18:28.470 [INFO] TestAgent: Requesting shutdown -18:28.470 [INFO] TestAgent.server: shutting down server -18:28.470 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:28.470 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:28.470 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:28.470 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:28.470 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:28.470 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:28.470 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:28.470 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:28.470 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:28.470 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:28.470 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:28.471Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:28.471Z [INFO] agent.router.manager: shutting down -18:28.471 [INFO] TestAgent: consul server down -18:28.471 [INFO] TestAgent: shutdown complete -18:28.471 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=tcp -18:28.471 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13071 network=udp -18:28.471 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13072 network=tcp -18:28.973 [INFO] TestAgent: Waiting for endpoints to shut down -18:28.973 [INFO] TestAgent: Endpoints down ---- PASS: TestMembersCommand_verticalBar (1.11s) -2021-10-31T23:18:28.114Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:28.236Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:28.237 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0f59acea-e2ed-491e-c8bf-fbe42f198c64 Address:127.0.0.1:13090}]" -18:28.237 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13090 [Follower]" leader= -18:28.238 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0f59acea-e2ed-491e-c8bf-fbe42f198c64.dc1 127.0.0.1 -18:28.238 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0f59acea-e2ed-491e-c8bf-fbe42f198c64 127.0.0.1 -2021-10-31T23:18:28.238Z [INFO] agent.router: Initializing LAN area manager -18:28.238 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13085 network=udp -18:28.239 [INFO] TestAgent.server: Adding LAN server: server="Node-0f59acea-e2ed-491e-c8bf-fbe42f198c64 (Addr: tcp/127.0.0.1:13090) (DC: dc1)" -18:28.239 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0f59acea-e2ed-491e-c8bf-fbe42f198c64.dc1 area=wan -18:28.285 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13085 network=tcp -18:28.286 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13086 network=tcp -18:28.286 [INFO] TestAgent: started state syncer -18:28.286 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13091 network=tcp -18:28.297 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:28.297 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13090 [Candidate]" term=2 -18:28.298 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:28.298 [DEBUG] TestAgent.server.raft: vote granted: from=0f59acea-e2ed-491e-c8bf-fbe42f198c64 term=2 tally=1 -18:28.298 [INFO] TestAgent.server.raft: election won: tally=1 -18:28.298 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13090 [Leader]" -18:28.298 [INFO] TestAgent.server: cluster leadership acquired -18:28.298 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13090 -18:28.298 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:28.300 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:28.300 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:28.300 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:28.300 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:28.300 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:28.300 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0f59acea-e2ed-491e-c8bf-fbe42f198c64 -18:28.300 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0f59acea-e2ed-491e-c8bf-fbe42f198c64 -18:28.300 [INFO] TestAgent.server: New leader elected: payload=Node-0f59acea-e2ed-491e-c8bf-fbe42f198c64 -18:28.300 [INFO] TestAgent.server: federation state anti-entropy synced -18:28.474 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:28.474 [INFO] TestAgent: Synced node info -18:28.482 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all from=127.0.0.1:60990 latency=283.383µs -18:28.482 [INFO] TestAgent: Requesting shutdown -18:28.482 [INFO] TestAgent.server: shutting down server -18:28.482 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:28.482 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:28.482 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:28.482 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:28.482 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:28.483 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:28.483 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:28.483 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:28.483 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:28.483 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:28.483 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:28.483Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:28.483Z [INFO] agent.router.manager: shutting down -18:28.483 [INFO] TestAgent: consul server down -18:28.483 [INFO] TestAgent: shutdown complete -18:28.483 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13085 network=tcp -18:28.483 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13085 network=udp -18:28.483 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13086 network=tcp -18:28.985 [INFO] TestAgent: Waiting for endpoints to shut down -18:28.985 [INFO] TestAgent: Endpoints down ---- PASS: TestMembersCommand_statusFilter (1.12s) -2021-10-31T23:18:28.114Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:28.194Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:28.197 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ebb3961e-e609-7474-8ea7-6bf6b740d6a4 Address:127.0.0.1:13069}]" -18:28.197 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ebb3961e-e609-7474-8ea7-6bf6b740d6a4.dc1 127.0.0.1 -18:28.198 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13069 [Follower]" leader= -18:28.203 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ebb3961e-e609-7474-8ea7-6bf6b740d6a4 127.0.0.1 -2021-10-31T23:18:28.203Z [INFO] agent.router: Initializing LAN area manager -18:28.204 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=udp -18:28.204 [INFO] TestAgent.server: Adding LAN server: server="Node-ebb3961e-e609-7474-8ea7-6bf6b740d6a4 (Addr: tcp/127.0.0.1:13069) (DC: dc1)" -18:28.204 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ebb3961e-e609-7474-8ea7-6bf6b740d6a4.dc1 area=wan -18:28.234 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13064 network=tcp -18:28.234 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13065 network=tcp -18:28.234 [INFO] TestAgent: started state syncer -18:28.235 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13070 network=tcp -18:28.241 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:28.242 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13069 [Candidate]" term=2 -18:28.242 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:28.242 [DEBUG] TestAgent.server.raft: vote granted: from=ebb3961e-e609-7474-8ea7-6bf6b740d6a4 term=2 tally=1 -18:28.242 [INFO] TestAgent.server.raft: election won: tally=1 -18:28.242 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13069 [Leader]" -18:28.242 [INFO] TestAgent.server: cluster leadership acquired -18:28.242 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13069 -18:28.243 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:28.244 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:28.244 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:28.244 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:28.244 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:28.244 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:28.244 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ebb3961e-e609-7474-8ea7-6bf6b740d6a4 -18:28.244 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ebb3961e-e609-7474-8ea7-6bf6b740d6a4 -18:28.245 [INFO] TestAgent.server: New leader elected: payload=Node-ebb3961e-e609-7474-8ea7-6bf6b740d6a4 -18:28.245 [INFO] TestAgent.server: federation state anti-entropy synced -18:28.493 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all from=127.0.0.1:52524 latency=359.83µs -18:28.494 [INFO] TestAgent: Requesting shutdown -18:28.494 [INFO] TestAgent.server: shutting down server -18:28.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:28.494 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:28.494 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:28.494 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:28.494 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:28.494 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:28.494 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:28.494 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -18:28.494 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:28.494 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:28.494 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:28.494 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:28.494Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:28.494Z [INFO] agent.router.manager: shutting down -18:28.494 [INFO] TestAgent: consul server down -18:28.494 [INFO] TestAgent: shutdown complete -18:28.494 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=tcp -18:28.494 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13064 network=udp -18:28.495 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13065 network=tcp -18:28.997 [INFO] TestAgent: Waiting for endpoints to shut down -18:28.997 [INFO] TestAgent: Endpoints down ---- PASS: TestMembersCommand_statusFilter_failed (1.14s) -2021-10-31T23:18:28.039Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:28.141Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:28.155 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f732bee3-2268-e2a8-7ef8-30c2e6f2426f Address:127.0.0.1:13083}]" -18:28.156 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f732bee3-2268-e2a8-7ef8-30c2e6f2426f.dc1 127.0.0.1 -18:28.156 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f732bee3-2268-e2a8-7ef8-30c2e6f2426f 127.0.0.1 -2021-10-31T23:18:28.156Z [INFO] agent.router: Initializing LAN area manager -18:28.156 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13078 network=udp -18:28.157 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:13083 [Follower]" leader= -18:28.158 [INFO] TestAgent.server: Adding LAN server: server="Node-f732bee3-2268-e2a8-7ef8-30c2e6f2426f (Addr: tcp/127.0.0.1:13083) (DC: dc1)" -18:28.158 [INFO] TestAgent: Started DNS server: address=127.0.0.1:13078 network=tcp -18:28.159 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:13079 network=tcp -18:28.159 [INFO] TestAgent: started state syncer -18:28.159 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:13084 network=tcp -18:28.162 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f732bee3-2268-e2a8-7ef8-30c2e6f2426f.dc1 area=wan -18:28.235 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:28.235 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:13083 [Candidate]" term=2 -18:28.236 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:28.236 [DEBUG] TestAgent.server.raft: vote granted: from=f732bee3-2268-e2a8-7ef8-30c2e6f2426f term=2 tally=1 -18:28.236 [INFO] TestAgent.server.raft: election won: tally=1 -18:28.236 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:13083 [Leader]" -18:28.236 [INFO] TestAgent.server: cluster leadership acquired -18:28.236 [INFO] TestAgent.server: New leader elected: payload=Node-f732bee3-2268-e2a8-7ef8-30c2e6f2426f -18:28.236 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:13083 -18:28.237 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:28.239 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:28.239 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:28.239 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:28.239 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:28.239 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:28.239 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f732bee3-2268-e2a8-7ef8-30c2e6f2426f -18:28.239 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f732bee3-2268-e2a8-7ef8-30c2e6f2426f -18:28.239 [INFO] TestAgent.server: federation state anti-entropy synced -18:28.471 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:28.471 [INFO] TestAgent: Synced node info -18:28.571 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all&wan=1 from=127.0.0.1:45326 latency=1.010643ms -18:28.574 [INFO] TestAgent: Requesting shutdown -18:28.574 [INFO] TestAgent.server: shutting down server -18:28.574 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:28.574 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:28.574 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:28.574 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:28.574 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:28.574 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:28.574 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:28.574 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:28.574 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:28.574 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:28.574 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:28.575Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:28.575Z [INFO] agent.router.manager: shutting down -18:28.575 [INFO] TestAgent: consul server down -18:28.575 [INFO] TestAgent: shutdown complete -18:28.575 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13078 network=tcp -18:28.575 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:13078 network=udp -18:28.575 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:13079 network=tcp -18:29.077 [INFO] TestAgent: Waiting for endpoints to shut down -18:29.077 [INFO] TestAgent: Endpoints down ---- PASS: TestMembersCommand_WAN (1.21s) +2022-12-04T05:50:54.210Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:54.232Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:54.238 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bd084820-6f7d-e742-28a5-a5e04683f69a Address:127.0.0.1:29405}]" +50:54.239 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29405 [Follower]" leader= +50:54.241 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bd084820-6f7d-e742-28a5-a5e04683f69a.dc1 127.0.0.1 +50:54.242 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bd084820-6f7d-e742-28a5-a5e04683f69a 127.0.0.1 +2022-12-04T05:50:54.242Z [INFO] agent.router: Initializing LAN area manager +50:54.243 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=udp +50:54.243 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29400 network=tcp +50:54.243 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bd084820-6f7d-e742-28a5-a5e04683f69a.dc1 area=wan +50:54.243 [INFO] TestAgent.server: Adding LAN server: server="Node-bd084820-6f7d-e742-28a5-a5e04683f69a (Addr: tcp/127.0.0.1:29405) (DC: dc1)" +50:54.243 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29401 network=tcp +50:54.243 [INFO] TestAgent: started state syncer +50:54.243 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29406 network=tcp +50:54.309 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:54.309 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29405 [Candidate]" term=2 +50:54.309 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:54.309 [DEBUG] TestAgent.server.raft: vote granted: from=bd084820-6f7d-e742-28a5-a5e04683f69a term=2 tally=1 +50:54.309 [INFO] TestAgent.server.raft: election won: tally=1 +50:54.309 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29405 [Leader]" +50:54.309 [INFO] TestAgent.server: cluster leadership acquired +50:54.309 [INFO] TestAgent.server: New leader elected: payload=Node-bd084820-6f7d-e742-28a5-a5e04683f69a +50:54.309 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29405 +50:54.316 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:54.317 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:54.317 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:54.317 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:54.317 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:54.317 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:54.317 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bd084820-6f7d-e742-28a5-a5e04683f69a +50:54.317 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bd084820-6f7d-e742-28a5-a5e04683f69a +50:54.318 [INFO] TestAgent.server: federation state anti-entropy synced +50:54.384 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all from=127.0.0.1:56172 latency=339.433µs +50:54.384 [INFO] TestAgent: Requesting shutdown +50:54.384 [INFO] TestAgent.server: shutting down server +50:54.384 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:54.384 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:54.384 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:54.384 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:54.384 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:54.384 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:54.384 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:54.384 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:54.384 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:54.384 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:54.384 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:54.384 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:54.384Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:54.384Z [INFO] agent.router.manager: shutting down +50:54.385 [INFO] TestAgent: consul server down +50:54.385 [INFO] TestAgent: shutdown complete +50:54.385 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=tcp +50:54.385 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29400 network=udp +50:54.385 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29401 network=tcp +50:54.885 [INFO] TestAgent: Waiting for endpoints to shut down +50:54.885 [INFO] TestAgent: Endpoints down +--- PASS: TestMembersCommand_statusFilter_failed (0.72s) +2022-12-04T05:50:54.228Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:54.264Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:54.266 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:dce641eb-e4e4-e258-416b-105a6acdc732 Address:127.0.0.1:29419}]" +50:54.266 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-dce641eb-e4e4-e258-416b-105a6acdc732.dc1 127.0.0.1 +50:54.267 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-dce641eb-e4e4-e258-416b-105a6acdc732 127.0.0.1 +2022-12-04T05:50:54.267Z [INFO] agent.router: Initializing LAN area manager +50:54.267 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29414 network=udp +50:54.267 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29419 [Follower]" leader= +50:54.267 [INFO] TestAgent.server: Adding LAN server: server="Node-dce641eb-e4e4-e258-416b-105a6acdc732 (Addr: tcp/127.0.0.1:29419) (DC: dc1)" +50:54.267 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-dce641eb-e4e4-e258-416b-105a6acdc732.dc1 area=wan +50:54.300 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29414 network=tcp +50:54.301 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29415 network=tcp +50:54.301 [INFO] TestAgent: started state syncer +50:54.301 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29420 network=tcp +50:54.325 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:54.325 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29419 [Candidate]" term=2 +50:54.325 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:54.325 [DEBUG] TestAgent.server.raft: vote granted: from=dce641eb-e4e4-e258-416b-105a6acdc732 term=2 tally=1 +50:54.325 [INFO] TestAgent.server.raft: election won: tally=1 +50:54.325 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29419 [Leader]" +50:54.325 [INFO] TestAgent.server: cluster leadership acquired +50:54.325 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29419 +50:54.325 [INFO] TestAgent.server: New leader elected: payload=Node-dce641eb-e4e4-e258-416b-105a6acdc732 +50:54.326 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:54.327 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:54.327 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:54.327 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:54.327 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:54.327 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:54.327 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-dce641eb-e4e4-e258-416b-105a6acdc732 +50:54.327 [INFO] TestAgent.server: member joined, marking health alive: member=Node-dce641eb-e4e4-e258-416b-105a6acdc732 +50:54.328 [INFO] TestAgent.server: federation state anti-entropy synced +50:54.383 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:54.383 [INFO] TestAgent: Synced node info +50:54.383 [DEBUG] TestAgent: Node info in sync +50:54.415 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all&wan=1 from=127.0.0.1:43902 latency=137.375µs +50:54.415 [INFO] TestAgent: Requesting shutdown +50:54.415 [INFO] TestAgent.server: shutting down server +50:54.415 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:54.415 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:54.415 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:54.415 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:54.415 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:54.415 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:54.415 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:54.415 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:54.415 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:54.415 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:54.415 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:54.416Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:54.416Z [INFO] agent.router.manager: shutting down +50:54.416 [INFO] TestAgent: consul server down +50:54.416 [INFO] TestAgent: shutdown complete +50:54.416 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29414 network=tcp +50:54.416 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29414 network=udp +50:54.416 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29415 network=tcp +50:54.916 [INFO] TestAgent: Waiting for endpoints to shut down +50:54.916 [INFO] TestAgent: Endpoints down +--- PASS: TestMembersCommand_WAN (0.75s) +2022-12-04T05:50:54.206Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:54.242Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:54.248 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2b0d840e-c8fa-56ec-81b0-80e4d304f8de Address:127.0.0.1:29433}]" +50:54.248 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29433 [Follower]" leader= +50:54.264 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2b0d840e-c8fa-56ec-81b0-80e4d304f8de.dc1 127.0.0.1 +50:54.265 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2b0d840e-c8fa-56ec-81b0-80e4d304f8de 127.0.0.1 +2022-12-04T05:50:54.265Z [INFO] agent.router: Initializing LAN area manager +50:54.265 [INFO] TestAgent.server: Adding LAN server: server="Node-2b0d840e-c8fa-56ec-81b0-80e4d304f8de (Addr: tcp/127.0.0.1:29433) (DC: dc1)" +50:54.265 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2b0d840e-c8fa-56ec-81b0-80e4d304f8de.dc1 area=wan +50:54.265 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29428 network=udp +50:54.265 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29428 network=tcp +50:54.265 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29429 network=tcp +50:54.265 [INFO] TestAgent: started state syncer +50:54.265 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29434 network=tcp +50:54.319 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:54.319 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29433 [Candidate]" term=2 +50:54.319 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:54.319 [DEBUG] TestAgent.server.raft: vote granted: from=2b0d840e-c8fa-56ec-81b0-80e4d304f8de term=2 tally=1 +50:54.319 [INFO] TestAgent.server.raft: election won: tally=1 +50:54.319 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29433 [Leader]" +50:54.319 [INFO] TestAgent.server: cluster leadership acquired +50:54.319 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29433 +50:54.320 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:54.321 [INFO] TestAgent.server: New leader elected: payload=Node-2b0d840e-c8fa-56ec-81b0-80e4d304f8de +50:54.321 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:54.321 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:54.321 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:54.321 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:54.321 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:54.321 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2b0d840e-c8fa-56ec-81b0-80e4d304f8de +50:54.321 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2b0d840e-c8fa-56ec-81b0-80e4d304f8de +50:54.321 [INFO] TestAgent.server: federation state anti-entropy synced +50:54.413 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:54.414 [INFO] TestAgent: Synced node info +50:54.414 [DEBUG] TestAgent: Node info in sync +50:54.526 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all from=127.0.0.1:43800 latency=124.614µs +50:54.527 [INFO] TestAgent: Requesting shutdown +50:54.527 [INFO] TestAgent.server: shutting down server +50:54.527 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:54.527 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:54.527 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:54.527 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:54.527 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:54.527 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:54.527 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:54.527 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:54.527 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:54.527 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:54.527 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:54.527Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:54.527Z [INFO] agent.router.manager: shutting down +50:54.527 [INFO] TestAgent: consul server down +50:54.527 [INFO] TestAgent: shutdown complete +50:54.527 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29428 network=tcp +50:54.527 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29428 network=udp +50:54.527 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29429 network=tcp +50:55.027 [INFO] TestAgent: Waiting for endpoints to shut down +50:55.027 [INFO] TestAgent: Endpoints down +--- PASS: TestMembersCommand_statusFilter (0.86s) +2022-12-04T05:50:54.204Z [WARN] agent: Node name "name|with|bars" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2022-12-04T05:50:54.204Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:54.231Z [WARN] agent.auto_config: Node name "name|with|bars" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2022-12-04T05:50:54.231Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:54.241 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1f2f1d2c-0d51-5e49-7731-138a9f8eddf2 Address:127.0.0.1:29412}]" +50:54.241 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29412 [Follower]" leader= +50:54.242 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: name|with|bars.dc1 127.0.0.1 +50:54.243 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: name|with|bars 127.0.0.1 +2022-12-04T05:50:54.243Z [INFO] agent.router: Initializing LAN area manager +50:54.243 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29407 network=udp +50:54.243 [INFO] TestAgent.server: Adding LAN server: server="name|with|bars (Addr: tcp/127.0.0.1:29412) (DC: dc1)" +50:54.243 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=name|with|bars.dc1 area=wan +50:54.243 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29407 network=tcp +50:54.248 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29408 network=tcp +50:54.252 [INFO] TestAgent: started state syncer +50:54.252 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29413 network=tcp +50:54.292 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:54.292 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29412 [Candidate]" term=2 +50:54.292 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:54.292 [DEBUG] TestAgent.server.raft: vote granted: from=1f2f1d2c-0d51-5e49-7731-138a9f8eddf2 term=2 tally=1 +50:54.292 [INFO] TestAgent.server.raft: election won: tally=1 +50:54.292 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29412 [Leader]" +50:54.292 [INFO] TestAgent.server: cluster leadership acquired +50:54.293 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29412 +50:54.293 [INFO] TestAgent.server: New leader elected: payload=name|with|bars +50:54.293 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:54.309 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:54.309 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:54.309 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:54.309 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:54.309 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:54.309 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=name|with|bars +50:54.309 [INFO] TestAgent.server: member joined, marking health alive: member=name|with|bars +50:54.310 [INFO] TestAgent.server: federation state anti-entropy synced +50:54.654 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all from=127.0.0.1:34430 latency=766.753µs +50:54.654 [INFO] TestAgent: Requesting shutdown +50:54.654 [INFO] TestAgent.server: shutting down server +50:54.654 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:54.654 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:54.654 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:54.654 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:54.654 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:54.654 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:54.654 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:54.654 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:54.654 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:54.654 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:54.654 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:54.654 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:54.655Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:54.655Z [INFO] agent.router.manager: shutting down +50:54.655 [INFO] TestAgent: consul server down +50:54.655 [INFO] TestAgent: shutdown complete +50:54.655 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29407 network=tcp +50:54.655 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29407 network=udp +50:54.655 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29408 network=tcp +50:55.155 [INFO] TestAgent: Waiting for endpoints to shut down +50:55.155 [INFO] TestAgent: Endpoints down +--- PASS: TestMembersCommand_verticalBar (0.99s) +2022-12-04T05:50:54.226Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:54.251Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:54.254 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:469759c9-9655-e4d6-2f7a-fd6ac7d4bf79 Address:127.0.0.1:29426}]" +50:54.254 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:29426 [Follower]" leader= +50:54.263 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-469759c9-9655-e4d6-2f7a-fd6ac7d4bf79.dc1 127.0.0.1 +50:54.264 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-469759c9-9655-e4d6-2f7a-fd6ac7d4bf79 127.0.0.1 +2022-12-04T05:50:54.264Z [INFO] agent.router: Initializing LAN area manager +50:54.264 [INFO] TestAgent.server: Adding LAN server: server="Node-469759c9-9655-e4d6-2f7a-fd6ac7d4bf79 (Addr: tcp/127.0.0.1:29426) (DC: dc1)" +50:54.264 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29421 network=udp +50:54.264 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-469759c9-9655-e4d6-2f7a-fd6ac7d4bf79.dc1 area=wan +50:54.264 [INFO] TestAgent: Started DNS server: address=127.0.0.1:29421 network=tcp +50:54.264 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:29422 network=tcp +50:54.264 [INFO] TestAgent: started state syncer +50:54.265 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:29427 network=tcp +50:54.293 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:54.293 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:29426 [Candidate]" term=2 +50:54.293 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:54.293 [DEBUG] TestAgent.server.raft: vote granted: from=469759c9-9655-e4d6-2f7a-fd6ac7d4bf79 term=2 tally=1 +50:54.293 [INFO] TestAgent.server.raft: election won: tally=1 +50:54.293 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:29426 [Leader]" +50:54.293 [INFO] TestAgent.server: cluster leadership acquired +50:54.294 [INFO] TestAgent.server: New leader elected: payload=Node-469759c9-9655-e4d6-2f7a-fd6ac7d4bf79 +50:54.294 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:29426 +50:54.295 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:54.309 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:54.309 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:54.309 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:54.309 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:54.309 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:54.309 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-469759c9-9655-e4d6-2f7a-fd6ac7d4bf79 +50:54.309 [INFO] TestAgent.server: member joined, marking health alive: member=Node-469759c9-9655-e4d6-2f7a-fd6ac7d4bf79 +50:54.315 [INFO] TestAgent.server: federation state anti-entropy synced +50:54.413 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:54.414 [INFO] TestAgent: Synced node info +50:54.656 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/members?segment=_all from=127.0.0.1:41764 latency=125.701µs +50:54.656 [INFO] TestAgent: Requesting shutdown +50:54.656 [INFO] TestAgent.server: shutting down server +50:54.656 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:54.656 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:54.656 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:54.656 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:54.656 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:54.656 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:54.656 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:54.656 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:54.656 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:54.656 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:54.656 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:54.656Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:54.656Z [INFO] agent.router.manager: shutting down +50:54.656 [INFO] TestAgent: consul server down +50:54.656 [INFO] TestAgent: shutdown complete +50:54.656 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29421 network=tcp +50:54.656 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:29421 network=udp +50:54.656 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:29422 network=tcp +50:55.157 [INFO] TestAgent: Waiting for endpoints to shut down +50:55.157 [INFO] TestAgent: Endpoints down +--- PASS: TestMembersCommand (0.99s) PASS -ok github.com/hashicorp/consul/command/members 1.408s +ok github.com/hashicorp/consul/command/members 1.043s === RUN TestOperatorCommand_noTabs === PAUSE TestOperatorCommand_noTabs === CONT TestOperatorCommand_noTabs --- PASS: TestOperatorCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/operator 0.040s +ok github.com/hashicorp/consul/command/operator 0.005s === RUN TestOperatorAutopilotCommand_noTabs === PAUSE TestOperatorAutopilotCommand_noTabs === CONT TestOperatorAutopilotCommand_noTabs --- PASS: TestOperatorAutopilotCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/operator/autopilot 0.019s +ok github.com/hashicorp/consul/command/operator/autopilot 0.005s === RUN TestOperatorAutopilotGetConfigCommand_noTabs === PAUSE TestOperatorAutopilotGetConfigCommand_noTabs === RUN TestOperatorAutopilotGetConfigCommand === PAUSE TestOperatorAutopilotGetConfigCommand === CONT TestOperatorAutopilotGetConfigCommand_noTabs ---- PASS: TestOperatorAutopilotGetConfigCommand_noTabs (0.00s) === CONT TestOperatorAutopilotGetConfigCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestOperatorAutopilotGetConfigCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:18:31.703Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:31.718Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:31.724 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:2818ef40-d50c-81d7-f9b0-84ba44971850 Address:127.0.0.1:11027}]" -18:31.727 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:11027 [Follower]" leader= -18:31.738 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-2818ef40-d50c-81d7-f9b0-84ba44971850.dc1 127.0.0.1 -18:31.739 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-2818ef40-d50c-81d7-f9b0-84ba44971850 127.0.0.1 -2021-10-31T23:18:31.739Z [INFO] agent.router: Initializing LAN area manager -18:31.739 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=udp -18:31.746 [INFO] TestAgent.server: Adding LAN server: server="Node-2818ef40-d50c-81d7-f9b0-84ba44971850 (Addr: tcp/127.0.0.1:11027) (DC: dc1)" -18:31.746 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-2818ef40-d50c-81d7-f9b0-84ba44971850.dc1 area=wan -18:31.746 [INFO] TestAgent: Started DNS server: address=127.0.0.1:11022 network=tcp -18:31.747 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:11023 network=tcp -18:31.747 [INFO] TestAgent: started state syncer -18:31.747 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:11028 network=tcp -18:31.797 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:31.797 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:11027 [Candidate]" term=2 -18:31.798 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:31.798 [DEBUG] TestAgent.server.raft: vote granted: from=2818ef40-d50c-81d7-f9b0-84ba44971850 term=2 tally=1 -18:31.798 [INFO] TestAgent.server.raft: election won: tally=1 -18:31.798 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:11027 [Leader]" -18:31.798 [INFO] TestAgent.server: cluster leadership acquired -18:31.798 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11027 -18:31.799 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:31.801 [INFO] TestAgent.server: New leader elected: payload=Node-2818ef40-d50c-81d7-f9b0-84ba44971850 -18:31.830 [INFO] TestAgent: Synced node info -18:31.830 [DEBUG] TestAgent: Node info in sync -18:31.835 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:31.835 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:31.835 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:31.835 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:31.835 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:31.835 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-2818ef40-d50c-81d7-f9b0-84ba44971850 -18:31.835 [INFO] TestAgent.server: member joined, marking health alive: member=Node-2818ef40-d50c-81d7-f9b0-84ba44971850 -18:31.836 [INFO] TestAgent.server: federation state anti-entropy synced -18:32.014 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/autopilot/configuration from=127.0.0.1:44322 latency=1.150277ms -18:32.015 [INFO] TestAgent: Requesting shutdown -18:32.015 [INFO] TestAgent.server: shutting down server -18:32.015 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:32.015 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:32.015 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:32.015 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:32.015 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:32.015 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:32.015 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:32.015 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:32.015 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:32.015 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:32.015 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:32.015Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:32.015Z [INFO] agent.router.manager: shutting down -18:32.015 [INFO] TestAgent: consul server down -18:32.015 [INFO] TestAgent: shutdown complete -18:32.015 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=tcp -18:32.015 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:11022 network=udp -18:32.016 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:11023 network=tcp -18:32.517 [INFO] TestAgent: Waiting for endpoints to shut down -18:32.517 [INFO] TestAgent: Endpoints down ---- PASS: TestOperatorAutopilotGetConfigCommand (0.87s) +2022-12-04T05:50:55.932Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:55.942Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:55.943 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:0d060fa1-dc7a-cb97-e8da-ee8141ba06e9 Address:127.0.0.1:16132}]" +50:55.944 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16132 [Follower]" leader= +50:55.945 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-0d060fa1-dc7a-cb97-e8da-ee8141ba06e9.dc1 127.0.0.1 +50:55.949 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-0d060fa1-dc7a-cb97-e8da-ee8141ba06e9 127.0.0.1 +2022-12-04T05:50:55.950Z [INFO] agent.router: Initializing LAN area manager +50:55.950 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-0d060fa1-dc7a-cb97-e8da-ee8141ba06e9.dc1 area=wan +50:55.950 [INFO] TestAgent.server: Adding LAN server: server="Node-0d060fa1-dc7a-cb97-e8da-ee8141ba06e9 (Addr: tcp/127.0.0.1:16132) (DC: dc1)" +50:55.950 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=tcp +50:55.950 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=udp +50:55.951 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16128 network=tcp +50:55.951 [INFO] TestAgent: started state syncer +50:55.952 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16133 network=tcp +50:56.012 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:56.013 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16132 [Candidate]" term=2 +50:56.013 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:56.013 [DEBUG] TestAgent.server.raft: vote granted: from=0d060fa1-dc7a-cb97-e8da-ee8141ba06e9 term=2 tally=1 +50:56.013 [INFO] TestAgent.server.raft: election won: tally=1 +50:56.013 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16132 [Leader]" +50:56.013 [INFO] TestAgent.server: cluster leadership acquired +50:56.013 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16132 +50:56.013 [INFO] TestAgent.server: New leader elected: payload=Node-0d060fa1-dc7a-cb97-e8da-ee8141ba06e9 +50:56.014 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:56.038 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:56.038 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:56.038 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:56.038 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:56.038 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:56.038 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-0d060fa1-dc7a-cb97-e8da-ee8141ba06e9 +50:56.038 [INFO] TestAgent.server: member joined, marking health alive: member=Node-0d060fa1-dc7a-cb97-e8da-ee8141ba06e9 +50:56.038 [INFO] TestAgent.server: federation state anti-entropy synced +50:56.050 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +50:56.051 [INFO] TestAgent: Synced node info +50:56.329 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/autopilot/configuration from=127.0.0.1:40532 latency=418.669µs +50:56.330 [INFO] TestAgent: Requesting shutdown +50:56.330 [INFO] TestAgent.server: shutting down server +50:56.330 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:56.330 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:56.330 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:56.330 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:56.330 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:56.330 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:56.330 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:56.330 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:56.330 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:56.330 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:56.330 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:56.330Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:56.330Z [INFO] agent.router.manager: shutting down +50:56.331 [INFO] TestAgent: consul server down +50:56.331 [INFO] TestAgent: shutdown complete +50:56.331 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=tcp +50:56.331 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=udp +50:56.331 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16128 network=tcp +50:56.831 [INFO] TestAgent: Waiting for endpoints to shut down +50:56.831 [INFO] TestAgent: Endpoints down +--- PASS: TestOperatorAutopilotGetConfigCommand (0.94s) PASS -ok github.com/hashicorp/consul/command/operator/autopilot/get 1.186s +ok github.com/hashicorp/consul/command/operator/autopilot/get 0.992s === RUN TestOperatorAutopilotSetConfigCommand_noTabs === PAUSE TestOperatorAutopilotSetConfigCommand_noTabs === RUN TestOperatorAutopilotSetConfigCommand === PAUSE TestOperatorAutopilotSetConfigCommand === CONT TestOperatorAutopilotSetConfigCommand_noTabs ---- PASS: TestOperatorAutopilotSetConfigCommand_noTabs (0.00s) === CONT TestOperatorAutopilotSetConfigCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestOperatorAutopilotSetConfigCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:18:32.048Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:32.071Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:32.074 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:85599081-4ab0-86b3-7640-e90a43f9fe9e Address:127.0.0.1:26342}]" -18:32.074 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:26342 [Follower]" leader= -18:32.074 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-85599081-4ab0-86b3-7640-e90a43f9fe9e.dc1 127.0.0.1 -18:32.075 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-85599081-4ab0-86b3-7640-e90a43f9fe9e 127.0.0.1 -2021-10-31T23:18:32.075Z [INFO] agent.router: Initializing LAN area manager -18:32.075 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26337 network=udp -18:32.075 [INFO] TestAgent.server: Adding LAN server: server="Node-85599081-4ab0-86b3-7640-e90a43f9fe9e (Addr: tcp/127.0.0.1:26342) (DC: dc1)" -18:32.076 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-85599081-4ab0-86b3-7640-e90a43f9fe9e.dc1 area=wan -18:32.076 [INFO] TestAgent: Started DNS server: address=127.0.0.1:26337 network=tcp -18:32.076 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:26338 network=tcp -18:32.076 [INFO] TestAgent: started state syncer -18:32.076 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:26343 network=tcp -18:32.141 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:32.142 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:26342 [Candidate]" term=2 -18:32.142 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:32.142 [DEBUG] TestAgent.server.raft: vote granted: from=85599081-4ab0-86b3-7640-e90a43f9fe9e term=2 tally=1 -18:32.142 [INFO] TestAgent.server.raft: election won: tally=1 -18:32.142 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:26342 [Leader]" -18:32.142 [INFO] TestAgent.server: cluster leadership acquired -18:32.142 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:26342 -18:32.143 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:32.145 [INFO] TestAgent.server: New leader elected: payload=Node-85599081-4ab0-86b3-7640-e90a43f9fe9e -18:32.175 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:32.175 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:32.175 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:32.175 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:32.175 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:32.175 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-85599081-4ab0-86b3-7640-e90a43f9fe9e -18:32.175 [INFO] TestAgent.server: member joined, marking health alive: member=Node-85599081-4ab0-86b3-7640-e90a43f9fe9e -18:32.176 [INFO] TestAgent.server: federation state anti-entropy synced -18:32.214 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:32.214 [INFO] TestAgent: Synced node info -18:32.279 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/autopilot/configuration from=127.0.0.1:33856 latency=8.386562ms -18:32.296 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/autopilot/configuration?cas=4 from=127.0.0.1:33856 latency=705.59µs -18:32.314 [INFO] TestAgent: Requesting shutdown -18:32.314 [INFO] TestAgent.server: shutting down server -18:32.314 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:32.314 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:32.314 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:32.314 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:32.314 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:32.314 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:32.314 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:32.314 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:32.314 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:32.314 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:32.314 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:32.314Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:32.314Z [INFO] agent.router.manager: shutting down -18:32.314 [INFO] TestAgent: consul server down -18:32.314 [INFO] TestAgent: shutdown complete -18:32.314 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26337 network=tcp -18:32.314 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:26337 network=udp -18:32.314 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:26338 network=tcp -18:32.817 [INFO] TestAgent: Waiting for endpoints to shut down -18:32.817 [INFO] TestAgent: Endpoints down ---- PASS: TestOperatorAutopilotSetConfigCommand (0.82s) +2022-12-04T05:50:57.958Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:50:57.973Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +50:57.975 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:390eeb52-01e5-a546-636d-7e90ea724c04 Address:127.0.0.1:19195}]" +50:57.975 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= +50:57.976 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-390eeb52-01e5-a546-636d-7e90ea724c04.dc1 127.0.0.1 +50:57.976 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-390eeb52-01e5-a546-636d-7e90ea724c04 127.0.0.1 +2022-12-04T05:50:57.976Z [INFO] agent.router: Initializing LAN area manager +50:57.977 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp +50:57.977 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-390eeb52-01e5-a546-636d-7e90ea724c04.dc1 area=wan +50:57.977 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp +50:57.977 [INFO] TestAgent.server: Adding LAN server: server="Node-390eeb52-01e5-a546-636d-7e90ea724c04 (Addr: tcp/127.0.0.1:19195) (DC: dc1)" +50:57.977 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp +50:57.977 [INFO] TestAgent: started state syncer +50:57.978 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp +50:58.030 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +50:58.030 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 +50:58.030 [DEBUG] TestAgent.server.raft: votes: needed=1 +50:58.030 [DEBUG] TestAgent.server.raft: vote granted: from=390eeb52-01e5-a546-636d-7e90ea724c04 term=2 tally=1 +50:58.030 [INFO] TestAgent.server.raft: election won: tally=1 +50:58.030 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" +50:58.030 [INFO] TestAgent.server: cluster leadership acquired +50:58.030 [INFO] TestAgent.server: New leader elected: payload=Node-390eeb52-01e5-a546-636d-7e90ea724c04 +50:58.030 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 +50:58.031 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +50:58.047 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +50:58.047 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +50:58.047 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +50:58.047 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +50:58.047 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +50:58.047 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-390eeb52-01e5-a546-636d-7e90ea724c04 +50:58.047 [INFO] TestAgent.server: member joined, marking health alive: member=Node-390eeb52-01e5-a546-636d-7e90ea724c04 +50:58.048 [INFO] TestAgent.server: federation state anti-entropy synced +50:58.272 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/autopilot/configuration from=127.0.0.1:40370 latency=1.232665ms +50:58.275 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/operator/autopilot/configuration?cas=4 from=127.0.0.1:40370 latency=2.425141ms +50:58.276 [INFO] TestAgent: Requesting shutdown +50:58.276 [INFO] TestAgent.server: shutting down server +50:58.276 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +50:58.276 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +50:58.276 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +50:58.276 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +50:58.276 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +50:58.276 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +50:58.276 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +50:58.276 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +50:58.276 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +50:58.276 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +50:58.276 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +50:58.276 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:50:58.276Z [INFO] agent.router.manager: shutting down +2022-12-04T05:50:58.276Z [INFO] agent.router.manager: shutting down +50:58.276 [INFO] TestAgent: consul server down +50:58.276 [INFO] TestAgent: shutdown complete +50:58.276 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp +50:58.276 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp +50:58.276 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp +50:58.776 [INFO] TestAgent: Waiting for endpoints to shut down +50:58.777 [INFO] TestAgent: Endpoints down +--- PASS: TestOperatorAutopilotSetConfigCommand (0.85s) PASS -ok github.com/hashicorp/consul/command/operator/autopilot/set 0.912s +ok github.com/hashicorp/consul/command/operator/autopilot/set 0.903s === RUN TestOperatorRaftCommand_noTabs === PAUSE TestOperatorRaftCommand_noTabs === CONT TestOperatorRaftCommand_noTabs --- PASS: TestOperatorRaftCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/operator/raft 0.031s +ok github.com/hashicorp/consul/command/operator/raft 0.005s === RUN TestOperatorRaftListPeersCommand_noTabs === PAUSE TestOperatorRaftListPeersCommand_noTabs === RUN TestOperatorRaftListPeersCommand @@ -89939,132 +90103,131 @@ === RUN TestOperatorRaftListPeersCommand_verticalBar === PAUSE TestOperatorRaftListPeersCommand_verticalBar === CONT TestOperatorRaftListPeersCommand_noTabs ---- PASS: TestOperatorRaftListPeersCommand_noTabs (0.00s) === CONT TestOperatorRaftListPeersCommand_verticalBar === CONT TestOperatorRaftListPeersCommand [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestOperatorRaftListPeersCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:18:35.164Z [WARN] agent: Node name "name|with|bars" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2021-10-31T23:18:35.164Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:35.179Z [WARN] agent.auto_config: Node name "name|with|bars" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. -2021-10-31T23:18:35.179Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:35.183 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b5625b3c-0db9-d6f2-3119-955cf1edbb75 Address:127.0.0.1:19195}]" -18:35.183 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: name|with|bars.dc1 127.0.0.1 -18:35.184 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: name|with|bars 127.0.0.1 -2021-10-31T23:18:35.184Z [INFO] agent.router: Initializing LAN area manager -18:35.184 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=udp -18:35.184 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19195 [Follower]" leader= -18:35.184 [INFO] TestAgent.server: Adding LAN server: server="name|with|bars (Addr: tcp/127.0.0.1:19195) (DC: dc1)" -18:35.184 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=name|with|bars.dc1 area=wan -18:35.185 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19190 network=tcp -18:35.185 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19191 network=tcp -18:35.185 [INFO] TestAgent: started state syncer -18:35.185 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19196 network=tcp -18:35.219 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:35.220 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19195 [Candidate]" term=2 -18:35.220 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:35.220 [DEBUG] TestAgent.server.raft: vote granted: from=b5625b3c-0db9-d6f2-3119-955cf1edbb75 term=2 tally=1 -18:35.220 [INFO] TestAgent.server.raft: election won: tally=1 -18:35.220 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19195 [Leader]" -18:35.220 [INFO] TestAgent.server: cluster leadership acquired -18:35.220 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19195 -18:35.221 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:35.221 [INFO] TestAgent.server: New leader elected: payload=name|with|bars -18:35.238 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:35.238 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:35.238 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:35.238 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:35.238 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:35.238 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=name|with|bars -18:35.238 [INFO] TestAgent.server: member joined, marking health alive: member=name|with|bars -18:35.239 [INFO] TestAgent.server: federation state anti-entropy synced -18:35.326 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:35.326 [INFO] TestAgent: Synced node info -18:35.326 [DEBUG] TestAgent: Node info in sync -18:35.499 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/raft/configuration from=127.0.0.1:35408 latency=4.803975ms -18:35.504 [INFO] TestAgent: Requesting shutdown -18:35.504 [INFO] TestAgent.server: shutting down server -18:35.504 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:35.504 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:35.504 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:35.504 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:35.504 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:35.505 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:35.505 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:35.505 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:35.505 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:35.505 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:35.505 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:35.505Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:35.505Z [INFO] agent.router.manager: shutting down -18:35.505 [INFO] TestAgent: consul server down -18:35.505 [INFO] TestAgent: shutdown complete -18:35.505 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=tcp -18:35.505 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19190 network=udp -18:35.505 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19191 network=tcp -18:36.005 [INFO] TestAgent: Waiting for endpoints to shut down -18:36.005 [INFO] TestAgent: Endpoints down ---- PASS: TestOperatorRaftListPeersCommand_verticalBar (0.90s) -2021-10-31T23:18:35.187Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:35.220Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:35.222 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d1eecd4b-ca1b-2327-dfd6-1635a570c23c Address:127.0.0.1:19202}]" -18:35.222 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d1eecd4b-ca1b-2327-dfd6-1635a570c23c.dc1 127.0.0.1 -18:35.223 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d1eecd4b-ca1b-2327-dfd6-1635a570c23c 127.0.0.1 -2021-10-31T23:18:35.223Z [INFO] agent.router: Initializing LAN area manager -18:35.223 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=udp -18:35.223 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:19202 [Follower]" leader= -18:35.223 [INFO] TestAgent.server: Adding LAN server: server="Node-d1eecd4b-ca1b-2327-dfd6-1635a570c23c (Addr: tcp/127.0.0.1:19202) (DC: dc1)" -18:35.223 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d1eecd4b-ca1b-2327-dfd6-1635a570c23c.dc1 area=wan -18:35.224 [INFO] TestAgent: Started DNS server: address=127.0.0.1:19197 network=tcp -18:35.224 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:19198 network=tcp -18:35.224 [INFO] TestAgent: started state syncer -18:35.224 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:19203 network=tcp -18:35.277 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:35.277 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:19202 [Candidate]" term=2 -18:35.278 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:35.278 [DEBUG] TestAgent.server.raft: vote granted: from=d1eecd4b-ca1b-2327-dfd6-1635a570c23c term=2 tally=1 -18:35.278 [INFO] TestAgent.server.raft: election won: tally=1 -18:35.278 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:19202 [Leader]" -18:35.278 [INFO] TestAgent.server: cluster leadership acquired -18:35.278 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:19202 -18:35.279 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:35.280 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:35.280 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:35.280 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:35.280 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:35.280 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:35.280 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d1eecd4b-ca1b-2327-dfd6-1635a570c23c -18:35.280 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d1eecd4b-ca1b-2327-dfd6-1635a570c23c -18:35.280 [INFO] TestAgent.server: New leader elected: payload=Node-d1eecd4b-ca1b-2327-dfd6-1635a570c23c -18:35.281 [INFO] TestAgent.server: federation state anti-entropy synced -18:35.516 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/raft/configuration from=127.0.0.1:50168 latency=1.179824ms -18:35.516 [INFO] TestAgent: Requesting shutdown -18:35.516 [INFO] TestAgent.server: shutting down server -18:35.516 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:35.516 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:35.516 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:35.516 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:35.516 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:35.516 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:35.516 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:35.516 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:35.516 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -18:35.516 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -18:35.516 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:35.516 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -2021-10-31T23:18:35.516Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:35.517Z [INFO] agent.router.manager: shutting down -18:35.517 [INFO] TestAgent: consul server down -18:35.517 [INFO] TestAgent: shutdown complete -18:35.517 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=tcp -18:35.517 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:19197 network=udp -18:35.517 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:19198 network=tcp -18:36.017 [INFO] TestAgent: Waiting for endpoints to shut down -18:36.017 [INFO] TestAgent: Endpoints down ---- PASS: TestOperatorRaftListPeersCommand (0.91s) +2022-12-04T05:51:01.139Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:01.151Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:01.156 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:caac34c5-1c3a-216f-1db9-63062af29341 Address:127.0.0.1:14097}]" +51:01.157 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14097 [Follower]" leader= +51:01.158 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-caac34c5-1c3a-216f-1db9-63062af29341.dc1 127.0.0.1 +51:01.158 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-caac34c5-1c3a-216f-1db9-63062af29341 127.0.0.1 +2022-12-04T05:51:01.158Z [INFO] agent.router: Initializing LAN area manager +51:01.158 [INFO] TestAgent.server: Adding LAN server: server="Node-caac34c5-1c3a-216f-1db9-63062af29341 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +51:01.158 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-caac34c5-1c3a-216f-1db9-63062af29341.dc1 area=wan +51:01.158 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=udp +51:01.158 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=tcp +51:01.159 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14093 network=tcp +51:01.159 [INFO] TestAgent: started state syncer +51:01.159 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14098 network=tcp +51:01.216 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:01.216 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14097 [Candidate]" term=2 +51:01.216 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:01.216 [DEBUG] TestAgent.server.raft: vote granted: from=caac34c5-1c3a-216f-1db9-63062af29341 term=2 tally=1 +51:01.216 [INFO] TestAgent.server.raft: election won: tally=1 +51:01.216 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14097 [Leader]" +51:01.216 [INFO] TestAgent.server: cluster leadership acquired +51:01.216 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14097 +51:01.216 [INFO] TestAgent.server: New leader elected: payload=Node-caac34c5-1c3a-216f-1db9-63062af29341 +51:01.217 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:01.227 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:01.227 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:01.227 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:01.227 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:01.227 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:01.227 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-caac34c5-1c3a-216f-1db9-63062af29341 +51:01.227 [INFO] TestAgent.server: member joined, marking health alive: member=Node-caac34c5-1c3a-216f-1db9-63062af29341 +51:01.228 [INFO] TestAgent.server: federation state anti-entropy synced +51:01.234 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/raft/configuration from=127.0.0.1:32772 latency=1.802475ms +51:01.236 [INFO] TestAgent: Requesting shutdown +51:01.236 [INFO] TestAgent.server: shutting down server +51:01.236 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:01.236 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:01.236 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:01.236 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:01.237 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:01.237 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:01.237 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:01.237 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +51:01.237 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:01.237 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:01.237 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:01.237 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:01.237Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:01.237Z [INFO] agent.router.manager: shutting down +51:01.237 [INFO] TestAgent: consul server down +51:01.237 [INFO] TestAgent: shutdown complete +51:01.237 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=tcp +51:01.237 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=udp +51:01.237 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14093 network=tcp +51:01.738 [INFO] TestAgent: Waiting for endpoints to shut down +51:01.738 [INFO] TestAgent: Endpoints down +--- PASS: TestOperatorRaftListPeersCommand (0.63s) +2022-12-04T05:51:01.140Z [WARN] agent: Node name "name|with|bars" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2022-12-04T05:51:01.140Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:01.150Z [WARN] agent.auto_config: Node name "name|with|bars" will not be discoverable via DNS due to invalid characters. Valid characters include all alpha-numerics and dashes. +2022-12-04T05:51:01.150Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:01.156 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1f2e8829-c7f3-5744-5a21-08e5a37da1b0 Address:127.0.0.1:14090}]" +51:01.157 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= +51:01.157 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: name|with|bars.dc1 127.0.0.1 +51:01.158 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: name|with|bars 127.0.0.1 +2022-12-04T05:51:01.158Z [INFO] agent.router: Initializing LAN area manager +51:01.158 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=name|with|bars.dc1 area=wan +51:01.158 [INFO] TestAgent.server: Adding LAN server: server="name|with|bars (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +51:01.158 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp +51:01.158 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp +51:01.159 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp +51:01.159 [INFO] TestAgent: started state syncer +51:01.159 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp +51:01.208 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:01.208 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 +51:01.208 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:01.208 [DEBUG] TestAgent.server.raft: vote granted: from=1f2e8829-c7f3-5744-5a21-08e5a37da1b0 term=2 tally=1 +51:01.208 [INFO] TestAgent.server.raft: election won: tally=1 +51:01.208 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" +51:01.209 [INFO] TestAgent.server: cluster leadership acquired +51:01.209 [INFO] TestAgent.server: New leader elected: payload=name|with|bars +51:01.209 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14090 +51:01.209 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:01.225 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:01.225 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:01.225 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:01.225 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:01.225 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:01.225 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=name|with|bars +51:01.225 [INFO] TestAgent.server: member joined, marking health alive: member=name|with|bars +51:01.227 [INFO] TestAgent.server: federation state anti-entropy synced +51:01.431 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +51:01.431 [INFO] TestAgent: Synced node info +51:01.510 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/operator/raft/configuration from=127.0.0.1:45804 latency=235.82µs +51:01.510 [INFO] TestAgent: Requesting shutdown +51:01.510 [INFO] TestAgent.server: shutting down server +51:01.510 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:01.510 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:01.510 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:01.510 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:01.510 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:01.510 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:01.511 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:01.511 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:01.511 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:01.511 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:01.511 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:01.511Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:01.511Z [INFO] agent.router.manager: shutting down +51:01.511 [INFO] TestAgent: consul server down +51:01.511 [INFO] TestAgent: shutdown complete +51:01.511 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp +51:01.511 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp +51:01.511 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp +51:02.011 [INFO] TestAgent: Waiting for endpoints to shut down +51:02.011 [INFO] TestAgent: Endpoints down +--- PASS: TestOperatorRaftListPeersCommand_verticalBar (0.91s) PASS -ok github.com/hashicorp/consul/command/operator/raft/listpeers 1.012s +ok github.com/hashicorp/consul/command/operator/raft/listpeers 0.954s === RUN TestOperatorRaftRemovePeerCommand_noTabs === PAUSE TestOperatorRaftRemovePeerCommand_noTabs === RUN TestOperatorRaftRemovePeerCommand @@ -90077,71 +90240,71 @@ [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestOperatorRaftRemovePeerCommand/Test_the_remove-peer_subcommand_directly === RUN TestOperatorRaftRemovePeerCommand/Test_the_remove-peer_subcommand_with_-id -2021-10-31T23:18:37.716Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:37.728Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:37.731 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ccac7ac7-1b74-ad81-a9a7-1c2fbade8e4d Address:127.0.0.1:12048}]" -18:37.731 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12048 [Follower]" leader= -18:37.732 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ccac7ac7-1b74-ad81-a9a7-1c2fbade8e4d.dc1 127.0.0.1 -18:37.746 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ccac7ac7-1b74-ad81-a9a7-1c2fbade8e4d 127.0.0.1 -2021-10-31T23:18:37.746Z [INFO] agent.router: Initializing LAN area manager -18:37.746 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=udp -18:37.746 [INFO] TestAgent.server: Adding LAN server: server="Node-ccac7ac7-1b74-ad81-a9a7-1c2fbade8e4d (Addr: tcp/127.0.0.1:12048) (DC: dc1)" -18:37.746 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ccac7ac7-1b74-ad81-a9a7-1c2fbade8e4d.dc1 area=wan -18:37.746 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=tcp -18:37.747 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12044 network=tcp -18:37.747 [INFO] TestAgent: started state syncer -18:37.747 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12049 network=tcp -18:37.785 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:37.785 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12048 [Candidate]" term=2 -18:37.786 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:37.786 [DEBUG] TestAgent.server.raft: vote granted: from=ccac7ac7-1b74-ad81-a9a7-1c2fbade8e4d term=2 tally=1 -18:37.786 [INFO] TestAgent.server.raft: election won: tally=1 -18:37.786 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12048 [Leader]" -18:37.786 [INFO] TestAgent.server: cluster leadership acquired -18:37.786 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12048 -18:37.787 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:37.789 [INFO] TestAgent.server: New leader elected: payload=Node-ccac7ac7-1b74-ad81-a9a7-1c2fbade8e4d -18:37.821 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:37.821 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:37.821 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:37.821 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:37.821 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:37.821 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ccac7ac7-1b74-ad81-a9a7-1c2fbade8e4d -18:37.821 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ccac7ac7-1b74-ad81-a9a7-1c2fbade8e4d -18:37.826 [INFO] TestAgent.server: federation state anti-entropy synced -18:37.846 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:37.846 [INFO] TestAgent: Synced node info -18:37.950 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/raft/peer?address=nope from=127.0.0.1:60858 error="address "nope" was not found in the Raft configuration" -18:37.950 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/raft/peer?address=nope from=127.0.0.1:60858 latency=333.364µs -18:37.958 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/raft/peer?id=nope from=127.0.0.1:60860 error="id "nope" was not found in the Raft configuration" -18:37.959 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/raft/peer?id=nope from=127.0.0.1:60860 latency=311.787µs -18:37.959 [INFO] TestAgent: Requesting shutdown -18:37.959 [INFO] TestAgent.server: shutting down server -18:37.959 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:37.959 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:37.959 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:37.959 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:37.959 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:37.959 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:37.959 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:37.959 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:37.959 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:37.959 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:37.959 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:37.959Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:37.959Z [INFO] agent.router.manager: shutting down -18:37.959 [INFO] TestAgent: consul server down -18:37.959 [INFO] TestAgent: shutdown complete -18:37.959 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=tcp -18:37.959 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=udp -18:37.960 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12044 network=tcp -18:38.461 [INFO] TestAgent: Waiting for endpoints to shut down -18:38.461 [INFO] TestAgent: Endpoints down ---- PASS: TestOperatorRaftRemovePeerCommand (0.80s) +2022-12-04T05:51:01.112Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:01.125Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:01.127 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e78a59c6-a288-949b-16df-140d7350ecf0 Address:127.0.0.1:28384}]" +51:01.127 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28384 [Follower]" leader= +51:01.128 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e78a59c6-a288-949b-16df-140d7350ecf0.dc1 127.0.0.1 +51:01.128 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e78a59c6-a288-949b-16df-140d7350ecf0 127.0.0.1 +2022-12-04T05:51:01.128Z [INFO] agent.router: Initializing LAN area manager +51:01.128 [INFO] TestAgent.server: Adding LAN server: server="Node-e78a59c6-a288-949b-16df-140d7350ecf0 (Addr: tcp/127.0.0.1:28384) (DC: dc1)" +51:01.128 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e78a59c6-a288-949b-16df-140d7350ecf0.dc1 area=wan +51:01.128 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=udp +51:01.128 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=tcp +51:01.129 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28380 network=tcp +51:01.129 [INFO] TestAgent: started state syncer +51:01.129 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28385 network=tcp +51:01.192 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:01.192 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28384 [Candidate]" term=2 +51:01.192 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:01.192 [DEBUG] TestAgent.server.raft: vote granted: from=e78a59c6-a288-949b-16df-140d7350ecf0 term=2 tally=1 +51:01.192 [INFO] TestAgent.server.raft: election won: tally=1 +51:01.192 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28384 [Leader]" +51:01.192 [INFO] TestAgent.server: cluster leadership acquired +51:01.192 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28384 +51:01.192 [INFO] TestAgent.server: New leader elected: payload=Node-e78a59c6-a288-949b-16df-140d7350ecf0 +51:01.193 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:01.210 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:01.210 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:01.210 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:01.210 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:01.210 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:01.210 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e78a59c6-a288-949b-16df-140d7350ecf0 +51:01.210 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e78a59c6-a288-949b-16df-140d7350ecf0 +51:01.211 [INFO] TestAgent.server: federation state anti-entropy synced +51:01.438 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +51:01.439 [INFO] TestAgent: Synced node info +51:01.501 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/raft/peer?address=nope from=127.0.0.1:54792 error="address "nope" was not found in the Raft configuration" +51:01.501 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/raft/peer?address=nope from=127.0.0.1:54792 latency=387.576µs +51:01.503 [ERROR] TestAgent.http: Request error: method=DELETE url=/v1/operator/raft/peer?id=nope from=127.0.0.1:54794 error="id "nope" was not found in the Raft configuration" +51:01.503 [DEBUG] TestAgent.http: Request finished: method=DELETE url=/v1/operator/raft/peer?id=nope from=127.0.0.1:54794 latency=627.509µs +51:01.504 [INFO] TestAgent: Requesting shutdown +51:01.504 [INFO] TestAgent.server: shutting down server +51:01.504 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:01.504 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:01.504 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:01.504 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:01.504 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:01.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:01.504 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:01.504 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:01.504 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:01.504 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:01.504 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:01.504Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:01.504Z [INFO] agent.router.manager: shutting down +51:01.504 [INFO] TestAgent: consul server down +51:01.504 [INFO] TestAgent: shutdown complete +51:01.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=tcp +51:01.504 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=udp +51:01.504 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28380 network=tcp +51:02.005 [INFO] TestAgent: Waiting for endpoints to shut down +51:02.005 [INFO] TestAgent: Endpoints down +--- PASS: TestOperatorRaftRemovePeerCommand (0.93s) --- PASS: TestOperatorRaftRemovePeerCommand/Test_the_remove-peer_subcommand_directly (0.01s) - --- PASS: TestOperatorRaftRemovePeerCommand/Test_the_remove-peer_subcommand_with_-id (0.01s) + --- PASS: TestOperatorRaftRemovePeerCommand/Test_the_remove-peer_subcommand_with_-id (0.00s) PASS -ok github.com/hashicorp/consul/command/operator/raft/removepeer 0.868s +ok github.com/hashicorp/consul/command/operator/raft/removepeer 0.975s === RUN TestReloadCommand_noTabs === PAUSE TestReloadCommand_noTabs === RUN TestReloadCommand @@ -90152,57 +90315,58 @@ [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:18:41.650Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:41.706Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:41.709 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5638aadb-7b11-fac7-57d1-cc3660efdb09 Address:127.0.0.1:15111}]" -18:41.709 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15111 [Follower]" leader= -18:41.718 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5638aadb-7b11-fac7-57d1-cc3660efdb09.dc1 127.0.0.1 -18:41.718 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5638aadb-7b11-fac7-57d1-cc3660efdb09 127.0.0.1 -2021-10-31T23:18:41.718Z [INFO] agent.router: Initializing LAN area manager -18:41.719 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=udp -18:41.719 [INFO] TestAgent.server: Adding LAN server: server="Node-5638aadb-7b11-fac7-57d1-cc3660efdb09 (Addr: tcp/127.0.0.1:15111) (DC: dc1)" -18:41.719 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5638aadb-7b11-fac7-57d1-cc3660efdb09.dc1 area=wan -18:41.719 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=tcp -18:41.720 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15107 network=tcp -18:41.720 [INFO] TestAgent: started state syncer -18:41.720 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15112 network=tcp -18:41.769 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:41.770 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15111 [Candidate]" term=2 -18:41.770 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:41.770 [DEBUG] TestAgent.server.raft: vote granted: from=5638aadb-7b11-fac7-57d1-cc3660efdb09 term=2 tally=1 -18:41.770 [INFO] TestAgent.server.raft: election won: tally=1 -18:41.770 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15111 [Leader]" -18:41.770 [INFO] TestAgent.server: cluster leadership acquired -18:41.770 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15111 -18:41.771 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:41.772 [INFO] TestAgent.server: New leader elected: payload=Node-5638aadb-7b11-fac7-57d1-cc3660efdb09 -18:41.799 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:41.799 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:41.799 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:41.799 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:41.799 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:41.800 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5638aadb-7b11-fac7-57d1-cc3660efdb09 -18:41.800 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5638aadb-7b11-fac7-57d1-cc3660efdb09 -18:41.800 [INFO] TestAgent.server: federation state anti-entropy synced -2021-10-31T23:18:41.961Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:41.978 [INFO] TestAgent: Requesting shutdown -18:41.978 [INFO] TestAgent.server: shutting down server -18:41.978 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:41.978 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:41.978 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -18:41.978 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:41.978Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:41.978Z [INFO] agent.router.manager: shutting down -18:41.978 [INFO] TestAgent: consul server down -18:41.978 [INFO] TestAgent: shutdown complete -18:41.978 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=tcp -18:41.978 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=udp -18:41.978 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15107 network=tcp -18:42.480 [INFO] TestAgent: Waiting for endpoints to shut down -18:42.480 [INFO] TestAgent: Endpoints down ---- PASS: TestReloadCommand (0.96s) +2022-12-04T05:51:02.278Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:02.294Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:02.296 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ca5920be-75c5-d682-172b-3b9e7c5f948c Address:127.0.0.1:24300}]" +51:02.296 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:24300 [Follower]" leader= +51:02.297 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ca5920be-75c5-d682-172b-3b9e7c5f948c.dc1 127.0.0.1 +51:02.297 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ca5920be-75c5-d682-172b-3b9e7c5f948c 127.0.0.1 +2022-12-04T05:51:02.297Z [INFO] agent.router: Initializing LAN area manager +51:02.297 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=udp +51:02.298 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ca5920be-75c5-d682-172b-3b9e7c5f948c.dc1 area=wan +51:02.298 [INFO] TestAgent.server: Adding LAN server: server="Node-ca5920be-75c5-d682-172b-3b9e7c5f948c (Addr: tcp/127.0.0.1:24300) (DC: dc1)" +51:02.298 [INFO] TestAgent: Started DNS server: address=127.0.0.1:24295 network=tcp +51:02.298 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:24296 network=tcp +51:02.298 [INFO] TestAgent: started state syncer +51:02.298 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:24301 network=tcp +51:02.352 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:02.352 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:24300 [Candidate]" term=2 +51:02.352 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:02.352 [DEBUG] TestAgent.server.raft: vote granted: from=ca5920be-75c5-d682-172b-3b9e7c5f948c term=2 tally=1 +51:02.352 [INFO] TestAgent.server.raft: election won: tally=1 +51:02.352 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:24300 [Leader]" +51:02.352 [INFO] TestAgent.server: cluster leadership acquired +51:02.352 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:24300 +51:02.353 [INFO] TestAgent.server: New leader elected: payload=Node-ca5920be-75c5-d682-172b-3b9e7c5f948c +51:02.360 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:02.376 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:02.376 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:02.376 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:02.376 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:02.376 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:02.376 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ca5920be-75c5-d682-172b-3b9e7c5f948c +51:02.376 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ca5920be-75c5-d682-172b-3b9e7c5f948c +51:02.377 [INFO] TestAgent.server: federation state anti-entropy synced +51:02.464 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +51:02.464 [INFO] TestAgent: Synced node info +2022-12-04T05:51:02.498Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:02.498 [INFO] TestAgent: Requesting shutdown +51:02.498 [INFO] TestAgent.server: shutting down server +51:02.498 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:02.498 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:02.498 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:02.498Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:02.498Z [INFO] agent.router.manager: shutting down +51:02.498 [INFO] TestAgent: consul server down +51:02.498 [INFO] TestAgent: shutdown complete +51:02.498 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=tcp +51:02.498 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:24295 network=udp +51:02.498 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:24296 network=tcp +51:02.999 [INFO] TestAgent: Waiting for endpoints to shut down +51:02.999 [INFO] TestAgent: Endpoints down +--- PASS: TestReloadCommand (0.76s) PASS -ok github.com/hashicorp/consul/command/reload 1.519s +ok github.com/hashicorp/consul/command/reload 0.808s === RUN TestRTTCommand_noTabs === PAUSE TestRTTCommand_noTabs === RUN TestRTTCommand_BadArgs @@ -90212,148 +90376,151 @@ === RUN TestRTTCommand_WAN === PAUSE TestRTTCommand_WAN === CONT TestRTTCommand_noTabs -=== CONT TestRTTCommand_LAN -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... ---- PASS: TestRTTCommand_noTabs (0.01s) === CONT TestRTTCommand_WAN +=== CONT TestRTTCommand_LAN === CONT TestRTTCommand_BadArgs === RUN TestRTTCommand_BadArgs/#00 +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === RUN TestRTTCommand_BadArgs/node1_node2_node3 +--- PASS: TestRTTCommand_noTabs (0.00s) === RUN TestRTTCommand_BadArgs/-wan_node1_node2 -=== RUN TestRTTCommand_BadArgs/-wan_node1.dc1_node2 -=== RUN TestRTTCommand_BadArgs/-wan_node1_node2.dc1 [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range ---- PASS: TestRTTCommand_BadArgs (0.02s) +=== RUN TestRTTCommand_BadArgs/-wan_node1.dc1_node2 +=== RUN TestRTTCommand_BadArgs/-wan_node1_node2.dc1 +--- PASS: TestRTTCommand_BadArgs (0.00s) --- PASS: TestRTTCommand_BadArgs/#00 (0.00s) --- PASS: TestRTTCommand_BadArgs/node1_node2_node3 (0.00s) --- PASS: TestRTTCommand_BadArgs/-wan_node1_node2 (0.00s) --- PASS: TestRTTCommand_BadArgs/-wan_node1.dc1_node2 (0.00s) - --- PASS: TestRTTCommand_BadArgs/-wan_node1_node2.dc1 (0.01s) -2021-10-31T23:18:41.667Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:41.753Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:41.755 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5237fcfe-fe27-4ca8-a3cb-da4af91e8624 Address:127.0.0.1:28391}]" -18:41.756 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5237fcfe-fe27-4ca8-a3cb-da4af91e8624.dc1 127.0.0.1 -18:41.757 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5237fcfe-fe27-4ca8-a3cb-da4af91e8624 127.0.0.1 -2021-10-31T23:18:41.757Z [INFO] agent.router: Initializing LAN area manager -18:41.757 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28386 network=udp -18:41.761 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28386 network=tcp -18:41.762 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28387 network=tcp -18:41.762 [INFO] TestAgent: started state syncer -18:41.758 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28391 [Follower]" leader= -18:41.758 [INFO] TestAgent.server: Adding LAN server: server="Node-5237fcfe-fe27-4ca8-a3cb-da4af91e8624 (Addr: tcp/127.0.0.1:28391) (DC: dc1)" -18:41.762 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28392 network=tcp -18:41.758 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5237fcfe-fe27-4ca8-a3cb-da4af91e8624.dc1 area=wan -18:41.800 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:41.800 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28391 [Candidate]" term=2 -18:41.800 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:41.800 [DEBUG] TestAgent.server.raft: vote granted: from=5237fcfe-fe27-4ca8-a3cb-da4af91e8624 term=2 tally=1 -18:41.800 [INFO] TestAgent.server.raft: election won: tally=1 -18:41.800 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28391 [Leader]" -18:41.800 [INFO] TestAgent.server: cluster leadership acquired -18:41.800 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28391 -18:41.812 [INFO] TestAgent.server: New leader elected: payload=Node-5237fcfe-fe27-4ca8-a3cb-da4af91e8624 -18:41.819 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:41.849 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:41.849 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:41.849 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:41.849 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:41.849 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:41.849 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5237fcfe-fe27-4ca8-a3cb-da4af91e8624 -18:41.849 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5237fcfe-fe27-4ca8-a3cb-da4af91e8624 -18:41.850 [INFO] TestAgent.server: federation state anti-entropy synced -18:41.879 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/datacenters from=127.0.0.1:49612 latency=446.322µs -18:41.882 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:49614 latency=1.794593ms -18:41.891 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/datacenters from=127.0.0.1:49614 latency=1.093195ms -18:41.904 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/datacenters from=127.0.0.1:49616 latency=510.78µs -18:41.914 [INFO] TestAgent: Requesting shutdown -18:41.914 [INFO] TestAgent.server: shutting down server -18:41.914 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:41.914 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:41.914 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:41.914 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:41.914 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:41.914 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:41.914 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:41.914 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -18:41.914 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:41.914 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:41.914 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:41.914 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:41.914Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:41.914Z [INFO] agent.router.manager: shutting down -18:41.915 [INFO] TestAgent: consul server down -18:41.915 [INFO] TestAgent: shutdown complete -18:41.915 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28386 network=tcp -18:41.915 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28386 network=udp -18:41.915 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28387 network=tcp -18:42.417 [INFO] TestAgent: Waiting for endpoints to shut down -18:42.417 [INFO] TestAgent: Endpoints down ---- PASS: TestRTTCommand_WAN (0.85s) -2021-10-31T23:18:41.670Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:41.704Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:41.756 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:7797b3b5-34d9-4df6-99b6-903096517870 Address:127.0.0.1:28384}]" -18:41.756 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-7797b3b5-34d9-4df6-99b6-903096517870.dc1 127.0.0.1 -18:41.757 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-7797b3b5-34d9-4df6-99b6-903096517870 127.0.0.1 -2021-10-31T23:18:41.757Z [INFO] agent.router: Initializing LAN area manager -18:41.758 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28384 [Follower]" leader= -18:41.759 [INFO] TestAgent.server: Adding LAN server: server="Node-7797b3b5-34d9-4df6-99b6-903096517870 (Addr: tcp/127.0.0.1:28384) (DC: dc1)" -18:41.759 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-7797b3b5-34d9-4df6-99b6-903096517870.dc1 area=wan -18:41.759 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=udp -18:41.759 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=tcp -18:41.759 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28380 network=tcp -18:41.760 [INFO] TestAgent: started state syncer -18:41.760 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28385 network=tcp -18:41.812 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:41.812 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28384 [Candidate]" term=2 -18:41.813 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:41.813 [DEBUG] TestAgent.server.raft: vote granted: from=7797b3b5-34d9-4df6-99b6-903096517870 term=2 tally=1 -18:41.813 [INFO] TestAgent.server.raft: election won: tally=1 -18:41.813 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28384 [Leader]" -18:41.813 [INFO] TestAgent.server: cluster leadership acquired -18:41.818 [INFO] TestAgent: Synced node info -18:41.818 [DEBUG] TestAgent: Node info in sync -18:41.821 [INFO] TestAgent.server: New leader elected: payload=Node-7797b3b5-34d9-4df6-99b6-903096517870 -18:41.822 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28384 -18:41.823 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:41.849 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:41.849 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:41.849 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:41.849 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:41.849 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:41.849 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-7797b3b5-34d9-4df6-99b6-903096517870 -18:41.850 [INFO] TestAgent.server: member joined, marking health alive: member=Node-7797b3b5-34d9-4df6-99b6-903096517870 -18:41.851 [INFO] TestAgent.server: federation state anti-entropy synced -18:42.142 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:48046 latency=291.601µs -18:42.179 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:48048 latency=263.621µs -18:42.181 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:48050 latency=1.188003ms -18:42.186 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:48050 latency=284.881µs -18:42.191 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:48052 latency=278.547µs -18:42.191 [INFO] TestAgent: Requesting shutdown -18:42.191 [INFO] TestAgent.server: shutting down server -18:42.192 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:42.192 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:42.192 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:42.192 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:42.192 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:42.192 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:42.192 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:42.192 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:42.192 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:42.192 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:42.192 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:42.192Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:42.192Z [INFO] agent.router.manager: shutting down -18:42.192 [INFO] TestAgent: consul server down -18:42.192 [INFO] TestAgent: shutdown complete -18:42.192 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=tcp -18:42.192 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=udp -18:42.192 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28380 network=tcp -18:42.696 [INFO] TestAgent: Waiting for endpoints to shut down -18:42.696 [INFO] TestAgent: Endpoints down ---- PASS: TestRTTCommand_LAN (1.14s) + --- PASS: TestRTTCommand_BadArgs/-wan_node1_node2.dc1 (0.00s) +2022-12-04T05:51:04.442Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:04.455Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:04.459 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:79c7f3e4-63ae-d2a5-d367-a943e2e299c7 Address:127.0.0.1:30433}]" +51:04.459 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30433 [Follower]" leader= +51:04.460 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-79c7f3e4-63ae-d2a5-d367-a943e2e299c7.dc1 127.0.0.1 +51:04.461 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-79c7f3e4-63ae-d2a5-d367-a943e2e299c7 127.0.0.1 +2022-12-04T05:51:04.461Z [INFO] agent.router: Initializing LAN area manager +51:04.461 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=udp +51:04.461 [INFO] TestAgent.server: Adding LAN server: server="Node-79c7f3e4-63ae-d2a5-d367-a943e2e299c7 (Addr: tcp/127.0.0.1:30433) (DC: dc1)" +51:04.461 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30428 network=tcp +51:04.462 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-79c7f3e4-63ae-d2a5-d367-a943e2e299c7.dc1 area=wan +51:04.462 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30429 network=tcp +51:04.462 [INFO] TestAgent: started state syncer +51:04.462 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30434 network=tcp +51:04.511 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:04.511 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30433 [Candidate]" term=2 +51:04.511 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:04.511 [DEBUG] TestAgent.server.raft: vote granted: from=79c7f3e4-63ae-d2a5-d367-a943e2e299c7 term=2 tally=1 +51:04.511 [INFO] TestAgent.server.raft: election won: tally=1 +51:04.511 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30433 [Leader]" +51:04.511 [INFO] TestAgent.server: cluster leadership acquired +51:04.511 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30433 +51:04.511 [INFO] TestAgent.server: New leader elected: payload=Node-79c7f3e4-63ae-d2a5-d367-a943e2e299c7 +51:04.512 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:04.513 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:04.513 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:04.513 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:04.513 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:04.513 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:04.513 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-79c7f3e4-63ae-d2a5-d367-a943e2e299c7 +51:04.513 [INFO] TestAgent.server: member joined, marking health alive: member=Node-79c7f3e4-63ae-d2a5-d367-a943e2e299c7 +51:04.514 [INFO] TestAgent.server: federation state anti-entropy synced +51:04.755 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/datacenters from=127.0.0.1:33410 latency=428.144µs +51:04.758 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:33412 latency=1.782274ms +51:04.760 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/datacenters from=127.0.0.1:33412 latency=189.751µs +51:04.761 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/datacenters from=127.0.0.1:33414 latency=129.174µs +51:04.762 [INFO] TestAgent: Requesting shutdown +51:04.762 [INFO] TestAgent.server: shutting down server +51:04.762 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:04.762 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:04.762 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:04.762 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:04.762 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:04.762 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +51:04.762 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:04.762 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:04.762 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:04.762 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:04.762 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:04.762 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:04.762Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:04.762Z [INFO] agent.router.manager: shutting down +51:04.762 [INFO] TestAgent: consul server down +51:04.762 [INFO] TestAgent: shutdown complete +51:04.762 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=tcp +51:04.762 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30428 network=udp +51:04.762 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30429 network=tcp +51:05.263 [INFO] TestAgent: Waiting for endpoints to shut down +51:05.263 [INFO] TestAgent: Endpoints down +--- PASS: TestRTTCommand_WAN (0.86s) +2022-12-04T05:51:04.441Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:04.453Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:04.456 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:db877781-78b6-b40c-db3b-a50dd14bb4cc Address:127.0.0.1:30426}]" +51:04.456 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= +51:04.459 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-db877781-78b6-b40c-db3b-a50dd14bb4cc.dc1 127.0.0.1 +51:04.460 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-db877781-78b6-b40c-db3b-a50dd14bb4cc 127.0.0.1 +2022-12-04T05:51:04.460Z [INFO] agent.router: Initializing LAN area manager +51:04.461 [INFO] TestAgent.server: Adding LAN server: server="Node-db877781-78b6-b40c-db3b-a50dd14bb4cc (Addr: tcp/127.0.0.1:30426) (DC: dc1)" +51:04.461 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp +51:04.461 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp +51:04.461 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-db877781-78b6-b40c-db3b-a50dd14bb4cc.dc1 area=wan +51:04.462 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp +51:04.462 [INFO] TestAgent: started state syncer +51:04.462 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp +51:04.496 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:04.496 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 +51:04.496 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:04.496 [DEBUG] TestAgent.server.raft: vote granted: from=db877781-78b6-b40c-db3b-a50dd14bb4cc term=2 tally=1 +51:04.496 [INFO] TestAgent.server.raft: election won: tally=1 +51:04.496 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" +51:04.496 [INFO] TestAgent.server: cluster leadership acquired +51:04.496 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30426 +51:04.497 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:04.500 [INFO] TestAgent.server: New leader elected: payload=Node-db877781-78b6-b40c-db3b-a50dd14bb4cc +51:04.513 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:04.513 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:04.513 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:04.513 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:04.513 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:04.513 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-db877781-78b6-b40c-db3b-a50dd14bb4cc +51:04.513 [INFO] TestAgent.server: member joined, marking health alive: member=Node-db877781-78b6-b40c-db3b-a50dd14bb4cc +51:04.514 [INFO] TestAgent.server: federation state anti-entropy synced +51:04.556 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +51:04.556 [INFO] TestAgent: Synced node info +51:04.556 [DEBUG] TestAgent: Node info in sync +51:04.887 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:50398 latency=1.323609ms +51:04.914 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:50400 latency=168.853µs +51:04.941 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:50402 latency=208.598µs +51:04.967 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:50404 latency=176.047µs +51:04.969 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:50406 latency=916.536µs +51:04.971 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:50406 latency=232.011µs +51:04.972 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/coordinate/nodes from=127.0.0.1:50408 latency=227.752µs +51:04.972 [INFO] TestAgent: Requesting shutdown +51:04.972 [INFO] TestAgent.server: shutting down server +51:04.972 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:04.972 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:04.972 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:04.972 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:04.973 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:04.973 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:04.973 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:04.973 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:04.973 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:04.973 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:04.973 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:04.973Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:04.973Z [INFO] agent.router.manager: shutting down +51:04.973 [INFO] TestAgent: consul server down +51:04.973 [INFO] TestAgent: shutdown complete +51:04.973 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp +51:04.973 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp +51:04.973 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp +51:05.473 [INFO] TestAgent: Waiting for endpoints to shut down +51:05.473 [INFO] TestAgent: Endpoints down +--- PASS: TestRTTCommand_LAN (1.07s) PASS -ok github.com/hashicorp/consul/command/rtt 1.355s +ok github.com/hashicorp/consul/command/rtt 1.122s === RUN TestDevModeHasNoServices === PAUSE TestDevModeHasNoServices === RUN TestStructsToAgentService @@ -90378,20 +90545,20 @@ === PAUSE TestStructsToAgentService/Proxy_service === CONT TestStructsToAgentService/Basic_service_with_port === CONT TestStructsToAgentService/Service_with_a_zero-value_check -=== CONT TestStructsToAgentService/Proxy_service +=== CONT TestStructsToAgentService/Service_with_checks === CONT TestStructsToAgentService/Service_with_an_unnamed_check +=== CONT TestStructsToAgentService/Proxy_service === CONT TestStructsToAgentService/Service_with_a_check -=== CONT TestStructsToAgentService/Service_with_checks --- PASS: TestStructsToAgentService (0.00s) --- PASS: TestStructsToAgentService/Service_with_a_zero-value_check (0.00s) --- PASS: TestStructsToAgentService/Basic_service_with_port (0.00s) --- PASS: TestStructsToAgentService/Service_with_an_unnamed_check (0.00s) - --- PASS: TestStructsToAgentService/Proxy_service (0.00s) - --- PASS: TestStructsToAgentService/Service_with_a_check (0.00s) --- PASS: TestStructsToAgentService/Service_with_checks (0.00s) ---- PASS: TestDevModeHasNoServices (0.02s) + --- PASS: TestStructsToAgentService/Service_with_a_check (0.00s) + --- PASS: TestStructsToAgentService/Proxy_service (0.00s) +--- PASS: TestDevModeHasNoServices (0.01s) PASS -ok github.com/hashicorp/consul/command/services 0.159s +ok github.com/hashicorp/consul/command/services 0.042s === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === RUN TestCommand_Validation @@ -90403,12 +90570,12 @@ === RUN TestCommand_Flag === PAUSE TestCommand_Flag === CONT TestCommand_noTabs ---- PASS: TestCommand_noTabs (0.00s) -=== CONT TestCommand_Flag -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestCommand_File_id === CONT TestCommand_File_nameOnly +=== CONT TestCommand_File_id +=== CONT TestCommand_Flag === CONT TestCommand_Validation +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestCommand_noTabs (0.00s) === RUN TestCommand_Validation/no_args_or_id === RUN TestCommand_Validation/args_and_-id --- PASS: TestCommand_Validation (0.00s) @@ -90416,216 +90583,220 @@ --- PASS: TestCommand_Validation/args_and_-id (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:18:51.401Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:51.468Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:51.475 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:95627ade-a701-6022-6fde-25b47b66a8db Address:127.0.0.1:27363}]" -18:51.476 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-95627ade-a701-6022-6fde-25b47b66a8db.dc1 127.0.0.1 -18:51.476 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-95627ade-a701-6022-6fde-25b47b66a8db 127.0.0.1 -2021-10-31T23:18:51.476Z [INFO] agent.router: Initializing LAN area manager -18:51.477 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27363 [Follower]" leader= -18:51.477 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=udp -18:51.477 [INFO] TestAgent.server: Adding LAN server: server="Node-95627ade-a701-6022-6fde-25b47b66a8db (Addr: tcp/127.0.0.1:27363) (DC: dc1)" -18:51.477 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27358 network=tcp -18:51.478 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27359 network=tcp -18:51.478 [INFO] TestAgent: started state syncer -18:51.478 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27364 network=tcp -18:51.480 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-95627ade-a701-6022-6fde-25b47b66a8db.dc1 area=wan -18:51.517 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:51.517 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27363 [Candidate]" term=2 -18:51.518 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:51.518 [DEBUG] TestAgent.server.raft: vote granted: from=95627ade-a701-6022-6fde-25b47b66a8db term=2 tally=1 -18:51.518 [INFO] TestAgent.server.raft: election won: tally=1 -18:51.518 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27363 [Leader]" -18:51.518 [INFO] TestAgent.server: cluster leadership acquired -18:51.518 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27363 -18:51.519 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:51.519 [INFO] TestAgent.server: New leader elected: payload=Node-95627ade-a701-6022-6fde-25b47b66a8db -18:51.560 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:51.560 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:51.560 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:51.560 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:51.560 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:51.560 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-95627ade-a701-6022-6fde-25b47b66a8db -18:51.560 [INFO] TestAgent.server: member joined, marking health alive: member=Node-95627ade-a701-6022-6fde-25b47b66a8db -18:51.561 [INFO] TestAgent.server: federation state anti-entropy synced -18:51.568 [INFO] TestAgent: Synced node info -18:51.568 [INFO] TestAgent: Synced service: service=web -18:51.568 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:55714 latency=1.635784ms -18:51.574 [DEBUG] TestAgent: Node info in sync -18:51.574 [DEBUG] TestAgent: Service in sync: service=web -18:51.574 [INFO] TestAgent: Synced service: service=db -18:51.574 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:55714 latency=887.903µs -18:51.576 [DEBUG] TestAgent: removed service: service=web -18:51.576 [DEBUG] TestAgent: Node info in sync -18:51.577 [INFO] TestAgent: Deregistered service: service=web -18:51.577 [DEBUG] TestAgent: Service in sync: service=db -18:51.577 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/web from=127.0.0.1:55716 latency=612.317µs -18:51.578 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:55714 latency=445.015µs -18:51.578 [INFO] TestAgent: Requesting shutdown -18:51.578 [INFO] TestAgent.server: shutting down server -18:51.578 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:51.578 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:51.578 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:51.578 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:51.578 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:51.578 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:51.578 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:51.578 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -18:51.578 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:51.578 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:51.578 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:51.578 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:51.578Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:51.579Z [INFO] agent.router.manager: shutting down -18:51.579 [INFO] TestAgent: consul server down -18:51.579 [INFO] TestAgent: shutdown complete -18:51.579 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=tcp -18:51.579 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27358 network=udp -18:51.579 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27359 network=tcp -18:52.081 [INFO] TestAgent: Waiting for endpoints to shut down -18:52.081 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_Flag (0.83s) -2021-10-31T23:18:51.384Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:51.423Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:51.450 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:40aa0324-5ada-6fc0-1ed4-b12ca331adb4 Address:127.0.0.1:27377}]" -18:51.453 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27377 [Follower]" leader= -18:51.453 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-40aa0324-5ada-6fc0-1ed4-b12ca331adb4.dc1 127.0.0.1 -18:51.470 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-40aa0324-5ada-6fc0-1ed4-b12ca331adb4 127.0.0.1 -2021-10-31T23:18:51.470Z [INFO] agent.router: Initializing LAN area manager -18:51.471 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27372 network=udp -18:51.471 [INFO] TestAgent.server: Adding LAN server: server="Node-40aa0324-5ada-6fc0-1ed4-b12ca331adb4 (Addr: tcp/127.0.0.1:27377) (DC: dc1)" -18:51.471 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-40aa0324-5ada-6fc0-1ed4-b12ca331adb4.dc1 area=wan -18:51.471 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27372 network=tcp -18:51.478 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27373 network=tcp -18:51.478 [INFO] TestAgent: started state syncer -18:51.478 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27378 network=tcp -18:51.510 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:51.510 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27377 [Candidate]" term=2 -18:51.510 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:51.510 [DEBUG] TestAgent.server.raft: vote granted: from=40aa0324-5ada-6fc0-1ed4-b12ca331adb4 term=2 tally=1 -18:51.510 [INFO] TestAgent.server.raft: election won: tally=1 -18:51.510 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27377 [Leader]" -18:51.510 [INFO] TestAgent.server: cluster leadership acquired -18:51.510 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27377 -18:51.511 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:51.511 [INFO] TestAgent.server: New leader elected: payload=Node-40aa0324-5ada-6fc0-1ed4-b12ca331adb4 -18:51.558 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:51.558 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:51.558 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:51.558 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:51.558 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:51.558 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-40aa0324-5ada-6fc0-1ed4-b12ca331adb4 -18:51.558 [INFO] TestAgent.server: member joined, marking health alive: member=Node-40aa0324-5ada-6fc0-1ed4-b12ca331adb4 -18:51.560 [INFO] TestAgent.server: federation state anti-entropy synced -18:51.593 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:51.593 [INFO] TestAgent: Synced node info -18:51.617 [DEBUG] TestAgent: Node info in sync -18:51.617 [INFO] TestAgent: Synced service: service=web -18:51.617 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:35156 latency=787.356µs -18:51.618 [DEBUG] TestAgent: Node info in sync -18:51.618 [DEBUG] TestAgent: Service in sync: service=web -18:51.618 [INFO] TestAgent: Synced service: service=db -18:51.618 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:35156 latency=451.416µs -18:51.644 [DEBUG] TestAgent: removed service: service=web -18:51.644 [DEBUG] TestAgent: Node info in sync -18:51.644 [INFO] TestAgent: Deregistered service: service=web -18:51.644 [DEBUG] TestAgent: Service in sync: service=db -18:51.644 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/web from=127.0.0.1:35158 latency=479.221µs -18:51.647 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:35156 latency=2.813256ms -18:51.662 [INFO] TestAgent: Requesting shutdown -18:51.662 [INFO] TestAgent.server: shutting down server -18:51.662 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:51.662 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:51.662 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:51.662 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:51.662 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:51.665 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:51.666 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:51.666 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:51.666 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:51.666 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:51.673 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:51.675Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:51.675Z [INFO] agent.router.manager: shutting down -18:51.676 [INFO] TestAgent: consul server down -18:51.676 [INFO] TestAgent: shutdown complete -18:51.676 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27372 network=tcp -18:51.676 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27372 network=udp -18:51.676 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27373 network=tcp -18:52.177 [INFO] TestAgent: Waiting for endpoints to shut down -18:52.177 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_File_nameOnly (0.93s) -2021-10-31T23:18:51.381Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:51.431Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:51.451 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5030ad0b-c967-35ac-80fd-c3f5b3e45496 Address:127.0.0.1:27370}]" -18:51.451 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5030ad0b-c967-35ac-80fd-c3f5b3e45496.dc1 127.0.0.1 -18:51.452 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5030ad0b-c967-35ac-80fd-c3f5b3e45496 127.0.0.1 -2021-10-31T23:18:51.452Z [INFO] agent.router: Initializing LAN area manager -18:51.452 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27365 network=udp -18:51.463 [INFO] TestAgent.server: Adding LAN server: server="Node-5030ad0b-c967-35ac-80fd-c3f5b3e45496 (Addr: tcp/127.0.0.1:27370) (DC: dc1)" -18:51.469 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:27370 [Follower]" leader= -18:51.470 [INFO] TestAgent: Started DNS server: address=127.0.0.1:27365 network=tcp -18:51.471 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:27366 network=tcp -18:51.471 [INFO] TestAgent: started state syncer -18:51.471 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5030ad0b-c967-35ac-80fd-c3f5b3e45496.dc1 area=wan -18:51.471 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:27371 network=tcp -18:51.505 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:51.505 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:27370 [Candidate]" term=2 -18:51.505 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:51.505 [DEBUG] TestAgent.server.raft: vote granted: from=5030ad0b-c967-35ac-80fd-c3f5b3e45496 term=2 tally=1 -18:51.505 [INFO] TestAgent.server.raft: election won: tally=1 -18:51.505 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:27370 [Leader]" -18:51.506 [INFO] TestAgent.server: cluster leadership acquired -18:51.506 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:27370 -18:51.507 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:51.509 [INFO] TestAgent.server: New leader elected: payload=Node-5030ad0b-c967-35ac-80fd-c3f5b3e45496 -18:51.531 [INFO] TestAgent: Synced node info -18:51.553 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:51.553 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:51.553 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:51.553 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:51.553 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:51.553 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5030ad0b-c967-35ac-80fd-c3f5b3e45496 -18:51.553 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5030ad0b-c967-35ac-80fd-c3f5b3e45496 -18:51.554 [INFO] TestAgent.server: federation state anti-entropy synced -18:51.891 [DEBUG] TestAgent: Node info in sync -18:51.892 [INFO] TestAgent: Synced service: service=web -18:51.892 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:42566 latency=839.039µs -18:51.892 [DEBUG] TestAgent: Node info in sync -18:51.892 [DEBUG] TestAgent: Service in sync: service=web -18:51.893 [INFO] TestAgent: Synced service: service=db -18:51.893 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:42566 latency=507.677µs -18:51.930 [DEBUG] TestAgent: removed service: service=web -18:51.930 [DEBUG] TestAgent: Node info in sync -18:51.930 [INFO] TestAgent: Deregistered service: service=web -18:51.930 [DEBUG] TestAgent: Service in sync: service=db -18:51.930 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/web from=127.0.0.1:42568 latency=599.58µs -18:51.931 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:42566 latency=498.259µs -18:51.932 [INFO] TestAgent: Requesting shutdown -18:51.932 [INFO] TestAgent.server: shutting down server -18:51.932 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:51.932 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:51.932 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:51.932 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:51.932 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:51.932 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:51.932 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:51.932 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:51.932 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:51.932 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:51.932 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:51.932Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:51.932Z [INFO] agent.router.manager: shutting down -18:51.932 [INFO] TestAgent: consul server down -18:51.932 [INFO] TestAgent: shutdown complete -18:51.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27365 network=tcp -18:51.932 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:27365 network=udp -18:51.932 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:27366 network=tcp -18:52.433 [INFO] TestAgent: Waiting for endpoints to shut down -18:52.433 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_File_id (1.18s) +2022-12-04T05:51:07.546Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:07.566Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:07.567 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:88b41d85-dbe6-0926-baf4-d11ef10b19fa Address:127.0.0.1:21237}]" +51:07.568 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-88b41d85-dbe6-0926-baf4-d11ef10b19fa.dc1 127.0.0.1 +51:07.568 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21237 [Follower]" leader= +51:07.568 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-88b41d85-dbe6-0926-baf4-d11ef10b19fa 127.0.0.1 +2022-12-04T05:51:07.568Z [INFO] agent.router: Initializing LAN area manager +51:07.568 [INFO] TestAgent.server: Adding LAN server: server="Node-88b41d85-dbe6-0926-baf4-d11ef10b19fa (Addr: tcp/127.0.0.1:21237) (DC: dc1)" +51:07.568 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-88b41d85-dbe6-0926-baf4-d11ef10b19fa.dc1 area=wan +51:07.568 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=udp +51:07.568 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21232 network=tcp +51:07.570 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21233 network=tcp +51:07.571 [INFO] TestAgent: started state syncer +51:07.571 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21238 network=tcp +51:07.625 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:07.625 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21237 [Candidate]" term=2 +51:07.625 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:07.625 [DEBUG] TestAgent.server.raft: vote granted: from=88b41d85-dbe6-0926-baf4-d11ef10b19fa term=2 tally=1 +51:07.625 [INFO] TestAgent.server.raft: election won: tally=1 +51:07.625 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21237 [Leader]" +51:07.625 [INFO] TestAgent.server: cluster leadership acquired +51:07.625 [INFO] TestAgent.server: New leader elected: payload=Node-88b41d85-dbe6-0926-baf4-d11ef10b19fa +51:07.625 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21237 +51:07.626 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:07.641 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:07.641 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:07.641 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:07.641 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:07.641 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:07.641 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-88b41d85-dbe6-0926-baf4-d11ef10b19fa +51:07.641 [INFO] TestAgent.server: member joined, marking health alive: member=Node-88b41d85-dbe6-0926-baf4-d11ef10b19fa +51:07.642 [INFO] TestAgent.server: federation state anti-entropy synced +51:07.710 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +51:07.711 [INFO] TestAgent: Synced node info +51:07.711 [DEBUG] TestAgent: Node info in sync +51:07.718 [DEBUG] TestAgent: Node info in sync +51:07.718 [INFO] TestAgent: Synced service: service=web +51:07.718 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:51990 latency=1.175527ms +51:07.719 [DEBUG] TestAgent: Node info in sync +51:07.719 [DEBUG] TestAgent: Service in sync: service=web +51:07.719 [INFO] TestAgent: Synced service: service=db +51:07.719 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:51990 latency=423.927µs +51:07.732 [DEBUG] TestAgent: removed service: service=web +51:07.732 [DEBUG] TestAgent: Node info in sync +51:07.732 [INFO] TestAgent: Deregistered service: service=web +51:07.733 [DEBUG] TestAgent: Service in sync: service=db +51:07.733 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/web from=127.0.0.1:51992 latency=528.359µs +51:07.734 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:51990 latency=872.41µs +51:07.734 [INFO] TestAgent: Requesting shutdown +51:07.734 [INFO] TestAgent.server: shutting down server +51:07.734 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:07.734 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:07.734 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:07.734 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:07.734 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:07.734 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:07.734 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:07.734 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:07.734 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:07.734 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:07.735 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:07.735Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:07.735Z [INFO] agent.router.manager: shutting down +51:07.735 [INFO] TestAgent: consul server down +51:07.735 [INFO] TestAgent: shutdown complete +51:07.735 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=tcp +51:07.735 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21232 network=udp +51:07.735 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21233 network=tcp +51:08.235 [INFO] TestAgent: Waiting for endpoints to shut down +51:08.235 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_File_nameOnly (0.72s) +2022-12-04T05:51:07.547Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:07.567Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:07.568 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f0cf2b7a-008e-19a1-fa56-b351152c0a93 Address:127.0.0.1:21244}]" +51:07.568 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21244 [Follower]" leader= +51:07.569 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f0cf2b7a-008e-19a1-fa56-b351152c0a93.dc1 127.0.0.1 +51:07.571 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f0cf2b7a-008e-19a1-fa56-b351152c0a93 127.0.0.1 +2022-12-04T05:51:07.571Z [INFO] agent.router: Initializing LAN area manager +51:07.572 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f0cf2b7a-008e-19a1-fa56-b351152c0a93.dc1 area=wan +51:07.573 [INFO] TestAgent.server: Adding LAN server: server="Node-f0cf2b7a-008e-19a1-fa56-b351152c0a93 (Addr: tcp/127.0.0.1:21244) (DC: dc1)" +51:07.573 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21239 network=udp +51:07.573 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21239 network=tcp +51:07.574 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21240 network=tcp +51:07.574 [INFO] TestAgent: started state syncer +51:07.574 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21245 network=tcp +51:07.632 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:07.632 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21244 [Candidate]" term=2 +51:07.632 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:07.632 [DEBUG] TestAgent.server.raft: vote granted: from=f0cf2b7a-008e-19a1-fa56-b351152c0a93 term=2 tally=1 +51:07.632 [INFO] TestAgent.server.raft: election won: tally=1 +51:07.632 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21244 [Leader]" +51:07.632 [INFO] TestAgent.server: cluster leadership acquired +51:07.632 [INFO] TestAgent.server: New leader elected: payload=Node-f0cf2b7a-008e-19a1-fa56-b351152c0a93 +51:07.632 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21244 +51:07.633 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:07.642 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:07.642 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:07.642 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:07.642 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:07.642 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:07.642 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f0cf2b7a-008e-19a1-fa56-b351152c0a93 +51:07.642 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f0cf2b7a-008e-19a1-fa56-b351152c0a93 +51:07.642 [INFO] TestAgent.server: federation state anti-entropy synced +51:07.748 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +51:07.749 [INFO] TestAgent: Synced node info +51:07.848 [DEBUG] TestAgent: Node info in sync +51:07.848 [INFO] TestAgent: Synced service: service=web +51:07.848 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:35480 latency=713.22µs +51:07.849 [DEBUG] TestAgent: Node info in sync +51:07.849 [DEBUG] TestAgent: Service in sync: service=web +51:07.849 [INFO] TestAgent: Synced service: service=db +51:07.849 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:35480 latency=407.862µs +51:07.850 [DEBUG] TestAgent: removed service: service=web +51:07.850 [DEBUG] TestAgent: Node info in sync +51:07.850 [INFO] TestAgent: Deregistered service: service=web +51:07.850 [DEBUG] TestAgent: Service in sync: service=db +51:07.850 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/web from=127.0.0.1:35482 latency=329.043µs +51:07.850 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:35480 latency=135.998µs +51:07.851 [INFO] TestAgent: Requesting shutdown +51:07.851 [INFO] TestAgent.server: shutting down server +51:07.851 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:07.851 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:07.851 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:07.851 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:07.851 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:07.851 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:07.851 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:07.851 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:07.851 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:07.851 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:07.851 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:07.851Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:07.851Z [INFO] agent.router.manager: shutting down +51:07.851 [INFO] TestAgent: consul server down +51:07.851 [INFO] TestAgent: shutdown complete +51:07.851 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21239 network=tcp +51:07.851 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21239 network=udp +51:07.851 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21240 network=tcp +51:08.351 [INFO] TestAgent: Waiting for endpoints to shut down +51:08.351 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_Flag (0.84s) +2022-12-04T05:51:07.546Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:07.564Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:07.567 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:477c41a6-8e24-e734-c8d9-d08d0ec3883a Address:127.0.0.1:21251}]" +51:07.567 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-477c41a6-8e24-e734-c8d9-d08d0ec3883a.dc1 127.0.0.1 +51:07.568 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-477c41a6-8e24-e734-c8d9-d08d0ec3883a 127.0.0.1 +2022-12-04T05:51:07.568Z [INFO] agent.router: Initializing LAN area manager +51:07.568 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:21251 [Follower]" leader= +51:07.568 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-477c41a6-8e24-e734-c8d9-d08d0ec3883a.dc1 area=wan +51:07.568 [INFO] TestAgent.server: Adding LAN server: server="Node-477c41a6-8e24-e734-c8d9-d08d0ec3883a (Addr: tcp/127.0.0.1:21251) (DC: dc1)" +51:07.569 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21246 network=udp +51:07.569 [INFO] TestAgent: Started DNS server: address=127.0.0.1:21246 network=tcp +51:07.573 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:21247 network=tcp +51:07.573 [INFO] TestAgent: started state syncer +51:07.573 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:21252 network=tcp +51:07.637 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:07.637 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:21251 [Candidate]" term=2 +51:07.638 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:07.638 [DEBUG] TestAgent.server.raft: vote granted: from=477c41a6-8e24-e734-c8d9-d08d0ec3883a term=2 tally=1 +51:07.638 [INFO] TestAgent.server.raft: election won: tally=1 +51:07.638 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:21251 [Leader]" +51:07.638 [INFO] TestAgent.server: cluster leadership acquired +51:07.638 [INFO] TestAgent.server: New leader elected: payload=Node-477c41a6-8e24-e734-c8d9-d08d0ec3883a +51:07.638 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:21251 +51:07.638 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:07.642 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:07.642 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:07.642 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:07.642 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:07.642 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:07.642 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-477c41a6-8e24-e734-c8d9-d08d0ec3883a +51:07.642 [INFO] TestAgent.server: member joined, marking health alive: member=Node-477c41a6-8e24-e734-c8d9-d08d0ec3883a +51:07.642 [INFO] TestAgent.server: federation state anti-entropy synced +51:07.828 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +51:07.828 [INFO] TestAgent: Synced node info +51:07.828 [DEBUG] TestAgent: Node info in sync +51:07.962 [DEBUG] TestAgent: Node info in sync +51:07.963 [INFO] TestAgent: Synced service: service=web +51:07.963 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:36010 latency=706.289µs +51:07.963 [DEBUG] TestAgent: Node info in sync +51:07.963 [DEBUG] TestAgent: Service in sync: service=web +51:07.963 [INFO] TestAgent: Synced service: service=db +51:07.963 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:36010 latency=346.903µs +51:07.973 [DEBUG] TestAgent: removed service: service=web +51:07.973 [DEBUG] TestAgent: Node info in sync +51:07.973 [DEBUG] TestAgent: Service in sync: service=db +51:07.973 [INFO] TestAgent: Deregistered service: service=web +51:07.973 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/deregister/web from=127.0.0.1:36012 latency=409.089µs +51:07.974 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:36010 latency=321.57µs +51:07.980 [INFO] TestAgent: Requesting shutdown +51:07.980 [INFO] TestAgent.server: shutting down server +51:07.980 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:07.980 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:07.980 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:07.980 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:07.980 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:07.980 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:07.980 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:07.980 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:07.980 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:07.980 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:07.980 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:07.980Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:07.980Z [INFO] agent.router.manager: shutting down +51:07.980 [INFO] TestAgent: consul server down +51:07.980 [INFO] TestAgent: shutdown complete +51:07.980 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21246 network=tcp +51:07.981 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:21246 network=udp +51:07.981 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:21247 network=tcp +51:08.481 [INFO] TestAgent: Waiting for endpoints to shut down +51:08.481 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_File_id (0.97s) PASS -ok github.com/hashicorp/consul/command/services/deregister 1.639s +ok github.com/hashicorp/consul/command/services/deregister 1.014s === RUN TestCommand_noTabs === PAUSE TestCommand_noTabs === RUN TestCommand_Validation @@ -90639,281 +90810,275 @@ === RUN TestCommand_FileWithUnnamedCheck === PAUSE TestCommand_FileWithUnnamedCheck === CONT TestCommand_noTabs ---- PASS: TestCommand_noTabs (0.00s) -=== CONT TestCommand_FileWithUnnamedCheck -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestCommand_Flags === CONT TestCommand_Flags_TaggedAddresses +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestCommand_File +=== CONT TestCommand_FileWithUnnamedCheck +=== CONT TestCommand_Flags +--- PASS: TestCommand_noTabs (0.00s) === CONT TestCommand_Validation -=== RUN TestCommand_Validation/no_args_or_id -=== RUN TestCommand_Validation/args_and_-name [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range ---- PASS: TestCommand_Validation (0.02s) +=== RUN TestCommand_Validation/no_args_or_id +=== RUN TestCommand_Validation/args_and_-name +--- PASS: TestCommand_Validation (0.00s) --- PASS: TestCommand_Validation/no_args_or_id (0.00s) - --- PASS: TestCommand_Validation/args_and_-name (0.01s) -2021-10-31T23:18:54.567Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:54.615Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:54.617 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a Address:127.0.0.1:15111}]" -18:54.617 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15111 [Follower]" leader= -18:54.642 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a.dc1 127.0.0.1 -18:54.643 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a 127.0.0.1 -2021-10-31T23:18:54.643Z [INFO] agent.router: Initializing LAN area manager -18:54.643 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=udp -18:54.661 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:54.661 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15111 [Candidate]" term=2 -18:54.662 [WARN] TestAgent.server.raft: unable to get address for sever, using fallback address: id=aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a fallback=127.0.0.1:15111 error="Could not find address for server id aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a" -18:54.662 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:54.662 [DEBUG] TestAgent.server.raft: vote granted: from=aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a term=2 tally=1 -18:54.662 [INFO] TestAgent.server.raft: election won: tally=1 -18:54.665 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15111 [Leader]" -18:54.662 [INFO] TestAgent.server: Adding LAN server: server="Node-aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a (Addr: tcp/127.0.0.1:15111) (DC: dc1)" -18:54.665 [INFO] TestAgent.server: cluster leadership acquired -18:54.666 [INFO] TestAgent.server: New leader elected: payload=Node-aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a -18:54.663 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a.dc1 area=wan -18:54.663 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15106 network=tcp -18:54.666 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15111 -18:54.666 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15107 network=tcp -18:54.666 [INFO] TestAgent: started state syncer -18:54.667 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:54.667 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15112 network=tcp -18:54.667 [INFO] TestAgent: Synced node info -18:54.667 [DEBUG] TestAgent: Node info in sync -18:54.720 [DEBUG] TestAgent: Node info in sync -18:54.721 [INFO] TestAgent: Synced service: service=web -18:54.721 [DEBUG] TestAgent: Check in sync: check=service:web -18:54.721 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:33224 latency=23.767961ms -18:54.750 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:33226 latency=23.955119ms -18:54.752 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:54.752 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:54.752 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:54.752 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:54.752 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:54.752 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a -18:54.752 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a -18:54.753 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:33226 latency=304.084µs -18:54.754 [INFO] TestAgent: Requesting shutdown -18:54.754 [INFO] TestAgent.server: shutting down server -18:54.754 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:54.754 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:54.754 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:54.754 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:54.754 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:54.754 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:54.754Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:54.754Z [INFO] agent.router.manager: shutting down -18:54.754 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:54.754 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:54.754 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:54.754 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="error performing federation state anti-entropy sync: leadership lost while committing log" -18:54.754 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:54.757 [ERROR] TestAgent.server: failed to reconcile member: member="{Node-aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a 127.0.0.1 15109 map[acls:0 bootstrap:1 build:1.8.7: dc:dc1 ft_fs:1 id:aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a port:15111 raft_vsn:3 role:consul segment: vsn:2 vsn_max:3 vsn_min:2 wan_join_port:15110] alive 1 5 2 2 5 4}" error="raft is already shutdown" -18:54.758 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a -18:54.758 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a -18:54.758 [ERROR] TestAgent.server: failed to reconcile member: member="{Node-aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a 127.0.0.1 15109 map[acls:0 bootstrap:1 build:1.8.7: dc:dc1 ft_fs:1 id:aecb587a-0f72-10b1-aeeb-17a5ffe6eb9a port:15111 raft_vsn:3 role:consul segment: vsn:2 vsn_max:3 vsn_min:2 wan_join_port:15110] alive 1 5 2 2 5 4}" error="raft is already shutdown" -18:54.762 [INFO] TestAgent: consul server down -18:54.765 [INFO] TestAgent: shutdown complete -18:54.765 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=tcp -18:54.766 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15106 network=udp -18:54.766 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15107 network=tcp -18:55.269 [INFO] TestAgent: Waiting for endpoints to shut down -18:55.269 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_FileWithUnnamedCheck (0.86s) -2021-10-31T23:18:54.522Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:54.597Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:54.616 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:334a10e5-f725-81a8-6363-63b855c856f1 Address:127.0.0.1:15118}]" -18:54.617 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-334a10e5-f725-81a8-6363-63b855c856f1.dc1 127.0.0.1 -18:54.617 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-334a10e5-f725-81a8-6363-63b855c856f1 127.0.0.1 -18:54.617 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15118 [Follower]" leader= -2021-10-31T23:18:54.637Z [INFO] agent.router: Initializing LAN area manager -18:54.642 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=udp -18:54.642 [INFO] TestAgent.server: Adding LAN server: server="Node-334a10e5-f725-81a8-6363-63b855c856f1 (Addr: tcp/127.0.0.1:15118) (DC: dc1)" -18:54.642 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-334a10e5-f725-81a8-6363-63b855c856f1.dc1 area=wan -18:54.643 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15113 network=tcp -18:54.662 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15114 network=tcp -18:54.662 [INFO] TestAgent: started state syncer -18:54.663 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15119 network=tcp -18:54.664 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:54.664 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15118 [Candidate]" term=2 -18:54.665 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:54.665 [DEBUG] TestAgent.server.raft: vote granted: from=334a10e5-f725-81a8-6363-63b855c856f1 term=2 tally=1 -18:54.665 [INFO] TestAgent.server.raft: election won: tally=1 -18:54.665 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15118 [Leader]" -18:54.665 [INFO] TestAgent.server: cluster leadership acquired -18:54.665 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15118 -18:54.665 [INFO] TestAgent.server: New leader elected: payload=Node-334a10e5-f725-81a8-6363-63b855c856f1 -18:54.666 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:54.696 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:54.696 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:54.696 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:54.696 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:54.696 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:54.696 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-334a10e5-f725-81a8-6363-63b855c856f1 -18:54.696 [INFO] TestAgent.server: member joined, marking health alive: member=Node-334a10e5-f725-81a8-6363-63b855c856f1 -18:54.718 [INFO] TestAgent.server: federation state anti-entropy synced -18:54.762 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:54.762 [INFO] TestAgent: Synced node info -18:54.762 [DEBUG] TestAgent: Node info in sync -18:54.872 [DEBUG] TestAgent: Node info in sync -18:54.873 [INFO] TestAgent: Synced service: service=web -18:54.873 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:42628 latency=784.573µs -18:54.874 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:42630 latency=410.221µs -18:54.874 [INFO] TestAgent: Requesting shutdown -18:54.874 [INFO] TestAgent.server: shutting down server -18:54.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:54.874 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:54.874 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:54.874 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:54.874 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:54.874 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:54.874 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:54.875 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:54.875 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:54.875 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:54.875 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:54.875Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:54.875Z [INFO] agent.router.manager: shutting down -18:54.875 [INFO] TestAgent: consul server down -18:54.875 [INFO] TestAgent: shutdown complete -18:54.875 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=tcp -18:54.875 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15113 network=udp -18:54.875 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15114 network=tcp -18:55.377 [INFO] TestAgent: Waiting for endpoints to shut down -18:55.377 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_Flags (0.97s) -2021-10-31T23:18:54.524Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:54.642Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:54.659 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:54b2b5b6-74b0-5a8b-65f5-da8ac3884671 Address:127.0.0.1:15125}]" -18:54.659 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-54b2b5b6-74b0-5a8b-65f5-da8ac3884671.dc1 127.0.0.1 -18:54.660 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-54b2b5b6-74b0-5a8b-65f5-da8ac3884671 127.0.0.1 -2021-10-31T23:18:54.660Z [INFO] agent.router: Initializing LAN area manager -18:54.660 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15120 network=udp -18:54.660 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15125 [Follower]" leader= -18:54.661 [INFO] TestAgent.server: Adding LAN server: server="Node-54b2b5b6-74b0-5a8b-65f5-da8ac3884671 (Addr: tcp/127.0.0.1:15125) (DC: dc1)" -18:54.661 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-54b2b5b6-74b0-5a8b-65f5-da8ac3884671.dc1 area=wan -18:54.661 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15120 network=tcp -18:54.661 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15121 network=tcp -18:54.669 [INFO] TestAgent: started state syncer -18:54.670 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15126 network=tcp -18:54.713 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:54.714 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15125 [Candidate]" term=2 -18:54.714 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:54.714 [DEBUG] TestAgent.server.raft: vote granted: from=54b2b5b6-74b0-5a8b-65f5-da8ac3884671 term=2 tally=1 -18:54.714 [INFO] TestAgent.server.raft: election won: tally=1 -18:54.714 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15125 [Leader]" -18:54.717 [INFO] TestAgent.server: cluster leadership acquired -18:54.717 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15125 -18:54.718 [INFO] TestAgent.server: New leader elected: payload=Node-54b2b5b6-74b0-5a8b-65f5-da8ac3884671 -18:54.722 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:54.724 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:54.724 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:54.724 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:54.724 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:54.724 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:54.724 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-54b2b5b6-74b0-5a8b-65f5-da8ac3884671 -18:54.724 [INFO] TestAgent.server: member joined, marking health alive: member=Node-54b2b5b6-74b0-5a8b-65f5-da8ac3884671 -18:54.726 [INFO] TestAgent.server: federation state anti-entropy synced -18:54.802 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:54.802 [INFO] TestAgent: Synced node info -18:54.802 [DEBUG] TestAgent: Node info in sync -18:55.075 [DEBUG] TestAgent: Node info in sync -18:55.076 [INFO] TestAgent: Synced service: service=web -18:55.076 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:53338 latency=1.002514ms -18:55.077 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:53342 latency=326.267µs -18:55.098 [INFO] TestAgent: Requesting shutdown -18:55.098 [INFO] TestAgent.server: shutting down server -18:55.098 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:55.098 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:55.098 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:55.098 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:55.098 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:55.098 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:55.098 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:55.098 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:55.098 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:55.098 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:55.098 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:55.098Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:55.098Z [INFO] agent.router.manager: shutting down -18:55.099 [INFO] TestAgent: consul server down -18:55.099 [INFO] TestAgent: shutdown complete -18:55.099 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15120 network=tcp -18:55.099 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15120 network=udp -18:55.099 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15121 network=tcp -18:55.601 [INFO] TestAgent: Waiting for endpoints to shut down -18:55.601 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_Flags_TaggedAddresses (1.20s) -2021-10-31T23:18:54.549Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:54.614Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:54.618 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aa7259bd-edf2-48cb-db9d-89c6dc7046c3 Address:127.0.0.1:15132}]" -18:54.638 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:15132 [Follower]" leader= -18:54.642 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aa7259bd-edf2-48cb-db9d-89c6dc7046c3.dc1 127.0.0.1 -18:54.662 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aa7259bd-edf2-48cb-db9d-89c6dc7046c3 127.0.0.1 -2021-10-31T23:18:54.662Z [INFO] agent.router: Initializing LAN area manager -18:54.663 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15127 network=udp -18:54.663 [INFO] TestAgent.server: Adding LAN server: server="Node-aa7259bd-edf2-48cb-db9d-89c6dc7046c3 (Addr: tcp/127.0.0.1:15132) (DC: dc1)" -18:54.663 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aa7259bd-edf2-48cb-db9d-89c6dc7046c3.dc1 area=wan -18:54.663 [INFO] TestAgent: Started DNS server: address=127.0.0.1:15127 network=tcp -18:54.663 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:15128 network=tcp -18:54.663 [INFO] TestAgent: started state syncer -18:54.664 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:15133 network=tcp -18:54.710 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:54.710 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:15132 [Candidate]" term=2 -18:54.710 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:54.710 [DEBUG] TestAgent.server.raft: vote granted: from=aa7259bd-edf2-48cb-db9d-89c6dc7046c3 term=2 tally=1 -18:54.710 [INFO] TestAgent.server.raft: election won: tally=1 -18:54.710 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:15132 [Leader]" -18:54.718 [INFO] TestAgent.server: cluster leadership acquired -18:54.718 [INFO] TestAgent.server: New leader elected: payload=Node-aa7259bd-edf2-48cb-db9d-89c6dc7046c3 -18:54.722 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15132 -18:54.723 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:54.725 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:54.725 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:54.725 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:54.725 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:54.725 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:54.725 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aa7259bd-edf2-48cb-db9d-89c6dc7046c3 -18:54.725 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aa7259bd-edf2-48cb-db9d-89c6dc7046c3 -18:54.726 [INFO] TestAgent.server: federation state anti-entropy synced -18:54.845 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:54.846 [INFO] TestAgent: Synced node info -18:54.846 [DEBUG] TestAgent: Node info in sync -18:55.076 [DEBUG] TestAgent: Node info in sync -18:55.076 [INFO] TestAgent: Synced service: service=web -18:55.076 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:48590 latency=754.115µs -18:55.089 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:48594 latency=11.243012ms -18:55.098 [INFO] TestAgent: Requesting shutdown -18:55.099 [INFO] TestAgent.server: shutting down server -18:55.099 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:55.099 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:55.099 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:55.099 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:55.099 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:55.099 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:55.099 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:55.099 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:55.099 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:55.099 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:55.099 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:55.099Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:55.099Z [INFO] agent.router.manager: shutting down -18:55.100 [INFO] TestAgent: consul server down -18:55.100 [INFO] TestAgent: shutdown complete -18:55.100 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15127 network=tcp -18:55.100 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:15127 network=udp -18:55.100 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:15128 network=tcp -18:55.605 [INFO] TestAgent: Waiting for endpoints to shut down -18:55.605 [INFO] TestAgent: Endpoints down ---- PASS: TestCommand_File (1.20s) + --- PASS: TestCommand_Validation/args_and_-name (0.00s) +2022-12-04T05:51:08.563Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:08.592Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:08.603 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b6c6bff9-dcd1-02c7-ea84-1232fb18718a Address:127.0.0.1:31468}]" +51:08.603 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31468 [Follower]" leader= +51:08.603 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-b6c6bff9-dcd1-02c7-ea84-1232fb18718a.dc1 127.0.0.1 +51:08.604 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-b6c6bff9-dcd1-02c7-ea84-1232fb18718a 127.0.0.1 +2022-12-04T05:51:08.604Z [INFO] agent.router: Initializing LAN area manager +51:08.604 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31463 network=udp +51:08.604 [INFO] TestAgent.server: Adding LAN server: server="Node-b6c6bff9-dcd1-02c7-ea84-1232fb18718a (Addr: tcp/127.0.0.1:31468) (DC: dc1)" +51:08.604 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-b6c6bff9-dcd1-02c7-ea84-1232fb18718a.dc1 area=wan +51:08.605 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31463 network=tcp +51:08.605 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31464 network=tcp +51:08.605 [INFO] TestAgent: started state syncer +51:08.605 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31469 network=tcp +51:08.641 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:08.641 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31468 [Candidate]" term=2 +51:08.641 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:08.641 [DEBUG] TestAgent.server.raft: vote granted: from=b6c6bff9-dcd1-02c7-ea84-1232fb18718a term=2 tally=1 +51:08.641 [INFO] TestAgent.server.raft: election won: tally=1 +51:08.641 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31468 [Leader]" +51:08.641 [INFO] TestAgent.server: cluster leadership acquired +51:08.641 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31468 +51:08.641 [INFO] TestAgent.server: New leader elected: payload=Node-b6c6bff9-dcd1-02c7-ea84-1232fb18718a +51:08.642 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:08.659 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:08.659 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:08.659 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:08.659 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:08.659 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:08.659 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-b6c6bff9-dcd1-02c7-ea84-1232fb18718a +51:08.659 [INFO] TestAgent.server: member joined, marking health alive: member=Node-b6c6bff9-dcd1-02c7-ea84-1232fb18718a +51:08.661 [INFO] TestAgent.server: federation state anti-entropy synced +51:08.705 [INFO] TestAgent: Synced node info +51:08.705 [INFO] TestAgent: Synced service: service=web +51:08.705 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:57214 latency=1.628435ms +51:08.706 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:57216 latency=300.319µs +51:08.706 [INFO] TestAgent: Requesting shutdown +51:08.706 [INFO] TestAgent.server: shutting down server +51:08.706 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:08.706 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:08.706 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:08.706 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:08.706 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:08.706 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:08.706 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:08.706 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +51:08.706 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:08.707 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:08.707 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:08.707 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:08.707Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:08.707Z [INFO] agent.router.manager: shutting down +51:08.707 [INFO] TestAgent: consul server down +51:08.707 [INFO] TestAgent: shutdown complete +51:08.707 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31463 network=tcp +51:08.707 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31463 network=udp +51:08.707 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31464 network=tcp +51:09.207 [INFO] TestAgent: Waiting for endpoints to shut down +51:09.207 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_Flags (0.71s) +2022-12-04T05:51:08.580Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:08.607Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:08.609 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:746f2f37-a4e3-5da4-13cb-264eaa0b1962 Address:127.0.0.1:31447}]" +51:08.609 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= +51:08.610 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-746f2f37-a4e3-5da4-13cb-264eaa0b1962.dc1 127.0.0.1 +51:08.610 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-746f2f37-a4e3-5da4-13cb-264eaa0b1962 127.0.0.1 +2022-12-04T05:51:08.610Z [INFO] agent.router: Initializing LAN area manager +51:08.610 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=udp +51:08.610 [INFO] TestAgent.server: Adding LAN server: server="Node-746f2f37-a4e3-5da4-13cb-264eaa0b1962 (Addr: tcp/127.0.0.1:31447) (DC: dc1)" +51:08.610 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-746f2f37-a4e3-5da4-13cb-264eaa0b1962.dc1 area=wan +51:08.611 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=tcp +51:08.611 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31443 network=tcp +51:08.611 [INFO] TestAgent: started state syncer +51:08.611 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31448 network=tcp +51:08.657 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:08.657 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 +51:08.657 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:08.657 [DEBUG] TestAgent.server.raft: vote granted: from=746f2f37-a4e3-5da4-13cb-264eaa0b1962 term=2 tally=1 +51:08.657 [INFO] TestAgent.server.raft: election won: tally=1 +51:08.657 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" +51:08.658 [INFO] TestAgent.server: cluster leadership acquired +51:08.658 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31447 +51:08.659 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:08.659 [INFO] TestAgent.server: New leader elected: payload=Node-746f2f37-a4e3-5da4-13cb-264eaa0b1962 +51:08.661 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:08.661 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:08.661 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:08.661 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:08.661 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:08.663 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-746f2f37-a4e3-5da4-13cb-264eaa0b1962 +51:08.663 [INFO] TestAgent.server: member joined, marking health alive: member=Node-746f2f37-a4e3-5da4-13cb-264eaa0b1962 +51:08.665 [INFO] TestAgent.server: federation state anti-entropy synced +51:08.671 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +51:08.671 [INFO] TestAgent: Synced node info +51:08.671 [DEBUG] TestAgent: Node info in sync +51:08.853 [DEBUG] TestAgent: Node info in sync +51:08.853 [INFO] TestAgent: Synced service: service=web +51:08.853 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:36808 latency=953.613µs +51:08.854 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:36810 latency=271.284µs +51:08.854 [INFO] TestAgent: Requesting shutdown +51:08.855 [INFO] TestAgent.server: shutting down server +51:08.855 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:08.855 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:08.855 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:08.855 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:08.855 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:08.855 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:08.855 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:08.855 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:08.855 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +51:08.855 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:08.855 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:51:08.855Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:08.855Z [INFO] agent.router.manager: shutting down +51:08.855 [INFO] TestAgent: consul server down +51:08.855 [INFO] TestAgent: shutdown complete +51:08.855 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp +51:08.855 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp +51:08.855 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp +51:09.355 [INFO] TestAgent: Waiting for endpoints to shut down +51:09.355 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_Flags_TaggedAddresses (0.86s) +2022-12-04T05:51:08.570Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:08.591Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:08.593 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:52bd7b65-c208-5e04-9024-eba56cd7c61a Address:127.0.0.1:31461}]" +51:08.594 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-52bd7b65-c208-5e04-9024-eba56cd7c61a.dc1 127.0.0.1 +51:08.595 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-52bd7b65-c208-5e04-9024-eba56cd7c61a 127.0.0.1 +2022-12-04T05:51:08.595Z [INFO] agent.router: Initializing LAN area manager +51:08.596 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31456 network=udp +51:08.596 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31461 [Follower]" leader= +51:08.596 [INFO] TestAgent.server: Adding LAN server: server="Node-52bd7b65-c208-5e04-9024-eba56cd7c61a (Addr: tcp/127.0.0.1:31461) (DC: dc1)" +51:08.596 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-52bd7b65-c208-5e04-9024-eba56cd7c61a.dc1 area=wan +51:08.597 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31456 network=tcp +51:08.602 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31457 network=tcp +51:08.602 [INFO] TestAgent: started state syncer +51:08.602 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31462 network=tcp +51:08.657 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:08.657 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31461 [Candidate]" term=2 +51:08.658 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:08.658 [DEBUG] TestAgent.server.raft: vote granted: from=52bd7b65-c208-5e04-9024-eba56cd7c61a term=2 tally=1 +51:08.658 [INFO] TestAgent.server.raft: election won: tally=1 +51:08.658 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31461 [Leader]" +51:08.658 [INFO] TestAgent.server: cluster leadership acquired +51:08.658 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31461 +51:08.659 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:08.659 [INFO] TestAgent.server: New leader elected: payload=Node-52bd7b65-c208-5e04-9024-eba56cd7c61a +51:08.661 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:08.661 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:08.661 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:08.661 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:08.661 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:08.661 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-52bd7b65-c208-5e04-9024-eba56cd7c61a +51:08.661 [INFO] TestAgent.server: member joined, marking health alive: member=Node-52bd7b65-c208-5e04-9024-eba56cd7c61a +51:08.664 [INFO] TestAgent.server: federation state anti-entropy synced +51:08.862 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +51:08.863 [INFO] TestAgent: Synced node info +51:08.863 [DEBUG] TestAgent: Node info in sync +51:08.863 [DEBUG] TestAgent: Node info in sync +51:08.864 [INFO] TestAgent: Synced service: service=web +51:08.864 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:57872 latency=917.007µs +51:08.865 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:57874 latency=244.652µs +51:08.865 [INFO] TestAgent: Requesting shutdown +51:08.865 [INFO] TestAgent.server: shutting down server +51:08.865 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:08.865 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:08.865 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:08.865 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:08.865 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:08.865 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:08.865 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:08.865 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:08.865 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:08.865 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:08.865 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:08.865Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:08.865Z [INFO] agent.router.manager: shutting down +51:08.865 [INFO] TestAgent: consul server down +51:08.865 [INFO] TestAgent: shutdown complete +51:08.865 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31456 network=tcp +51:08.865 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31456 network=udp +51:08.865 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31457 network=tcp +51:09.366 [INFO] TestAgent: Waiting for endpoints to shut down +51:09.366 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_File (0.86s) +2022-12-04T05:51:08.561Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:08.583Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:08.588 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cf2c122a-0c4c-e6c8-e202-d4b774e19c72 Address:127.0.0.1:31454}]" +51:08.589 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31454 [Follower]" leader= +51:08.589 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cf2c122a-0c4c-e6c8-e202-d4b774e19c72.dc1 127.0.0.1 +51:08.590 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cf2c122a-0c4c-e6c8-e202-d4b774e19c72 127.0.0.1 +2022-12-04T05:51:08.590Z [INFO] agent.router: Initializing LAN area manager +51:08.590 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31449 network=udp +51:08.590 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cf2c122a-0c4c-e6c8-e202-d4b774e19c72.dc1 area=wan +51:08.591 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31449 network=tcp +51:08.591 [INFO] TestAgent.server: Adding LAN server: server="Node-cf2c122a-0c4c-e6c8-e202-d4b774e19c72 (Addr: tcp/127.0.0.1:31454) (DC: dc1)" +51:08.591 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31450 network=tcp +51:08.591 [INFO] TestAgent: started state syncer +51:08.592 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31455 network=tcp +51:08.641 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:08.641 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31454 [Candidate]" term=2 +51:08.641 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:08.641 [DEBUG] TestAgent.server.raft: vote granted: from=cf2c122a-0c4c-e6c8-e202-d4b774e19c72 term=2 tally=1 +51:08.641 [INFO] TestAgent.server.raft: election won: tally=1 +51:08.641 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31454 [Leader]" +51:08.642 [INFO] TestAgent.server: cluster leadership acquired +51:08.642 [INFO] TestAgent.server: New leader elected: payload=Node-cf2c122a-0c4c-e6c8-e202-d4b774e19c72 +51:08.642 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31454 +51:08.643 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:08.660 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:08.660 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:08.660 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:08.660 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:08.660 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:08.660 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cf2c122a-0c4c-e6c8-e202-d4b774e19c72 +51:08.660 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cf2c122a-0c4c-e6c8-e202-d4b774e19c72 +51:08.663 [INFO] TestAgent.server: federation state anti-entropy synced +51:08.701 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +51:08.701 [INFO] TestAgent: Synced node info +51:08.887 [DEBUG] TestAgent: Node info in sync +51:08.892 [INFO] TestAgent: Synced service: service=web +51:08.893 [DEBUG] TestAgent: Check in sync: check=service:web +51:08.893 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:33980 latency=6.525492ms +51:08.894 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/services from=127.0.0.1:33982 latency=746.848µs +51:08.895 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/checks from=127.0.0.1:33982 latency=276.72µs +51:08.896 [INFO] TestAgent: Requesting shutdown +51:08.896 [INFO] TestAgent.server: shutting down server +51:08.896 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:08.896 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:08.896 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:08.896 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:08.896 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:08.896 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:08.896 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:08.896 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:08.896 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:08.896 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:08.896 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:08.896Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:08.896Z [INFO] agent.router.manager: shutting down +51:08.896 [INFO] TestAgent: consul server down +51:08.896 [INFO] TestAgent: shutdown complete +51:08.896 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31449 network=tcp +51:08.896 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31449 network=udp +51:08.896 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31450 network=tcp +51:09.396 [INFO] TestAgent: Waiting for endpoints to shut down +51:09.396 [INFO] TestAgent: Endpoints down +--- PASS: TestCommand_FileWithUnnamedCheck (0.90s) PASS -ok github.com/hashicorp/consul/command/services/register 1.343s +ok github.com/hashicorp/consul/command/services/register 0.962s === RUN TestSnapshotCommand_noTabs === PAUSE TestSnapshotCommand_noTabs === CONT TestSnapshotCommand_noTabs --- PASS: TestSnapshotCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/snapshot 0.040s +ok github.com/hashicorp/consul/command/snapshot 0.005s === RUN TestSnapshotInspectCommand_noTabs === PAUSE TestSnapshotInspectCommand_noTabs === RUN TestSnapshotInspectCommand_Validation @@ -90921,78 +91086,76 @@ === RUN TestSnapshotInspectCommand === PAUSE TestSnapshotInspectCommand === CONT TestSnapshotInspectCommand_noTabs ---- PASS: TestSnapshotInspectCommand_noTabs (0.00s) === CONT TestSnapshotInspectCommand -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestSnapshotInspectCommand_noTabs (0.00s) === CONT TestSnapshotInspectCommand_Validation --- PASS: TestSnapshotInspectCommand_Validation (0.00s) +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -2021-10-31T23:18:56.000Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:56.048Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:56.051 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ef75e777-b1f5-b1cd-09e7-8dbea63d6e4e Address:127.0.0.1:30426}]" -18:56.052 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:30426 [Follower]" leader= -18:56.052 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ef75e777-b1f5-b1cd-09e7-8dbea63d6e4e.dc1 127.0.0.1 -18:56.053 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ef75e777-b1f5-b1cd-09e7-8dbea63d6e4e 127.0.0.1 -2021-10-31T23:18:56.053Z [INFO] agent.router: Initializing LAN area manager -18:56.054 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ef75e777-b1f5-b1cd-09e7-8dbea63d6e4e.dc1 area=wan -18:56.054 [INFO] TestAgent.server: Adding LAN server: server="Node-ef75e777-b1f5-b1cd-09e7-8dbea63d6e4e (Addr: tcp/127.0.0.1:30426) (DC: dc1)" -18:56.058 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=udp -18:56.058 [INFO] TestAgent: Started DNS server: address=127.0.0.1:30421 network=tcp -18:56.059 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:30422 network=tcp -18:56.059 [INFO] TestAgent: started state syncer -18:56.059 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:30427 network=tcp -18:56.109 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:56.109 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:30426 [Candidate]" term=2 -18:56.110 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:56.110 [DEBUG] TestAgent.server.raft: vote granted: from=ef75e777-b1f5-b1cd-09e7-8dbea63d6e4e term=2 tally=1 -18:56.110 [INFO] TestAgent.server.raft: election won: tally=1 -18:56.110 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:30426 [Leader]" -18:56.110 [INFO] TestAgent.server: cluster leadership acquired -18:56.110 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:30426 -18:56.111 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:56.113 [INFO] TestAgent.server: New leader elected: payload=Node-ef75e777-b1f5-b1cd-09e7-8dbea63d6e4e -18:56.143 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:56.143 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:56.143 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:56.143 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:56.143 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:56.143 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ef75e777-b1f5-b1cd-09e7-8dbea63d6e4e -18:56.143 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ef75e777-b1f5-b1cd-09e7-8dbea63d6e4e -18:56.145 [INFO] TestAgent.server: federation state anti-entropy synced -18:56.198 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:56.199 [INFO] TestAgent: Synced node info -18:56.199 [DEBUG] TestAgent: Node info in sync -18:56.262 [INFO] TestAgent.server.fsm: snapshot created: duration=73.875µs -18:56.262 [INFO] TestAgent.server.raft: starting snapshot up to: index=12 -18:56.262 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotInspectCommand-TestAgent-agent340672596/raft/snapshots/2-12-1635722336262.tmp -18:56.263 [INFO] TestAgent.server.raft: snapshot complete up to: index=12 -18:56.271 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:45316 latency=9.055825ms -18:56.277 [INFO] TestAgent: Requesting shutdown -18:56.278 [INFO] TestAgent.server: shutting down server -18:56.278 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:56.278 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:56.278 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:56.278 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:56.278 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:56.278 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:56.278 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:56.279 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:56.279 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:56.279 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:56.279 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:56.279Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:56.279Z [INFO] agent.router.manager: shutting down -18:56.279 [INFO] TestAgent: consul server down -18:56.279 [INFO] TestAgent: shutdown complete -18:56.279 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=tcp -18:56.279 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:30421 network=udp -18:56.279 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:30422 network=tcp -18:56.780 [INFO] TestAgent: Waiting for endpoints to shut down -18:56.780 [INFO] TestAgent: Endpoints down ---- PASS: TestSnapshotInspectCommand (0.90s) +2022-12-04T05:51:11.206Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:11.221Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:11.223 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8ed0406b-831b-c37d-0ad7-02fcad8e851f Address:127.0.0.1:12048}]" +51:11.223 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:12048 [Follower]" leader= +51:11.224 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8ed0406b-831b-c37d-0ad7-02fcad8e851f.dc1 127.0.0.1 +51:11.224 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8ed0406b-831b-c37d-0ad7-02fcad8e851f 127.0.0.1 +2022-12-04T05:51:11.224Z [INFO] agent.router: Initializing LAN area manager +51:11.225 [INFO] TestAgent.server: Adding LAN server: server="Node-8ed0406b-831b-c37d-0ad7-02fcad8e851f (Addr: tcp/127.0.0.1:12048) (DC: dc1)" +51:11.225 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8ed0406b-831b-c37d-0ad7-02fcad8e851f.dc1 area=wan +51:11.225 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=udp +51:11.225 [INFO] TestAgent: Started DNS server: address=127.0.0.1:12043 network=tcp +51:11.225 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:12044 network=tcp +51:11.225 [INFO] TestAgent: started state syncer +51:11.225 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:12049 network=tcp +51:11.271 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:11.271 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:12048 [Candidate]" term=2 +51:11.272 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:11.272 [DEBUG] TestAgent.server.raft: vote granted: from=8ed0406b-831b-c37d-0ad7-02fcad8e851f term=2 tally=1 +51:11.272 [INFO] TestAgent.server.raft: election won: tally=1 +51:11.272 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:12048 [Leader]" +51:11.272 [INFO] TestAgent.server: cluster leadership acquired +51:11.272 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:12048 +51:11.272 [INFO] TestAgent.server: New leader elected: payload=Node-8ed0406b-831b-c37d-0ad7-02fcad8e851f +51:11.273 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:11.289 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:11.289 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:11.289 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:11.289 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:11.289 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:11.289 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8ed0406b-831b-c37d-0ad7-02fcad8e851f +51:11.289 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8ed0406b-831b-c37d-0ad7-02fcad8e851f +51:11.290 [INFO] TestAgent.server: federation state anti-entropy synced +51:11.482 [INFO] TestAgent.server.fsm: snapshot created: duration=51.775µs +51:11.482 [INFO] TestAgent.server.raft: starting snapshot up to: index=11 +51:11.482 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotInspectCommand-TestAgent-agent489118376/raft/snapshots/2-11-1670133071482.tmp +51:11.484 [INFO] TestAgent.server.raft: snapshot complete up to: index=11 +51:11.491 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:48016 latency=8.245491ms +51:11.493 [INFO] TestAgent: Requesting shutdown +51:11.493 [INFO] TestAgent.server: shutting down server +51:11.493 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:11.493 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:11.493 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:11.493 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:11.493 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:11.493 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:11.493 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +51:11.493 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:11.493 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:11.493 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:11.493 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:11.493 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:11.493Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:11.493Z [INFO] agent.router.manager: shutting down +51:11.493 [INFO] TestAgent: consul server down +51:11.493 [INFO] TestAgent: shutdown complete +51:11.493 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=tcp +51:11.493 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:12043 network=udp +51:11.493 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:12044 network=tcp +51:11.993 [INFO] TestAgent: Waiting for endpoints to shut down +51:11.993 [INFO] TestAgent: Endpoints down +--- PASS: TestSnapshotInspectCommand (0.82s) PASS -ok github.com/hashicorp/consul/command/snapshot/inspect 1.279s +ok github.com/hashicorp/consul/command/snapshot/inspect 0.876s === RUN TestSnapshotRestoreCommand_noTabs === PAUSE TestSnapshotRestoreCommand_noTabs === RUN TestSnapshotRestoreCommand_Validation @@ -91002,12 +91165,12 @@ === RUN TestSnapshotRestoreCommand_TruncatedSnapshot === PAUSE TestSnapshotRestoreCommand_TruncatedSnapshot === CONT TestSnapshotRestoreCommand_noTabs ---- PASS: TestSnapshotRestoreCommand_noTabs (0.00s) === CONT TestSnapshotRestoreCommand_TruncatedSnapshot -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... -=== CONT TestSnapshotRestoreCommand === CONT TestSnapshotRestoreCommand_Validation ---- PASS: TestSnapshotRestoreCommand_Validation (0.01s) +=== CONT TestSnapshotRestoreCommand +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +--- PASS: TestSnapshotRestoreCommand_Validation (0.00s) +--- PASS: TestSnapshotRestoreCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range === RUN TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_200_bytes_from_end @@ -91016,172 +91179,171 @@ === RUN TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_4_bytes_from_end === RUN TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_2_bytes_from_end === RUN TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_1_bytes_from_end -2021-10-31T23:18:56.675Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:56.703Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:56.718 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:36f4bb55-3f33-26a6-24c8-c0d7cc08682d Address:127.0.0.1:28391}]" -18:56.719 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-36f4bb55-3f33-26a6-24c8-c0d7cc08682d.dc1 127.0.0.1 -18:56.719 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28391 [Follower]" leader= -18:56.721 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-36f4bb55-3f33-26a6-24c8-c0d7cc08682d 127.0.0.1 -2021-10-31T23:18:56.721Z [INFO] agent.router: Initializing LAN area manager -18:56.721 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28386 network=udp -18:56.721 [INFO] TestAgent.server: Adding LAN server: server="Node-36f4bb55-3f33-26a6-24c8-c0d7cc08682d (Addr: tcp/127.0.0.1:28391) (DC: dc1)" -18:56.721 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-36f4bb55-3f33-26a6-24c8-c0d7cc08682d.dc1 area=wan -18:56.721 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28386 network=tcp -18:56.742 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28387 network=tcp -18:56.742 [INFO] TestAgent: started state syncer -18:56.742 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28392 network=tcp -18:56.787 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:56.787 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28391 [Candidate]" term=2 -18:56.787 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:56.787 [DEBUG] TestAgent.server.raft: vote granted: from=36f4bb55-3f33-26a6-24c8-c0d7cc08682d term=2 tally=1 -18:56.787 [INFO] TestAgent.server.raft: election won: tally=1 -18:56.787 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28391 [Leader]" -18:56.787 [INFO] TestAgent.server: cluster leadership acquired -18:56.788 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28391 -18:56.788 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:56.789 [INFO] TestAgent.server: New leader elected: payload=Node-36f4bb55-3f33-26a6-24c8-c0d7cc08682d -18:56.807 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:56.807 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:56.807 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:56.807 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:56.807 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:56.808 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-36f4bb55-3f33-26a6-24c8-c0d7cc08682d -18:56.808 [INFO] TestAgent.server: member joined, marking health alive: member=Node-36f4bb55-3f33-26a6-24c8-c0d7cc08682d -18:56.808 [INFO] TestAgent.server: federation state anti-entropy synced -18:56.926 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:56.926 [INFO] TestAgent: Synced node info -18:57.053 [INFO] TestAgent.server.fsm: snapshot created: duration=87.531µs -18:57.053 [INFO] TestAgent.server.raft: starting snapshot up to: index=12 -18:57.053 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotRestoreCommand-TestAgent-agent698583612/raft/snapshots/2-12-1635722337053.tmp -18:57.061 [INFO] TestAgent.server.raft: snapshot complete up to: index=12 -18:57.063 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:49660 latency=10.570914ms -18:57.065 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotRestoreCommand-TestAgent-agent698583612/raft/snapshots/2-13-1635722337065.tmp -18:57.065 [INFO] TestAgent.server.raft: copied to local snapshot: bytes=5146 -18:57.067 [INFO] TestAgent.server.raft: restored user snapshot: index=1 -18:57.067 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:57.067 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:57.067 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:57.067 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:57.067 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:57.067 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:57.067 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:57.067 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:57.067 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:57.067 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28391 -18:57.068 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:57.068 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:57.068 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:57.068 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:57.068 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:49662 latency=3.470168ms -18:57.068 [INFO] TestAgent: Requesting shutdown -18:57.068 [INFO] TestAgent.server: shutting down server -18:57.068 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:57.068 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:57.068 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:57.068 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:57.069 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:57.069 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:57.069 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:57.069 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:57.069 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:57.069 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:57.069 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:57.069Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:57.069Z [INFO] agent.router.manager: shutting down -18:57.069 [INFO] TestAgent: consul server down -18:57.069 [INFO] TestAgent: shutdown complete -18:57.069 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28386 network=tcp -18:57.069 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28386 network=udp -18:57.069 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28387 network=tcp -18:57.573 [INFO] TestAgent: Waiting for endpoints to shut down -18:57.573 [INFO] TestAgent: Endpoints down ---- PASS: TestSnapshotRestoreCommand (0.98s) -2021-10-31T23:18:56.681Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:18:56.722Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -18:56.738 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:83deaca1-df82-6e75-66c4-6d5b32acb70f Address:127.0.0.1:28384}]" -18:56.739 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-83deaca1-df82-6e75-66c4-6d5b32acb70f.dc1 127.0.0.1 -18:56.739 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:28384 [Follower]" leader= -18:56.740 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-83deaca1-df82-6e75-66c4-6d5b32acb70f 127.0.0.1 -2021-10-31T23:18:56.740Z [INFO] agent.router: Initializing LAN area manager -18:56.740 [INFO] TestAgent.server: Adding LAN server: server="Node-83deaca1-df82-6e75-66c4-6d5b32acb70f (Addr: tcp/127.0.0.1:28384) (DC: dc1)" -18:56.740 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-83deaca1-df82-6e75-66c4-6d5b32acb70f.dc1 area=wan -18:56.741 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=udp -18:56.741 [INFO] TestAgent: Started DNS server: address=127.0.0.1:28379 network=tcp -18:56.743 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:28380 network=tcp -18:56.743 [INFO] TestAgent: started state syncer -18:56.743 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:28385 network=tcp -18:56.805 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -18:56.805 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:28384 [Candidate]" term=2 -18:56.805 [DEBUG] TestAgent.server.raft: votes: needed=1 -18:56.805 [DEBUG] TestAgent.server.raft: vote granted: from=83deaca1-df82-6e75-66c4-6d5b32acb70f term=2 tally=1 -18:56.805 [INFO] TestAgent.server.raft: election won: tally=1 -18:56.805 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:28384 [Leader]" -18:56.805 [INFO] TestAgent.server: cluster leadership acquired -18:56.806 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:28384 -18:56.806 [INFO] TestAgent.server: New leader elected: payload=Node-83deaca1-df82-6e75-66c4-6d5b32acb70f -18:56.807 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -18:56.808 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -18:56.808 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -18:56.808 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -18:56.808 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -18:56.808 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -18:56.808 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-83deaca1-df82-6e75-66c4-6d5b32acb70f -18:56.808 [INFO] TestAgent.server: member joined, marking health alive: member=Node-83deaca1-df82-6e75-66c4-6d5b32acb70f -18:56.809 [INFO] TestAgent.server: federation state anti-entropy synced -18:57.038 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/blob from=127.0.0.1:48084 latency=12.351741ms -18:57.053 [INFO] TestAgent.server.fsm: snapshot created: duration=112.974µs -18:57.053 [INFO] TestAgent.server.raft: starting snapshot up to: index=12 -18:57.053 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotRestoreCommand_TruncatedSnapshot-TestAgent-agent599890961/raft/snapshots/2-12-1635722337053.tmp -18:57.059 [INFO] TestAgent.server.raft: snapshot complete up to: index=12 -18:57.086 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:48084 latency=33.878373ms -18:57.092 [ERROR] TestAgent.server: Failed to close snapshot decompressor: error="unexpected EOF" -18:57.095 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:48090 error="failed to read snapshot file: failed reading snapshot: unexpected EOF" -18:57.095 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:48090 latency=6.412709ms -18:57.106 [ERROR] TestAgent.server: Failed to close snapshot decompressor: error="unexpected EOF" -18:57.106 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:48092 error="failed to read snapshot file: failed reading snapshot: unexpected EOF" -18:57.112 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:48092 latency=7.381591ms -18:57.122 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -18:57.123 [INFO] TestAgent: Synced node info -18:57.123 [DEBUG] TestAgent: Node info in sync -18:57.129 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:48094 error="unexpected EOF" -18:57.129 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:48094 latency=2.200936ms -18:57.145 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:48096 error="unexpected EOF" -18:57.145 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:48096 latency=2.138106ms -18:57.153 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:48098 error="unexpected EOF" -18:57.153 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:48098 latency=1.87128ms -18:57.160 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:48100 error="unexpected EOF" -18:57.160 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:48100 latency=1.994322ms -18:57.161 [INFO] TestAgent: Requesting shutdown -18:57.161 [INFO] TestAgent.server: shutting down server -18:57.161 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -18:57.161 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -18:57.161 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -18:57.161 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -18:57.161 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -18:57.161 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -18:57.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -18:57.161 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -18:57.161 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -18:57.161 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -18:57.161 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:18:57.161Z [INFO] agent.router.manager: shutting down -2021-10-31T23:18:57.161Z [INFO] agent.router.manager: shutting down -18:57.161 [INFO] TestAgent: consul server down -18:57.161 [INFO] TestAgent: shutdown complete -18:57.161 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=tcp -18:57.161 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:28379 network=udp -18:57.161 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:28380 network=tcp -18:57.666 [INFO] TestAgent: Waiting for endpoints to shut down -18:57.666 [INFO] TestAgent: Endpoints down ---- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot (1.08s) - --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_200_bytes_from_end (0.01s) - --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_16_bytes_from_end (0.03s) - --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_8_bytes_from_end (0.01s) - --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_4_bytes_from_end (0.02s) - --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_2_bytes_from_end (0.00s) - --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_1_bytes_from_end (0.01s) +2022-12-04T05:51:11.440Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:11.451Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:11.453 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ec998420-0e7b-4c25-e528-ed40f3db0736 Address:127.0.0.1:14090}]" +51:11.454 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14090 [Follower]" leader= +51:11.454 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-ec998420-0e7b-4c25-e528-ed40f3db0736.dc1 127.0.0.1 +51:11.454 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-ec998420-0e7b-4c25-e528-ed40f3db0736 127.0.0.1 +2022-12-04T05:51:11.454Z [INFO] agent.router: Initializing LAN area manager +51:11.455 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=udp +51:11.455 [INFO] TestAgent.server: Adding LAN server: server="Node-ec998420-0e7b-4c25-e528-ed40f3db0736 (Addr: tcp/127.0.0.1:14090) (DC: dc1)" +51:11.455 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14085 network=tcp +51:11.455 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-ec998420-0e7b-4c25-e528-ed40f3db0736.dc1 area=wan +51:11.465 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14086 network=tcp +51:11.466 [INFO] TestAgent: started state syncer +51:11.466 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14091 network=tcp +51:11.517 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:11.517 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14090 [Candidate]" term=2 +51:11.517 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:11.517 [DEBUG] TestAgent.server.raft: vote granted: from=ec998420-0e7b-4c25-e528-ed40f3db0736 term=2 tally=1 +51:11.517 [INFO] TestAgent.server.raft: election won: tally=1 +51:11.517 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14090 [Leader]" +51:11.517 [INFO] TestAgent.server: cluster leadership acquired +51:11.518 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14090 +51:11.518 [INFO] TestAgent.server: New leader elected: payload=Node-ec998420-0e7b-4c25-e528-ed40f3db0736 +51:11.518 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:11.533 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:11.533 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:11.533 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:11.533 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:11.533 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:11.534 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-ec998420-0e7b-4c25-e528-ed40f3db0736 +51:11.534 [INFO] TestAgent.server: member joined, marking health alive: member=Node-ec998420-0e7b-4c25-e528-ed40f3db0736 +51:11.534 [INFO] TestAgent.server: federation state anti-entropy synced +51:11.695 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/blob from=127.0.0.1:45806 latency=5.281744ms +51:11.696 [INFO] TestAgent.server.fsm: snapshot created: duration=55.334µs +51:11.696 [INFO] TestAgent.server.raft: starting snapshot up to: index=12 +51:11.696 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotRestoreCommand_TruncatedSnapshot-TestAgent-agent545876895/raft/snapshots/2-12-1670133071696.tmp +51:11.697 [INFO] TestAgent.server.raft: snapshot complete up to: index=12 +51:11.706 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:45806 latency=9.28491ms +51:11.709 [ERROR] TestAgent.server: Failed to close snapshot decompressor: error="unexpected EOF" +51:11.709 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:45808 error="failed to read snapshot file: failed reading snapshot: unexpected EOF" +51:11.709 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:45808 latency=2.095137ms +51:11.716 [ERROR] TestAgent.server: Failed to close snapshot decompressor: error="unexpected EOF" +51:11.716 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:45810 error="failed to read snapshot file: failed reading snapshot: unexpected EOF" +51:11.716 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:45810 latency=2.213474ms +51:11.721 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:45812 error="unexpected EOF" +51:11.721 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:45812 latency=1.688957ms +51:11.724 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:45814 error="unexpected EOF" +51:11.724 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:45814 latency=1.457982ms +51:11.732 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:45816 error="unexpected EOF" +51:11.732 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:45816 latency=1.808998ms +51:11.735 [ERROR] TestAgent.http: Request error: method=PUT url=/v1/snapshot from=127.0.0.1:45818 error="unexpected EOF" +51:11.735 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:45818 latency=2.273566ms +51:11.736 [INFO] TestAgent: Requesting shutdown +51:11.736 [INFO] TestAgent.server: shutting down server +51:11.736 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:11.736 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:11.736 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:11.736 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:11.736 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:11.736 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:11.736 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:11.736 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +51:11.736 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:11.736 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:11.736 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:11.736 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:11.736Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:11.736Z [INFO] agent.router.manager: shutting down +51:11.736 [INFO] TestAgent: consul server down +51:11.736 [INFO] TestAgent: shutdown complete +51:11.736 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=tcp +51:11.736 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14085 network=udp +51:11.736 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14086 network=tcp +51:12.237 [INFO] TestAgent: Waiting for endpoints to shut down +51:12.237 [INFO] TestAgent: Endpoints down +--- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot (0.83s) + --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_200_bytes_from_end (0.00s) + --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_16_bytes_from_end (0.01s) + --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_8_bytes_from_end (0.00s) + --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_4_bytes_from_end (0.00s) + --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_2_bytes_from_end (0.01s) + --- PASS: TestSnapshotRestoreCommand_TruncatedSnapshot/truncate_1_bytes_from_end (0.00s) +2022-12-04T05:51:11.445Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:11.470Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:11.471 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d89dfba1-cfba-b5c9-3db5-7fe470c1d855 Address:127.0.0.1:14097}]" +51:11.471 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:14097 [Follower]" leader= +51:11.472 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-d89dfba1-cfba-b5c9-3db5-7fe470c1d855.dc1 127.0.0.1 +51:11.472 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-d89dfba1-cfba-b5c9-3db5-7fe470c1d855 127.0.0.1 +2022-12-04T05:51:11.472Z [INFO] agent.router: Initializing LAN area manager +51:11.473 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-d89dfba1-cfba-b5c9-3db5-7fe470c1d855.dc1 area=wan +51:11.473 [INFO] TestAgent.server: Adding LAN server: server="Node-d89dfba1-cfba-b5c9-3db5-7fe470c1d855 (Addr: tcp/127.0.0.1:14097) (DC: dc1)" +51:11.473 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=udp +51:11.473 [INFO] TestAgent: Started DNS server: address=127.0.0.1:14092 network=tcp +51:11.473 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:14093 network=tcp +51:11.473 [INFO] TestAgent: started state syncer +51:11.473 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:14098 network=tcp +51:11.532 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:11.532 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:14097 [Candidate]" term=2 +51:11.533 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:11.533 [DEBUG] TestAgent.server.raft: vote granted: from=d89dfba1-cfba-b5c9-3db5-7fe470c1d855 term=2 tally=1 +51:11.533 [INFO] TestAgent.server.raft: election won: tally=1 +51:11.533 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:14097 [Leader]" +51:11.533 [INFO] TestAgent.server: cluster leadership acquired +51:11.533 [INFO] TestAgent.server: New leader elected: payload=Node-d89dfba1-cfba-b5c9-3db5-7fe470c1d855 +51:11.533 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14097 +51:11.534 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:11.535 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:11.535 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:11.535 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:11.535 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:11.535 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:11.535 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-d89dfba1-cfba-b5c9-3db5-7fe470c1d855 +51:11.535 [INFO] TestAgent.server: member joined, marking health alive: member=Node-d89dfba1-cfba-b5c9-3db5-7fe470c1d855 +51:11.536 [INFO] TestAgent.server: federation state anti-entropy synced +51:11.538 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +51:11.538 [INFO] TestAgent: Synced node info +51:11.538 [DEBUG] TestAgent: Node info in sync +51:11.903 [INFO] TestAgent.server.fsm: snapshot created: duration=56.527µs +51:11.903 [INFO] TestAgent.server.raft: starting snapshot up to: index=12 +51:11.903 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotRestoreCommand-TestAgent-agent025315442/raft/snapshots/2-12-1670133071903.tmp +51:11.904 [INFO] TestAgent.server.raft: snapshot complete up to: index=12 +51:11.905 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:32774 latency=1.798121ms +51:11.907 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotRestoreCommand-TestAgent-agent025315442/raft/snapshots/2-13-1670133071907.tmp +51:11.907 [INFO] TestAgent.server.raft: copied to local snapshot: bytes=5146 +51:11.908 [INFO] TestAgent.server.raft: restored user snapshot: index=1 +51:11.908 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:11.908 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:11.908 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:11.908 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:11.908 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:11.908 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:11.908 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:11.908 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:14097 +51:11.909 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:11.909 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:11.909 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:11.909 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:11.909 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:32776 latency=2.666192ms +51:11.909 [INFO] TestAgent: Requesting shutdown +51:11.909 [INFO] TestAgent.server: shutting down server +51:11.909 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:11.909 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:11.909 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:11.909 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:11.909 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:11.909 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:11.909 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:11.909 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:11.909 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:11.909 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:11.909 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:11.909 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:11.909 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:11.909Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:11.909Z [INFO] agent.router.manager: shutting down +51:11.910 [INFO] TestAgent: consul server down +51:11.910 [INFO] TestAgent: shutdown complete +51:11.910 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=tcp +51:11.910 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:14092 network=udp +51:11.910 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:14093 network=tcp +51:12.410 [INFO] TestAgent: Waiting for endpoints to shut down +51:12.410 [INFO] TestAgent: Endpoints down +--- PASS: TestSnapshotRestoreCommand (1.01s) PASS -ok github.com/hashicorp/consul/command/snapshot/restore 1.163s +ok github.com/hashicorp/consul/command/snapshot/restore 1.051s === RUN TestSnapshotSaveCommand_noTabs === PAUSE TestSnapshotSaveCommand_noTabs === RUN TestSnapshotSaveCommand_Validation @@ -91191,170 +91353,171 @@ === RUN TestSnapshotSaveCommand_TruncatedStream === PAUSE TestSnapshotSaveCommand_TruncatedStream === CONT TestSnapshotSaveCommand_noTabs ---- PASS: TestSnapshotSaveCommand_noTabs (0.00s) === CONT TestSnapshotSaveCommand_TruncatedStream -[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... === CONT TestSnapshotSaveCommand_Validation +[INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... +=== CONT TestSnapshotSaveCommand +--- PASS: TestSnapshotSaveCommand_noTabs (0.00s) [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range -=== CONT TestSnapshotSaveCommand ---- PASS: TestSnapshotSaveCommand_Validation (0.02s) +--- PASS: TestSnapshotSaveCommand_Validation (0.00s) === RUN TestSnapshotSaveCommand_TruncatedStream/truncate_200_bytes_from_end === RUN TestSnapshotSaveCommand_TruncatedStream/truncate_16_bytes_from_end === RUN TestSnapshotSaveCommand_TruncatedStream/truncate_8_bytes_from_end === RUN TestSnapshotSaveCommand_TruncatedStream/truncate_4_bytes_from_end === RUN TestSnapshotSaveCommand_TruncatedStream/truncate_2_bytes_from_end === RUN TestSnapshotSaveCommand_TruncatedStream/truncate_1_bytes_from_end -2021-10-31T23:19:04.240Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:19:04.259Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -19:04.262 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8e3701c5-85e2-9dfe-8046-d662bc20b8b9 Address:127.0.0.1:16139}]" -19:04.267 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-8e3701c5-85e2-9dfe-8046-d662bc20b8b9.dc1 127.0.0.1 -19:04.271 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16139 [Follower]" leader= -19:04.271 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-8e3701c5-85e2-9dfe-8046-d662bc20b8b9 127.0.0.1 -2021-10-31T23:19:04.272Z [INFO] agent.router: Initializing LAN area manager -19:04.273 [INFO] TestAgent.server: Adding LAN server: server="Node-8e3701c5-85e2-9dfe-8046-d662bc20b8b9 (Addr: tcp/127.0.0.1:16139) (DC: dc1)" -19:04.273 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=udp -19:04.273 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-8e3701c5-85e2-9dfe-8046-d662bc20b8b9.dc1 area=wan -19:04.273 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16134 network=tcp -19:04.286 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16135 network=tcp -19:04.286 [INFO] TestAgent: started state syncer -19:04.287 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16140 network=tcp -19:04.332 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -19:04.332 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16139 [Candidate]" term=2 -19:04.332 [DEBUG] TestAgent.server.raft: votes: needed=1 -19:04.332 [DEBUG] TestAgent.server.raft: vote granted: from=8e3701c5-85e2-9dfe-8046-d662bc20b8b9 term=2 tally=1 -19:04.332 [INFO] TestAgent.server.raft: election won: tally=1 -19:04.332 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16139 [Leader]" -19:04.332 [INFO] TestAgent.server: cluster leadership acquired -19:04.332 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16139 -19:04.333 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -19:04.333 [INFO] TestAgent.server: New leader elected: payload=Node-8e3701c5-85e2-9dfe-8046-d662bc20b8b9 -19:04.354 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -19:04.354 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -19:04.354 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -19:04.354 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -19:04.354 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -19:04.354 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-8e3701c5-85e2-9dfe-8046-d662bc20b8b9 -19:04.354 [INFO] TestAgent.server: member joined, marking health alive: member=Node-8e3701c5-85e2-9dfe-8046-d662bc20b8b9 -19:04.355 [INFO] TestAgent.server: federation state anti-entropy synced -19:04.386 [INFO] TestAgent.server.fsm: snapshot created: duration=83.374µs -19:04.386 [INFO] TestAgent.server.raft: starting snapshot up to: index=11 -19:04.386 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotSaveCommand-TestAgent-agent157447745/raft/snapshots/2-11-1635722344386.tmp -19:04.395 [INFO] TestAgent.server.raft: snapshot complete up to: index=11 -19:04.406 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:51254 latency=20.40462ms -19:04.426 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotSaveCommand-TestAgent-agent157447745/raft/snapshots/2-12-1635722344426.tmp -19:04.427 [INFO] TestAgent.server.raft: copied to local snapshot: bytes=4876 -19:04.428 [INFO] TestAgent.server.raft: restored user snapshot: index=1 -19:04.428 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -19:04.428 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -19:04.428 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -19:04.428 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -19:04.428 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -19:04.428 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -19:04.428 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -19:04.428 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -19:04.428 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -19:04.428 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16139 -19:04.428 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -19:04.428 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -19:04.429 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -19:04.429 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -19:04.429 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:51256 latency=2.996423ms -19:04.434 [INFO] TestAgent: Requesting shutdown -19:04.434 [INFO] TestAgent.server: shutting down server -19:04.434 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -19:04.434 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -19:04.434 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -19:04.434 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -19:04.434 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -19:04.434 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -19:04.434 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -19:04.434 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -19:04.434 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -19:04.434 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -19:04.434 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -19:04.434 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:19:04.435Z [INFO] agent.router.manager: shutting down -2021-10-31T23:19:04.435Z [INFO] agent.router.manager: shutting down -19:04.435 [INFO] TestAgent: consul server down -19:04.435 [INFO] TestAgent: shutdown complete -19:04.435 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=tcp -19:04.435 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16134 network=udp -19:04.435 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16135 network=tcp -19:04.936 [INFO] TestAgent: Waiting for endpoints to shut down -19:04.936 [INFO] TestAgent: Endpoints down ---- PASS: TestSnapshotSaveCommand (0.77s) -2021-10-31T23:19:04.251Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:19:04.293Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -19:04.299 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:cde76d54-95fb-ba00-fc02-afbb74116e61 Address:127.0.0.1:16132}]" -19:04.300 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-cde76d54-95fb-ba00-fc02-afbb74116e61.dc1 127.0.0.1 -19:04.300 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-cde76d54-95fb-ba00-fc02-afbb74116e61 127.0.0.1 -2021-10-31T23:19:04.300Z [INFO] agent.router: Initializing LAN area manager -19:04.300 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=udp -19:04.300 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:16132 [Follower]" leader= -19:04.300 [INFO] TestAgent: Started DNS server: address=127.0.0.1:16127 network=tcp -19:04.301 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-cde76d54-95fb-ba00-fc02-afbb74116e61.dc1 area=wan -19:04.301 [INFO] TestAgent.server: Adding LAN server: server="Node-cde76d54-95fb-ba00-fc02-afbb74116e61 (Addr: tcp/127.0.0.1:16132) (DC: dc1)" -19:04.301 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:16128 network=tcp -19:04.301 [INFO] TestAgent: started state syncer -19:04.301 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:16133 network=tcp -19:04.360 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -19:04.360 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:16132 [Candidate]" term=2 -19:04.360 [DEBUG] TestAgent.server.raft: votes: needed=1 -19:04.360 [DEBUG] TestAgent.server.raft: vote granted: from=cde76d54-95fb-ba00-fc02-afbb74116e61 term=2 tally=1 -19:04.360 [INFO] TestAgent.server.raft: election won: tally=1 -19:04.360 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:16132 [Leader]" -19:04.360 [INFO] TestAgent.server: cluster leadership acquired -19:04.360 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:16132 -19:04.361 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -19:04.361 [INFO] TestAgent.server: New leader elected: payload=Node-cde76d54-95fb-ba00-fc02-afbb74116e61 -19:04.362 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -19:04.362 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -19:04.362 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -19:04.362 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -19:04.362 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -19:04.363 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-cde76d54-95fb-ba00-fc02-afbb74116e61 -19:04.363 [INFO] TestAgent.server: member joined, marking health alive: member=Node-cde76d54-95fb-ba00-fc02-afbb74116e61 -19:04.363 [INFO] TestAgent.server: federation state anti-entropy synced -19:04.504 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/blob from=127.0.0.1:43980 latency=1.959374ms -19:04.507 [INFO] TestAgent.server.fsm: snapshot created: duration=94.023µs -19:04.507 [INFO] TestAgent.server.raft: starting snapshot up to: index=12 -19:04.507 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotSaveCommand_TruncatedStream-TestAgent-agent773941226/raft/snapshots/2-12-1635722344507.tmp -19:04.508 [INFO] TestAgent.server.raft: snapshot complete up to: index=12 -19:04.514 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -19:04.514 [INFO] TestAgent: Synced node info -19:04.527 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:43980 latency=20.257604ms -19:04.613 [INFO] TestAgent: Requesting shutdown -19:04.613 [INFO] TestAgent.server: shutting down server -19:04.613 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -19:04.613 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -19:04.613 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -19:04.613 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -19:04.613 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -19:04.613 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -19:04.613 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -19:04.613 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -19:04.613 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -19:04.613 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -19:04.613 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:19:04.614Z [INFO] agent.router.manager: shutting down -2021-10-31T23:19:04.614Z [INFO] agent.router.manager: shutting down -19:04.614 [INFO] TestAgent: consul server down -19:04.614 [INFO] TestAgent: shutdown complete -19:04.614 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=tcp -19:04.614 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:16127 network=udp -19:04.614 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:16128 network=tcp -19:05.117 [INFO] TestAgent: Waiting for endpoints to shut down -19:05.117 [INFO] TestAgent: Endpoints down ---- PASS: TestSnapshotSaveCommand_TruncatedStream (0.97s) +2022-12-04T05:51:14.203Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:14.218Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:14.220 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:3892d831-0cb7-bb29-32af-2ba42255ab56 Address:127.0.0.1:25321}]" +51:14.220 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25321 [Follower]" leader= +51:14.220 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-3892d831-0cb7-bb29-32af-2ba42255ab56.dc1 127.0.0.1 +51:14.224 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-3892d831-0cb7-bb29-32af-2ba42255ab56 127.0.0.1 +2022-12-04T05:51:14.224Z [INFO] agent.router: Initializing LAN area manager +51:14.225 [INFO] TestAgent.server: Adding LAN server: server="Node-3892d831-0cb7-bb29-32af-2ba42255ab56 (Addr: tcp/127.0.0.1:25321) (DC: dc1)" +51:14.225 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-3892d831-0cb7-bb29-32af-2ba42255ab56.dc1 area=wan +51:14.226 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=udp +51:14.227 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25316 network=tcp +51:14.228 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25317 network=tcp +51:14.228 [INFO] TestAgent: started state syncer +51:14.228 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25322 network=tcp +51:14.281 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:14.281 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25321 [Candidate]" term=2 +51:14.281 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:14.281 [DEBUG] TestAgent.server.raft: vote granted: from=3892d831-0cb7-bb29-32af-2ba42255ab56 term=2 tally=1 +51:14.281 [INFO] TestAgent.server.raft: election won: tally=1 +51:14.281 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25321 [Leader]" +51:14.281 [INFO] TestAgent.server: cluster leadership acquired +51:14.281 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25321 +51:14.281 [INFO] TestAgent.server: New leader elected: payload=Node-3892d831-0cb7-bb29-32af-2ba42255ab56 +51:14.282 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:14.291 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:14.291 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:14.291 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:14.291 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:14.291 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:14.291 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-3892d831-0cb7-bb29-32af-2ba42255ab56 +51:14.291 [INFO] TestAgent.server: member joined, marking health alive: member=Node-3892d831-0cb7-bb29-32af-2ba42255ab56 +51:14.292 [INFO] TestAgent.server: federation state anti-entropy synced +51:14.343 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +51:14.343 [INFO] TestAgent: Synced node info +51:14.343 [DEBUG] TestAgent: Node info in sync +51:14.435 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/kv/blob from=127.0.0.1:33964 latency=1.102175ms +51:14.435 [INFO] TestAgent.server.fsm: snapshot created: duration=68.97µs +51:14.435 [INFO] TestAgent.server.raft: starting snapshot up to: index=13 +51:14.435 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotSaveCommand_TruncatedStream-TestAgent-agent008360975/raft/snapshots/2-13-1670133074435.tmp +51:14.436 [INFO] TestAgent.server.raft: snapshot complete up to: index=13 +51:14.447 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:33964 latency=12.132697ms +51:14.465 [INFO] TestAgent: Requesting shutdown +51:14.465 [INFO] TestAgent.server: shutting down server +51:14.465 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:14.465 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:14.465 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:14.465 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:14.465 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:14.465 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:14.465 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:14.465 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:14.465 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:14.465 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:14.465 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:14.465Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:14.465Z [INFO] agent.router.manager: shutting down +51:14.465 [INFO] TestAgent: consul server down +51:14.465 [INFO] TestAgent: shutdown complete +51:14.465 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=tcp +51:14.465 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25316 network=udp +51:14.465 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25317 network=tcp +51:14.966 [INFO] TestAgent: Waiting for endpoints to shut down +51:14.966 [INFO] TestAgent: Endpoints down +--- PASS: TestSnapshotSaveCommand_TruncatedStream (0.80s) --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_200_bytes_from_end (0.00s) --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_16_bytes_from_end (0.00s) - --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_8_bytes_from_end (0.01s) - --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_4_bytes_from_end (0.02s) - --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_2_bytes_from_end (0.01s) - --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_1_bytes_from_end (0.01s) + --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_8_bytes_from_end (0.00s) + --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_4_bytes_from_end (0.00s) + --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_2_bytes_from_end (0.00s) + --- PASS: TestSnapshotSaveCommand_TruncatedStream/truncate_1_bytes_from_end (0.00s) +2022-12-04T05:51:14.202Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:14.219Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:14.221 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:fa22bc5a-a013-6cb8-0fef-5883e02091e4 Address:127.0.0.1:25328}]" +51:14.221 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:25328 [Follower]" leader= +51:14.224 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-fa22bc5a-a013-6cb8-0fef-5883e02091e4.dc1 127.0.0.1 +51:14.225 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-fa22bc5a-a013-6cb8-0fef-5883e02091e4 127.0.0.1 +2022-12-04T05:51:14.225Z [INFO] agent.router: Initializing LAN area manager +51:14.225 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-fa22bc5a-a013-6cb8-0fef-5883e02091e4.dc1 area=wan +51:14.227 [INFO] TestAgent.server: Adding LAN server: server="Node-fa22bc5a-a013-6cb8-0fef-5883e02091e4 (Addr: tcp/127.0.0.1:25328) (DC: dc1)" +51:14.227 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=udp +51:14.227 [INFO] TestAgent: Started DNS server: address=127.0.0.1:25323 network=tcp +51:14.228 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:25324 network=tcp +51:14.228 [INFO] TestAgent: started state syncer +51:14.228 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:25329 network=tcp +51:14.273 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:14.273 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:25328 [Candidate]" term=2 +51:14.274 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:14.274 [DEBUG] TestAgent.server.raft: vote granted: from=fa22bc5a-a013-6cb8-0fef-5883e02091e4 term=2 tally=1 +51:14.274 [INFO] TestAgent.server.raft: election won: tally=1 +51:14.274 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:25328 [Leader]" +51:14.274 [INFO] TestAgent.server: cluster leadership acquired +51:14.274 [INFO] TestAgent.server: New leader elected: payload=Node-fa22bc5a-a013-6cb8-0fef-5883e02091e4 +51:14.274 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25328 +51:14.275 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:14.290 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:14.291 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:14.291 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:14.291 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:14.291 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:14.291 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-fa22bc5a-a013-6cb8-0fef-5883e02091e4 +51:14.291 [INFO] TestAgent.server: member joined, marking health alive: member=Node-fa22bc5a-a013-6cb8-0fef-5883e02091e4 +51:14.292 [INFO] TestAgent.server: federation state anti-entropy synced +51:14.539 [INFO] TestAgent.server.fsm: snapshot created: duration=50.684µs +51:14.539 [INFO] TestAgent.server.raft: starting snapshot up to: index=11 +51:14.539 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotSaveCommand-TestAgent-agent206277666/raft/snapshots/2-11-1670133074539.tmp +51:14.540 [INFO] TestAgent.server.raft: snapshot complete up to: index=11 +51:14.540 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/snapshot from=127.0.0.1:49036 latency=1.70981ms +51:14.543 [INFO] TestAgent.server.snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshotSaveCommand-TestAgent-agent206277666/raft/snapshots/2-12-1670133074543.tmp +51:14.543 [INFO] TestAgent.server.raft: copied to local snapshot: bytes=4876 +51:14.544 [INFO] TestAgent.server.raft: restored user snapshot: index=1 +51:14.545 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:14.545 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:14.545 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:14.545 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:14.545 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:14.545 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:14.545 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:14.545 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:25328 +51:14.545 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:14.545 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:14.545 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:14.545 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:14.545 [DEBUG] TestAgent.http: Request finished: method=PUT url=/v1/snapshot from=127.0.0.1:49038 latency=2.685183ms +51:14.546 [INFO] TestAgent: Requesting shutdown +51:14.546 [INFO] TestAgent.server: shutting down server +51:14.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:14.546 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:14.546 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:14.546 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:14.546 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:14.546 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:14.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:14.546 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:14.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:14.546 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:14.546 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +51:14.546 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +51:14.546 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:14.546 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:51:14.546Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:14.546Z [INFO] agent.router.manager: shutting down +51:14.546 [INFO] TestAgent: consul server down +51:14.546 [INFO] TestAgent: shutdown complete +51:14.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=tcp +51:14.546 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:25323 network=udp +51:14.546 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:25324 network=tcp +51:15.046 [INFO] TestAgent: Waiting for endpoints to shut down +51:15.046 [INFO] TestAgent: Endpoints down +--- PASS: TestSnapshotSaveCommand (0.88s) PASS -ok github.com/hashicorp/consul/command/snapshot/save 1.054s +ok github.com/hashicorp/consul/command/snapshot/save 0.924s === RUN TestValidateCommand_noTabs === PAUSE TestValidateCommand_noTabs === RUN TestValidateCommand_FailOnEmptyFile @@ -91374,25 +91537,25 @@ === RUN TestValidateCommand_Quiet === PAUSE TestValidateCommand_Quiet === CONT TestValidateCommand_noTabs ---- PASS: TestValidateCommand_noTabs (0.00s) +=== CONT TestValidateCommand_SucceedWithJSONAsHCL === CONT TestValidateCommand_Quiet -=== CONT TestValidateCommand_SucceedWithMinimalHCLConfigFormat +=== CONT TestValidateCommand_SucceedOnMinimalConfigFile === CONT TestValidateCommand_SucceedOnMinimalConfigDir +--- PASS: TestValidateCommand_noTabs (0.00s) +=== CONT TestValidateCommand_SucceedWithMinimalHCLConfigFormat +=== CONT TestValidateCommand_FailOnEmptyFile === CONT TestValidateCommand_FailForInvalidJSONConfigFormat -=== CONT TestValidateCommand_SucceedWithJSONAsHCL -=== CONT TestValidateCommand_SucceedOnMinimalConfigFile === CONT TestValidateCommand_SucceedWithMinimalJSONConfigFormat -=== CONT TestValidateCommand_FailOnEmptyFile ---- PASS: TestValidateCommand_FailOnEmptyFile (0.04s) ---- PASS: TestValidateCommand_Quiet (0.05s) ---- PASS: TestValidateCommand_SucceedWithMinimalHCLConfigFormat (0.06s) ---- PASS: TestValidateCommand_FailForInvalidJSONConfigFormat (0.06s) ---- PASS: TestValidateCommand_SucceedWithJSONAsHCL (0.07s) ---- PASS: TestValidateCommand_SucceedOnMinimalConfigDir (0.08s) ---- PASS: TestValidateCommand_SucceedWithMinimalJSONConfigFormat (0.08s) ---- PASS: TestValidateCommand_SucceedOnMinimalConfigFile (0.09s) +--- PASS: TestValidateCommand_FailForInvalidJSONConfigFormat (0.01s) +--- PASS: TestValidateCommand_FailOnEmptyFile (0.03s) +--- PASS: TestValidateCommand_SucceedOnMinimalConfigDir (0.03s) +--- PASS: TestValidateCommand_SucceedOnMinimalConfigFile (0.03s) +--- PASS: TestValidateCommand_SucceedWithJSONAsHCL (0.03s) +--- PASS: TestValidateCommand_SucceedWithMinimalJSONConfigFormat (0.03s) +--- PASS: TestValidateCommand_SucceedWithMinimalHCLConfigFormat (0.03s) +--- PASS: TestValidateCommand_Quiet (0.04s) PASS -ok github.com/hashicorp/consul/command/validate 0.157s +ok github.com/hashicorp/consul/command/validate 0.067s === RUN TestFormat === RUN TestFormat/pretty === RUN TestFormat/json @@ -91404,7 +91567,7 @@ === CONT TestVersionCommand_noTabs --- PASS: TestVersionCommand_noTabs (0.00s) PASS -ok github.com/hashicorp/consul/command/version 0.043s +ok github.com/hashicorp/consul/command/version 0.029s === RUN TestWatchCommand_noTabs === PAUSE TestWatchCommand_noTabs === RUN TestWatchCommand @@ -91417,62 +91580,63 @@ === RUN TestWatchCommand_loadToken/token_env === RUN TestWatchCommand_loadToken/token_file_arg === RUN TestWatchCommand_loadToken/token_file_env -2021-10-31T23:19:06.930Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:19:06.943Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -19:06.947 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:bcfa31b8-49e6-f9e5-09b5-b31a55f66cf1 Address:127.0.0.1:17153}]" -19:06.947 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17153 [Follower]" leader= -19:06.948 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-bcfa31b8-49e6-f9e5-09b5-b31a55f66cf1.dc1 127.0.0.1 -19:06.957 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-bcfa31b8-49e6-f9e5-09b5-b31a55f66cf1 127.0.0.1 -2021-10-31T23:19:06.957Z [INFO] agent.router: Initializing LAN area manager -19:06.957 [INFO] TestAgent.server: Adding LAN server: server="Node-bcfa31b8-49e6-f9e5-09b5-b31a55f66cf1 (Addr: tcp/127.0.0.1:17153) (DC: dc1)" -19:06.958 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17148 network=udp -19:06.958 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17148 network=tcp -19:06.959 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17149 network=tcp -19:06.959 [INFO] TestAgent: started state syncer -19:06.959 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17154 network=tcp -19:06.963 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-bcfa31b8-49e6-f9e5-09b5-b31a55f66cf1.dc1 area=wan -19:06.994 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -19:06.994 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17153 [Candidate]" term=2 -19:06.994 [DEBUG] TestAgent.server.raft: votes: needed=1 -19:06.994 [DEBUG] TestAgent.server.raft: vote granted: from=bcfa31b8-49e6-f9e5-09b5-b31a55f66cf1 term=2 tally=1 -19:06.994 [INFO] TestAgent.server.raft: election won: tally=1 -19:06.994 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17153 [Leader]" -19:06.995 [INFO] TestAgent.server: cluster leadership acquired -19:06.995 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17153 -19:06.995 [INFO] TestAgent.server: New leader elected: payload=Node-bcfa31b8-49e6-f9e5-09b5-b31a55f66cf1 -19:06.996 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -19:07.013 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -19:07.013 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -19:07.013 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -19:07.013 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -19:07.013 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -19:07.013 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-bcfa31b8-49e6-f9e5-09b5-b31a55f66cf1 -19:07.013 [INFO] TestAgent.server: member joined, marking health alive: member=Node-bcfa31b8-49e6-f9e5-09b5-b31a55f66cf1 -19:07.014 [INFO] TestAgent.server: federation state anti-entropy synced -19:07.153 [INFO] TestAgent: Requesting shutdown -19:07.153 [INFO] TestAgent.server: shutting down server -19:07.153 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -19:07.153 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -19:07.153 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -19:07.153 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -19:07.153 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -19:07.153 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -19:07.153 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -19:07.153 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -19:07.153 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -19:07.153 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -19:07.153 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -19:07.153 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:19:07.153Z [INFO] agent.router.manager: shutting down -2021-10-31T23:19:07.153Z [INFO] agent.router.manager: shutting down -19:07.153 [INFO] TestAgent: consul server down -19:07.153 [INFO] TestAgent: shutdown complete -19:07.153 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17148 network=tcp -19:07.153 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17148 network=udp -19:07.153 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17149 network=tcp -19:07.654 [INFO] TestAgent: Waiting for endpoints to shut down -19:07.654 [INFO] TestAgent: Endpoints down ---- PASS: TestWatchCommand_loadToken (0.77s) +2022-12-04T05:51:18.009Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:18.025Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:18.027 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:1882baf5-c8b9-3054-50d6-353a858252e1 Address:127.0.0.1:31447}]" +51:18.027 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31447 [Follower]" leader= +51:18.028 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-1882baf5-c8b9-3054-50d6-353a858252e1.dc1 127.0.0.1 +51:18.028 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-1882baf5-c8b9-3054-50d6-353a858252e1 127.0.0.1 +2022-12-04T05:51:18.028Z [INFO] agent.router: Initializing LAN area manager +51:18.029 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=udp +51:18.029 [INFO] TestAgent.server: Adding LAN server: server="Node-1882baf5-c8b9-3054-50d6-353a858252e1 (Addr: tcp/127.0.0.1:31447) (DC: dc1)" +51:18.029 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-1882baf5-c8b9-3054-50d6-353a858252e1.dc1 area=wan +51:18.029 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31442 network=tcp +51:18.029 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31443 network=tcp +51:18.029 [INFO] TestAgent: started state syncer +51:18.029 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31448 network=tcp +51:18.066 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:18.066 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31447 [Candidate]" term=2 +51:18.066 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:18.066 [DEBUG] TestAgent.server.raft: vote granted: from=1882baf5-c8b9-3054-50d6-353a858252e1 term=2 tally=1 +51:18.066 [INFO] TestAgent.server.raft: election won: tally=1 +51:18.066 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31447 [Leader]" +51:18.066 [INFO] TestAgent.server: cluster leadership acquired +51:18.066 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31447 +51:18.067 [INFO] TestAgent.server: New leader elected: payload=Node-1882baf5-c8b9-3054-50d6-353a858252e1 +51:18.067 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:18.085 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:18.085 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:18.085 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:18.085 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:18.085 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:18.085 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-1882baf5-c8b9-3054-50d6-353a858252e1 +51:18.085 [INFO] TestAgent.server: member joined, marking health alive: member=Node-1882baf5-c8b9-3054-50d6-353a858252e1 +51:18.085 [INFO] TestAgent.server: federation state anti-entropy synced +51:18.166 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth +51:18.166 [INFO] TestAgent: Synced node info +51:18.464 [INFO] TestAgent: Requesting shutdown +51:18.464 [INFO] TestAgent.server: shutting down server +51:18.464 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:18.464 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:18.464 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:18.464 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:18.464 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:18.464 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:18.464 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:18.464 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:18.464 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:18.464 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:18.464 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:18.464Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:18.464Z [INFO] agent.router.manager: shutting down +51:18.464 [INFO] TestAgent: consul server down +51:18.464 [INFO] TestAgent: shutdown complete +51:18.464 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=tcp +51:18.464 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31442 network=udp +51:18.464 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31443 network=tcp +51:18.965 [INFO] TestAgent: Waiting for endpoints to shut down +51:18.965 [INFO] TestAgent: Endpoints down +--- PASS: TestWatchCommand_loadToken (0.99s) --- PASS: TestWatchCommand_loadToken/token_arg (0.00s) --- PASS: TestWatchCommand_loadToken/token_env (0.00s) --- PASS: TestWatchCommand_loadToken/token_file_arg (0.00s) @@ -91483,182 +91647,182 @@ === PAUSE TestWatchCommandNoAgentService === CONT TestWatchCommand_noTabs --- PASS: TestWatchCommand_noTabs (0.00s) -=== CONT TestWatchCommandNoAgentService === CONT TestWatchCommandNoConnect === CONT TestWatchCommand -2021-10-31T23:19:07.684Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:19:07.707Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -19:07.720 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:f984729b-c8bb-48fb-81ab-35e26fd77c1a Address:127.0.0.1:17167}]" -19:07.720 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17167 [Follower]" leader= -19:07.722 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-f984729b-c8bb-48fb-81ab-35e26fd77c1a.dc1 127.0.0.1 -19:07.723 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-f984729b-c8bb-48fb-81ab-35e26fd77c1a 127.0.0.1 -2021-10-31T23:19:07.723Z [INFO] agent.router: Initializing LAN area manager -19:07.723 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17162 network=udp -19:07.723 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-f984729b-c8bb-48fb-81ab-35e26fd77c1a.dc1 area=wan -19:07.723 [INFO] TestAgent.server: Adding LAN server: server="Node-f984729b-c8bb-48fb-81ab-35e26fd77c1a (Addr: tcp/127.0.0.1:17167) (DC: dc1)" -19:07.723 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17162 network=tcp -19:07.723 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17163 network=tcp -19:07.724 [INFO] TestAgent: started state syncer -19:07.724 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17168 network=tcp -19:07.773 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -19:07.773 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17167 [Candidate]" term=2 -19:07.774 [DEBUG] TestAgent.server.raft: votes: needed=1 -19:07.774 [DEBUG] TestAgent.server.raft: vote granted: from=f984729b-c8bb-48fb-81ab-35e26fd77c1a term=2 tally=1 -19:07.774 [INFO] TestAgent.server.raft: election won: tally=1 -19:07.774 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17167 [Leader]" -19:07.774 [INFO] TestAgent.server: cluster leadership acquired -19:07.774 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17167 -19:07.774 [INFO] TestAgent.server: New leader elected: payload=Node-f984729b-c8bb-48fb-81ab-35e26fd77c1a -19:07.775 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -19:07.776 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -19:07.776 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -19:07.776 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -19:07.776 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -19:07.776 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -19:07.776 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-f984729b-c8bb-48fb-81ab-35e26fd77c1a -19:07.776 [INFO] TestAgent.server: member joined, marking health alive: member=Node-f984729b-c8bb-48fb-81ab-35e26fd77c1a -19:07.776 [INFO] TestAgent.server: federation state anti-entropy synced -19:07.782 [INFO] TestAgent: Requesting shutdown -19:07.782 [INFO] TestAgent.server: shutting down server -19:07.782 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -19:07.782 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -19:07.782 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -19:07.782 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -19:07.782 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -19:07.782 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -19:07.782 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -19:07.782 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -19:07.782 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -19:07.782 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -19:07.782 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -19:07.782 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:19:07.782Z [INFO] agent.router.manager: shutting down -2021-10-31T23:19:07.782Z [INFO] agent.router.manager: shutting down -19:07.783 [INFO] TestAgent: consul server down -19:07.783 [INFO] TestAgent: shutdown complete -19:07.783 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17162 network=tcp -19:07.783 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17162 network=udp -19:07.783 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17163 network=tcp -19:08.283 [INFO] TestAgent: Waiting for endpoints to shut down -19:08.283 [INFO] TestAgent: Endpoints down ---- PASS: TestWatchCommandNoConnect (0.62s) -2021-10-31T23:19:07.686Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:19:07.708Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -19:07.710 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:e547799c-127e-e0ec-4889-a7de49fbd113 Address:127.0.0.1:17174}]" -19:07.710 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-e547799c-127e-e0ec-4889-a7de49fbd113.dc1 127.0.0.1 -19:07.711 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-e547799c-127e-e0ec-4889-a7de49fbd113 127.0.0.1 -2021-10-31T23:19:07.711Z [INFO] agent.router: Initializing LAN area manager -19:07.711 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17169 network=udp -19:07.711 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17174 [Follower]" leader= -19:07.711 [INFO] TestAgent.server: Adding LAN server: server="Node-e547799c-127e-e0ec-4889-a7de49fbd113 (Addr: tcp/127.0.0.1:17174) (DC: dc1)" -19:07.711 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-e547799c-127e-e0ec-4889-a7de49fbd113.dc1 area=wan -19:07.712 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17169 network=tcp -19:07.712 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17170 network=tcp -19:07.712 [INFO] TestAgent: started state syncer -19:07.712 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17175 network=tcp -19:07.779 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -19:07.779 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17174 [Candidate]" term=2 -19:07.779 [DEBUG] TestAgent.server.raft: votes: needed=1 -19:07.779 [DEBUG] TestAgent.server.raft: vote granted: from=e547799c-127e-e0ec-4889-a7de49fbd113 term=2 tally=1 -19:07.779 [INFO] TestAgent.server.raft: election won: tally=1 -19:07.779 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17174 [Leader]" -19:07.779 [INFO] TestAgent.server: cluster leadership acquired -19:07.779 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17174 -19:07.780 [INFO] TestAgent.server: New leader elected: payload=Node-e547799c-127e-e0ec-4889-a7de49fbd113 -19:07.780 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -19:07.782 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -19:07.782 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -19:07.782 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -19:07.782 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -19:07.782 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -19:07.782 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-e547799c-127e-e0ec-4889-a7de49fbd113 -19:07.782 [INFO] TestAgent.server: member joined, marking health alive: member=Node-e547799c-127e-e0ec-4889-a7de49fbd113 -19:07.782 [INFO] TestAgent.server: federation state anti-entropy synced -19:07.908 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:54250 latency=2.353906ms -19:07.912 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:54252 latency=464.992µs -19:07.913 [INFO] TestAgent: Requesting shutdown -19:07.913 [INFO] TestAgent.server: shutting down server -19:07.913 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -19:07.913 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -19:07.913 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -19:07.913 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -19:07.913 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -19:07.913 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -19:07.913 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -19:07.913 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" -19:07.913 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -19:07.913 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -19:07.913 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -19:07.913 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:19:07.913Z [INFO] agent.router.manager: shutting down -2021-10-31T23:19:07.913Z [INFO] agent.router.manager: shutting down -19:07.913 [INFO] TestAgent: consul server down -19:07.913 [INFO] TestAgent: shutdown complete -19:07.913 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17169 network=tcp -19:07.913 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17169 network=udp -19:07.913 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17170 network=tcp -19:08.413 [INFO] TestAgent: Waiting for endpoints to shut down -19:08.413 [INFO] TestAgent: Endpoints down ---- PASS: TestWatchCommand (0.75s) -2021-10-31T23:19:07.682Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:19:07.712Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -19:07.718 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:99298d56-3652-6930-63a6-757139b902a9 Address:127.0.0.1:17160}]" -19:07.719 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-99298d56-3652-6930-63a6-757139b902a9.dc1 127.0.0.1 -19:07.720 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-99298d56-3652-6930-63a6-757139b902a9 127.0.0.1 -2021-10-31T23:19:07.720Z [INFO] agent.router: Initializing LAN area manager -19:07.720 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:17160 [Follower]" leader= -19:07.720 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-99298d56-3652-6930-63a6-757139b902a9.dc1 area=wan -19:07.720 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17155 network=udp -19:07.720 [INFO] TestAgent: Started DNS server: address=127.0.0.1:17155 network=tcp -19:07.721 [INFO] TestAgent.server: Adding LAN server: server="Node-99298d56-3652-6930-63a6-757139b902a9 (Addr: tcp/127.0.0.1:17160) (DC: dc1)" -19:07.722 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:17156 network=tcp -19:07.722 [INFO] TestAgent: started state syncer -19:07.722 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:17161 network=tcp -19:07.761 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= -19:07.761 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:17160 [Candidate]" term=2 -19:07.761 [DEBUG] TestAgent.server.raft: votes: needed=1 -19:07.762 [DEBUG] TestAgent.server.raft: vote granted: from=99298d56-3652-6930-63a6-757139b902a9 term=2 tally=1 -19:07.762 [INFO] TestAgent.server.raft: election won: tally=1 -19:07.762 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:17160 [Leader]" -19:07.762 [INFO] TestAgent.server: cluster leadership acquired -19:07.762 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:17160 -19:07.762 [INFO] TestAgent.server: New leader elected: payload=Node-99298d56-3652-6930-63a6-757139b902a9 -19:07.762 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -19:07.764 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul -19:07.764 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" -19:07.764 [INFO] TestAgent.leader: started routine: routine="federation state pruning" -19:07.764 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" -19:07.764 [INFO] TestAgent.leader: started routine: routine="CA root pruning" -19:07.764 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-99298d56-3652-6930-63a6-757139b902a9 -19:07.764 [INFO] TestAgent.server: member joined, marking health alive: member=Node-99298d56-3652-6930-63a6-757139b902a9 -19:07.764 [INFO] TestAgent.server: federation state anti-entropy synced -19:07.870 [DEBUG] TestAgent: Skipping remote check since it is managed automatically: check=serfHealth -19:07.870 [INFO] TestAgent: Synced node info -19:08.048 [INFO] TestAgent: Requesting shutdown -19:08.048 [INFO] TestAgent.server: shutting down server -19:08.048 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" -19:08.048 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" -19:08.048 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" -19:08.048 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" -19:08.048 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave -19:08.048 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" -19:08.048 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" -19:08.048 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" -19:08.048 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" -19:08.048 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" -19:08.049 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:19:08.049Z [INFO] agent.router.manager: shutting down -2021-10-31T23:19:08.049Z [INFO] agent.router.manager: shutting down -19:08.049 [INFO] TestAgent: consul server down -19:08.049 [INFO] TestAgent: shutdown complete -19:08.049 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17155 network=tcp -19:08.049 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:17155 network=udp -19:08.049 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:17156 network=tcp -19:08.549 [INFO] TestAgent: Waiting for endpoints to shut down -19:08.549 [INFO] TestAgent: Endpoints down ---- PASS: TestWatchCommandNoAgentService (0.89s) +=== CONT TestWatchCommandNoAgentService +2022-12-04T05:51:19.021Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:19.042Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:19.044 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:aea1226c-2f8e-3f9e-2325-af6866b1e11a Address:127.0.0.1:31461}]" +51:19.044 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31461 [Follower]" leader= +51:19.044 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-aea1226c-2f8e-3f9e-2325-af6866b1e11a.dc1 127.0.0.1 +51:19.045 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-aea1226c-2f8e-3f9e-2325-af6866b1e11a 127.0.0.1 +2022-12-04T05:51:19.045Z [INFO] agent.router: Initializing LAN area manager +51:19.045 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31456 network=udp +51:19.045 [INFO] TestAgent.server: Adding LAN server: server="Node-aea1226c-2f8e-3f9e-2325-af6866b1e11a (Addr: tcp/127.0.0.1:31461) (DC: dc1)" +51:19.045 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-aea1226c-2f8e-3f9e-2325-af6866b1e11a.dc1 area=wan +51:19.045 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31456 network=tcp +51:19.046 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31457 network=tcp +51:19.046 [INFO] TestAgent: started state syncer +51:19.046 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31462 network=tcp +51:19.108 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:19.108 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31461 [Candidate]" term=2 +51:19.108 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:19.108 [DEBUG] TestAgent.server.raft: vote granted: from=aea1226c-2f8e-3f9e-2325-af6866b1e11a term=2 tally=1 +51:19.108 [INFO] TestAgent.server.raft: election won: tally=1 +51:19.108 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31461 [Leader]" +51:19.108 [INFO] TestAgent.server: cluster leadership acquired +51:19.108 [INFO] TestAgent.server: New leader elected: payload=Node-aea1226c-2f8e-3f9e-2325-af6866b1e11a +51:19.108 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31461 +51:19.109 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:19.110 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:19.110 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:19.110 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:19.110 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:19.110 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:19.110 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-aea1226c-2f8e-3f9e-2325-af6866b1e11a +51:19.110 [INFO] TestAgent.server: member joined, marking health alive: member=Node-aea1226c-2f8e-3f9e-2325-af6866b1e11a +51:19.111 [INFO] TestAgent.server: federation state anti-entropy synced +51:19.142 [INFO] TestAgent: Requesting shutdown +51:19.142 [INFO] TestAgent.server: shutting down server +51:19.142 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:19.142 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:19.142 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:19.142 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:19.142 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:19.142 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:19.142 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:19.142 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:19.142 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:19.142 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +51:19.142 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +51:19.142 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +2022-12-04T05:51:19.143Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:19.143Z [INFO] agent.router.manager: shutting down +51:19.143 [INFO] TestAgent: consul server down +51:19.143 [INFO] TestAgent: shutdown complete +51:19.143 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31456 network=tcp +51:19.143 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31456 network=udp +51:19.143 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31457 network=tcp +51:19.644 [INFO] TestAgent: Waiting for endpoints to shut down +51:19.644 [INFO] TestAgent: Endpoints down +--- PASS: TestWatchCommandNoAgentService (0.67s) +2022-12-04T05:51:18.992Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:19.025Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:19.026 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9700a355-4ec0-6283-9442-95df66c68199 Address:127.0.0.1:31454}]" +51:19.026 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31454 [Follower]" leader= +51:19.027 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-9700a355-4ec0-6283-9442-95df66c68199.dc1 127.0.0.1 +51:19.030 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-9700a355-4ec0-6283-9442-95df66c68199 127.0.0.1 +2022-12-04T05:51:19.030Z [INFO] agent.router: Initializing LAN area manager +51:19.032 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31449 network=udp +51:19.032 [INFO] TestAgent.server: Adding LAN server: server="Node-9700a355-4ec0-6283-9442-95df66c68199 (Addr: tcp/127.0.0.1:31454) (DC: dc1)" +51:19.032 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-9700a355-4ec0-6283-9442-95df66c68199.dc1 area=wan +51:19.032 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31449 network=tcp +51:19.041 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31450 network=tcp +51:19.041 [INFO] TestAgent: started state syncer +51:19.041 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31455 network=tcp +51:19.063 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:19.063 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31454 [Candidate]" term=2 +51:19.064 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:19.064 [DEBUG] TestAgent.server.raft: vote granted: from=9700a355-4ec0-6283-9442-95df66c68199 term=2 tally=1 +51:19.064 [INFO] TestAgent.server.raft: election won: tally=1 +51:19.064 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31454 [Leader]" +51:19.064 [INFO] TestAgent.server: cluster leadership acquired +51:19.064 [INFO] TestAgent.server: New leader elected: payload=Node-9700a355-4ec0-6283-9442-95df66c68199 +51:19.064 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31454 +51:19.065 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:19.066 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:19.066 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:19.066 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:19.066 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:19.066 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:19.066 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-9700a355-4ec0-6283-9442-95df66c68199 +51:19.066 [INFO] TestAgent.server: member joined, marking health alive: member=Node-9700a355-4ec0-6283-9442-95df66c68199 +51:19.067 [INFO] TestAgent.server: federation state anti-entropy synced +51:19.067 [INFO] TestAgent: Synced node info +51:19.067 [DEBUG] TestAgent: Node info in sync +51:19.146 [INFO] TestAgent: Requesting shutdown +51:19.146 [INFO] TestAgent.server: shutting down server +51:19.146 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:19.146 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:19.146 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:19.146 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:19.146 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:19.146 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:19.146 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:19.146 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:19.146 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:19.146 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:19.146 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:19.146Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:19.146Z [INFO] agent.router.manager: shutting down +51:19.146 [INFO] TestAgent: consul server down +51:19.146 [INFO] TestAgent: shutdown complete +51:19.146 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31449 network=tcp +51:19.146 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31449 network=udp +51:19.146 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31450 network=tcp +51:19.647 [INFO] TestAgent: Waiting for endpoints to shut down +51:19.647 [INFO] TestAgent: Endpoints down +--- PASS: TestWatchCommandNoConnect (0.68s) +2022-12-04T05:51:19.027Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:19.049Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:19.050 [INFO] TestAgent.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:5e7a22c6-71ed-3c85-333e-f348d2799515 Address:127.0.0.1:31468}]" +51:19.050 [INFO] TestAgent.server.raft: entering follower state: follower="Node at 127.0.0.1:31468 [Follower]" leader= +51:19.050 [INFO] TestAgent.server.serf.wan: serf: EventMemberJoin: Node-5e7a22c6-71ed-3c85-333e-f348d2799515.dc1 127.0.0.1 +51:19.051 [INFO] TestAgent.server.serf.lan: serf: EventMemberJoin: Node-5e7a22c6-71ed-3c85-333e-f348d2799515 127.0.0.1 +2022-12-04T05:51:19.051Z [INFO] agent.router: Initializing LAN area manager +51:19.051 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31463 network=udp +51:19.051 [INFO] TestAgent.server: Adding LAN server: server="Node-5e7a22c6-71ed-3c85-333e-f348d2799515 (Addr: tcp/127.0.0.1:31468) (DC: dc1)" +51:19.051 [INFO] TestAgent.server: Handled event for server in area: event=member-join server=Node-5e7a22c6-71ed-3c85-333e-f348d2799515.dc1 area=wan +51:19.051 [INFO] TestAgent: Started DNS server: address=127.0.0.1:31463 network=tcp +51:19.059 [INFO] TestAgent: Started HTTP server: address=127.0.0.1:31464 network=tcp +51:19.059 [INFO] TestAgent: started state syncer +51:19.059 [INFO] TestAgent: Started gRPC server: address=127.0.0.1:31469 network=tcp +51:19.087 [WARN] TestAgent.server.raft: heartbeat timeout reached, starting election: last-leader= +51:19.087 [INFO] TestAgent.server.raft: entering candidate state: node="Node at 127.0.0.1:31468 [Candidate]" term=2 +51:19.088 [DEBUG] TestAgent.server.raft: votes: needed=1 +51:19.088 [DEBUG] TestAgent.server.raft: vote granted: from=5e7a22c6-71ed-3c85-333e-f348d2799515 term=2 tally=1 +51:19.088 [INFO] TestAgent.server.raft: election won: tally=1 +51:19.088 [INFO] TestAgent.server.raft: entering leader state: leader="Node at 127.0.0.1:31468 [Leader]" +51:19.088 [INFO] TestAgent.server: cluster leadership acquired +51:19.088 [INFO] TestAgent.server: New leader elected: payload=Node-5e7a22c6-71ed-3c85-333e-f348d2799515 +51:19.088 [DEBUG] TestAgent.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:31468 +51:19.089 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:19.090 [INFO] TestAgent.server.connect: initialized primary datacenter CA with provider: provider=consul +51:19.090 [INFO] TestAgent.leader: started routine: routine="federation state anti-entropy" +51:19.090 [INFO] TestAgent.leader: started routine: routine="federation state pruning" +51:19.090 [INFO] TestAgent.leader: started routine: routine="intermediate cert renew watch" +51:19.090 [INFO] TestAgent.leader: started routine: routine="CA root pruning" +51:19.090 [DEBUG] TestAgent.server: Skipping self join check for node since the cluster is too small: node=Node-5e7a22c6-71ed-3c85-333e-f348d2799515 +51:19.090 [INFO] TestAgent.server: member joined, marking health alive: member=Node-5e7a22c6-71ed-3c85-333e-f348d2799515 +51:19.091 [INFO] TestAgent.server: federation state anti-entropy synced +51:19.281 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/agent/self from=127.0.0.1:57220 latency=1.217897ms +51:19.284 [DEBUG] TestAgent.http: Request finished: method=GET url=/v1/catalog/nodes from=127.0.0.1:57222 latency=269.967µs +51:19.284 [INFO] TestAgent: Requesting shutdown +51:19.284 [INFO] TestAgent.server: shutting down server +51:19.284 [DEBUG] TestAgent.leader: stopping routine: routine="federation state anti-entropy" +51:19.284 [DEBUG] TestAgent.leader: stopping routine: routine="federation state pruning" +51:19.284 [DEBUG] TestAgent.leader: stopping routine: routine="intermediate cert renew watch" +51:19.284 [DEBUG] TestAgent.leader: stopping routine: routine="CA root pruning" +51:19.284 [WARN] TestAgent.server.serf.lan: serf: Shutdown without a Leave +51:19.284 [ERROR] TestAgent.server: error performing anti-entropy sync of federation state: error="context canceled" +51:19.284 [DEBUG] TestAgent.leader: stopped routine: routine="federation state anti-entropy" +51:19.284 [ERROR] TestAgent.anti_entropy: failed to sync remote state: error="No cluster leader" +51:19.284 [DEBUG] TestAgent.leader: stopped routine: routine="federation state pruning" +51:19.284 [DEBUG] TestAgent.leader: stopped routine: routine="intermediate cert renew watch" +51:19.284 [DEBUG] TestAgent.leader: stopped routine: routine="CA root pruning" +51:19.284 [WARN] TestAgent.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:19.284Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:19.284Z [INFO] agent.router.manager: shutting down +51:19.284 [INFO] TestAgent: consul server down +51:19.284 [INFO] TestAgent: shutdown complete +51:19.284 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31463 network=tcp +51:19.284 [INFO] TestAgent: Stopping server: protocol=DNS address=127.0.0.1:31463 network=udp +51:19.284 [INFO] TestAgent: Stopping server: protocol=HTTP address=127.0.0.1:31464 network=tcp +51:19.785 [INFO] TestAgent: Waiting for endpoints to shut down +51:19.785 [INFO] TestAgent: Endpoints down +--- PASS: TestWatchCommand (0.81s) PASS -ok github.com/hashicorp/consul/command/watch 1.729s +ok github.com/hashicorp/consul/command/watch 1.856s === RUN TestStaticResolver_Resolve === RUN TestStaticResolver_Resolve/simples --- PASS: TestStaticResolver_Resolve (0.00s) @@ -91674,94 +91838,96 @@ === RUN TestConsulResolver_Resolve/timeout_errors === RUN TestConsulResolver_Resolve/prepared_query_by_id === RUN TestConsulResolver_Resolve/prepared_query_by_name -2021-10-31T23:19:10.511Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:19:10.543Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -19:10.545 [INFO] test-consul.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:ac3882a1-a3f8-dc2d-1616-4e7bac03e987 Address:127.0.0.1:11027}]" -19:10.545 [INFO] test-consul.server.raft: entering follower state: follower="Node at 127.0.0.1:11027 [Follower]" leader= -19:10.546 [INFO] test-consul.server.serf.wan: serf: EventMemberJoin: Node-ac3882a1-a3f8-dc2d-1616-4e7bac03e987.dc1 127.0.0.1 -19:10.546 [INFO] test-consul.server.serf.lan: serf: EventMemberJoin: Node-ac3882a1-a3f8-dc2d-1616-4e7bac03e987 127.0.0.1 -2021-10-31T23:19:10.546Z [INFO] agent.router: Initializing LAN area manager -19:10.547 [INFO] test-consul.server: Adding LAN server: server="Node-ac3882a1-a3f8-dc2d-1616-4e7bac03e987 (Addr: tcp/127.0.0.1:11027) (DC: dc1)" -19:10.547 [INFO] test-consul.server: Handled event for server in area: event=member-join server=Node-ac3882a1-a3f8-dc2d-1616-4e7bac03e987.dc1 area=wan -19:10.547 [INFO] test-consul: Started DNS server: address=127.0.0.1:11022 network=udp -19:10.547 [INFO] test-consul: Started DNS server: address=127.0.0.1:11022 network=tcp -19:10.547 [INFO] test-consul: Started HTTP server: address=127.0.0.1:11023 network=tcp -19:10.548 [INFO] test-consul: started state syncer -19:10.548 [INFO] test-consul: Started gRPC server: address=127.0.0.1:11028 network=tcp -19:10.584 [WARN] test-consul.server.raft: heartbeat timeout reached, starting election: last-leader= -19:10.584 [INFO] test-consul.server.raft: entering candidate state: node="Node at 127.0.0.1:11027 [Candidate]" term=2 -19:10.584 [DEBUG] test-consul.server.raft: votes: needed=1 -19:10.584 [DEBUG] test-consul.server.raft: vote granted: from=ac3882a1-a3f8-dc2d-1616-4e7bac03e987 term=2 tally=1 -19:10.584 [INFO] test-consul.server.raft: election won: tally=1 -19:10.584 [INFO] test-consul.server.raft: entering leader state: leader="Node at 127.0.0.1:11027 [Leader]" -19:10.585 [INFO] test-consul.server: cluster leadership acquired -19:10.585 [INFO] test-consul.server: New leader elected: payload=Node-ac3882a1-a3f8-dc2d-1616-4e7bac03e987 -19:10.585 [DEBUG] test-consul.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11027 -19:10.586 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -19:10.605 [INFO] test-consul.server.connect: initialized primary datacenter CA with provider: provider=consul -19:10.605 [INFO] test-consul.leader: started routine: routine="federation state anti-entropy" -19:10.605 [INFO] test-consul.leader: started routine: routine="federation state pruning" -19:10.605 [INFO] test-consul.leader: started routine: routine="intermediate cert renew watch" -19:10.605 [INFO] test-consul.leader: started routine: routine="CA root pruning" -19:10.605 [DEBUG] test-consul.server: Skipping self join check for node since the cluster is too small: node=Node-ac3882a1-a3f8-dc2d-1616-4e7bac03e987 -19:10.605 [INFO] test-consul.server: member joined, marking health alive: member=Node-ac3882a1-a3f8-dc2d-1616-4e7bac03e987 -19:10.606 [INFO] test-consul.server: federation state anti-entropy synced -19:10.695 [INFO] test-consul: Synced node info -19:10.695 [INFO] test-consul: Synced service: service=web -19:10.695 [DEBUG] test-consul.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:44928 latency=1.617192ms -19:10.696 [DEBUG] test-consul: Node info in sync -19:10.696 [DEBUG] test-consul: Service in sync: service=web -19:10.696 [INFO] test-consul: Synced service: service=web-proxy -19:10.696 [DEBUG] test-consul.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:44928 latency=545.642µs -19:10.698 [DEBUG] test-consul: Node info in sync -19:10.698 [DEBUG] test-consul: Service in sync: service=web -19:10.698 [DEBUG] test-consul: Service in sync: service=web-proxy -19:10.704 [INFO] test-consul: Synced service: service=web-proxy-2 -19:10.704 [DEBUG] test-consul.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:44928 latency=6.425408ms -19:10.705 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -19:10.705 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -19:10.705 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -19:10.705 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -19:10.705 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy-2 id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -19:10.705 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy-2 id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -19:10.705 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy-2 id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -19:10.705 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy-2 id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" -19:10.706 [DEBUG] test-consul: Node info in sync -19:10.706 [INFO] test-consul: Synced service: service=db -19:10.706 [DEBUG] test-consul: Service in sync: service=web -19:10.706 [DEBUG] test-consul: Service in sync: service=web-proxy -19:10.706 [DEBUG] test-consul: Service in sync: service=web-proxy-2 -19:10.706 [DEBUG] test-consul.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:44928 latency=733.243µs -19:10.709 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/query from=127.0.0.1:44928 latency=973.835µs -19:10.710 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/health/connect/web?connect=true&passing=1&stale= from=127.0.0.1:44928 latency=996.656µs -19:10.711 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/health/connect/db?connect=true&passing=1&stale= from=127.0.0.1:44928 latency=240.88µs -19:10.712 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/health/connect/foo?connect=true&passing=1&stale= from=127.0.0.1:44928 latency=159.535µs -19:10.713 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/query/c0149225-c00d-d70a-54f2-a648165788f7/execute?connect=true&stale= from=127.0.0.1:44928 latency=527.691µs -19:10.714 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/query/test-query/execute?connect=true&stale= from=127.0.0.1:44928 latency=404.86µs -19:10.716 [INFO] test-consul: Requesting shutdown -19:10.716 [INFO] test-consul.server: shutting down server -19:10.716 [DEBUG] test-consul.leader: stopping routine: routine="CA root pruning" -19:10.716 [DEBUG] test-consul.leader: stopping routine: routine="federation state anti-entropy" -19:10.716 [DEBUG] test-consul.leader: stopping routine: routine="federation state pruning" -19:10.716 [DEBUG] test-consul.leader: stopping routine: routine="intermediate cert renew watch" -19:10.716 [WARN] test-consul.server.serf.lan: serf: Shutdown without a Leave -19:10.716 [ERROR] test-consul.server: error performing anti-entropy sync of federation state: error="context canceled" -19:10.716 [DEBUG] test-consul.leader: stopped routine: routine="federation state anti-entropy" -19:10.716 [ERROR] test-consul.anti_entropy: failed to sync remote state: error="No cluster leader" -19:10.716 [DEBUG] test-consul.leader: stopped routine: routine="CA root pruning" -19:10.716 [DEBUG] test-consul.leader: stopped routine: routine="federation state pruning" -19:10.716 [DEBUG] test-consul.leader: stopped routine: routine="intermediate cert renew watch" -19:10.716 [WARN] test-consul.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:19:10.716Z [INFO] agent.router.manager: shutting down -2021-10-31T23:19:10.716Z [INFO] agent.router.manager: shutting down -19:10.716 [INFO] test-consul: consul server down -19:10.716 [INFO] test-consul: shutdown complete -19:10.716 [INFO] test-consul: Stopping server: protocol=DNS address=127.0.0.1:11022 network=tcp -19:10.716 [INFO] test-consul: Stopping server: protocol=DNS address=127.0.0.1:11022 network=udp -19:10.716 [INFO] test-consul: Stopping server: protocol=HTTP address=127.0.0.1:11023 network=tcp -19:11.216 [INFO] test-consul: Waiting for endpoints to shut down -19:11.216 [INFO] test-consul: Endpoints down ---- PASS: TestConsulResolver_Resolve (0.75s) +2022-12-04T05:51:18.530Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:18.548Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:18.550 [INFO] test-consul.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b279c12b-1f7c-eebb-e7aa-ff7fef6cbb28 Address:127.0.0.1:26342}]" +51:18.550 [INFO] test-consul.server.raft: entering follower state: follower="Node at 127.0.0.1:26342 [Follower]" leader= +51:18.551 [INFO] test-consul.server.serf.wan: serf: EventMemberJoin: Node-b279c12b-1f7c-eebb-e7aa-ff7fef6cbb28.dc1 127.0.0.1 +51:18.551 [INFO] test-consul.server.serf.lan: serf: EventMemberJoin: Node-b279c12b-1f7c-eebb-e7aa-ff7fef6cbb28 127.0.0.1 +2022-12-04T05:51:18.551Z [INFO] agent.router: Initializing LAN area manager +51:18.552 [INFO] test-consul.server: Adding LAN server: server="Node-b279c12b-1f7c-eebb-e7aa-ff7fef6cbb28 (Addr: tcp/127.0.0.1:26342) (DC: dc1)" +51:18.552 [INFO] test-consul: Started DNS server: address=127.0.0.1:26337 network=udp +51:18.552 [INFO] test-consul.server: Handled event for server in area: event=member-join server=Node-b279c12b-1f7c-eebb-e7aa-ff7fef6cbb28.dc1 area=wan +51:18.552 [INFO] test-consul: Started DNS server: address=127.0.0.1:26337 network=tcp +51:18.552 [INFO] test-consul: Started HTTP server: address=127.0.0.1:26338 network=tcp +51:18.552 [INFO] test-consul: started state syncer +51:18.552 [INFO] test-consul: Started gRPC server: address=127.0.0.1:26343 network=tcp +51:18.606 [WARN] test-consul.server.raft: heartbeat timeout reached, starting election: last-leader= +51:18.607 [INFO] test-consul.server.raft: entering candidate state: node="Node at 127.0.0.1:26342 [Candidate]" term=2 +51:18.607 [DEBUG] test-consul.server.raft: votes: needed=1 +51:18.607 [DEBUG] test-consul.server.raft: vote granted: from=b279c12b-1f7c-eebb-e7aa-ff7fef6cbb28 term=2 tally=1 +51:18.607 [INFO] test-consul.server.raft: election won: tally=1 +51:18.607 [INFO] test-consul.server.raft: entering leader state: leader="Node at 127.0.0.1:26342 [Leader]" +51:18.607 [INFO] test-consul.server: cluster leadership acquired +51:18.607 [DEBUG] test-consul.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:26342 +51:18.607 [INFO] test-consul.server: New leader elected: payload=Node-b279c12b-1f7c-eebb-e7aa-ff7fef6cbb28 +51:18.608 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:18.623 [INFO] test-consul.server.connect: initialized primary datacenter CA with provider: provider=consul +51:18.623 [INFO] test-consul.leader: started routine: routine="federation state anti-entropy" +51:18.623 [INFO] test-consul.leader: started routine: routine="federation state pruning" +51:18.623 [INFO] test-consul.leader: started routine: routine="intermediate cert renew watch" +51:18.623 [INFO] test-consul.leader: started routine: routine="CA root pruning" +51:18.623 [DEBUG] test-consul.server: Skipping self join check for node since the cluster is too small: node=Node-b279c12b-1f7c-eebb-e7aa-ff7fef6cbb28 +51:18.623 [INFO] test-consul.server: member joined, marking health alive: member=Node-b279c12b-1f7c-eebb-e7aa-ff7fef6cbb28 +51:18.624 [INFO] test-consul.server: federation state anti-entropy synced +51:18.764 [DEBUG] test-consul: Skipping remote check since it is managed automatically: check=serfHealth +51:18.765 [INFO] test-consul: Synced node info +51:18.765 [DEBUG] test-consul: Node info in sync +51:18.918 [DEBUG] test-consul: Node info in sync +51:18.919 [INFO] test-consul: Synced service: service=web +51:18.919 [DEBUG] test-consul.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:58576 latency=1.28697ms +51:18.920 [DEBUG] test-consul: Node info in sync +51:18.920 [DEBUG] test-consul: Service in sync: service=web +51:18.920 [INFO] test-consul: Synced service: service=web-proxy +51:18.920 [DEBUG] test-consul.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:58576 latency=764.81µs +51:18.922 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +51:18.922 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +51:18.928 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +51:18.932 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +51:18.934 [DEBUG] test-consul: Node info in sync +51:18.934 [DEBUG] test-consul: Service in sync: service=web +51:18.934 [DEBUG] test-consul: Service in sync: service=web-proxy +51:18.934 [INFO] test-consul: Synced service: service=web-proxy-2 +51:18.935 [DEBUG] test-consul.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:58576 latency=1.523592ms +51:18.935 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy-2 id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +51:18.935 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy-2 id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +51:18.935 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy-2 id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +51:18.935 [ERROR] test-consul.proxycfg: watch error: service_id=web-proxy-2 id=service-http-checks: error="error filling agent cache: Internal cache failure: service '' not in agent state" +51:18.935 [DEBUG] test-consul: Node info in sync +51:18.935 [DEBUG] test-consul: Service in sync: service=web +51:18.935 [DEBUG] test-consul: Service in sync: service=web-proxy +51:18.935 [DEBUG] test-consul: Service in sync: service=web-proxy-2 +51:18.936 [INFO] test-consul: Synced service: service=db +51:18.936 [DEBUG] test-consul.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:58576 latency=1.15332ms +51:18.937 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/query from=127.0.0.1:58576 latency=582.868µs +51:18.939 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/health/connect/web?connect=true&passing=1&stale= from=127.0.0.1:58576 latency=464.744µs +51:18.940 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/health/connect/db?connect=true&passing=1&stale= from=127.0.0.1:58576 latency=283.741µs +51:18.941 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/health/connect/foo?connect=true&passing=1&stale= from=127.0.0.1:58576 latency=498.859µs +51:18.942 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/query/f00a0e9b-8766-e4de-2088-8ad711ce728e/execute?connect=true&stale= from=127.0.0.1:58576 latency=303.845µs +51:18.943 [DEBUG] test-consul.http: Request finished: method=GET url=/v1/query/test-query/execute?connect=true&stale= from=127.0.0.1:58576 latency=287.253µs +51:18.944 [INFO] test-consul: Requesting shutdown +51:18.944 [INFO] test-consul.server: shutting down server +51:18.944 [DEBUG] test-consul.leader: stopping routine: routine="federation state anti-entropy" +51:18.944 [DEBUG] test-consul.leader: stopping routine: routine="federation state pruning" +51:18.944 [DEBUG] test-consul.leader: stopping routine: routine="intermediate cert renew watch" +51:18.944 [DEBUG] test-consul.leader: stopping routine: routine="CA root pruning" +51:18.944 [WARN] test-consul.server.serf.lan: serf: Shutdown without a Leave +51:18.944 [ERROR] test-consul.server: error performing anti-entropy sync of federation state: error="context canceled" +51:18.944 [DEBUG] test-consul.leader: stopped routine: routine="federation state anti-entropy" +51:18.944 [DEBUG] test-consul.leader: stopped routine: routine="federation state pruning" +51:18.944 [DEBUG] test-consul.leader: stopped routine: routine="intermediate cert renew watch" +51:18.944 [DEBUG] test-consul.leader: stopped routine: routine="CA root pruning" +51:18.944 [WARN] test-consul.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:18.945Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:18.945Z [INFO] agent.router.manager: shutting down +51:18.945 [INFO] test-consul: consul server down +51:18.945 [INFO] test-consul: shutdown complete +51:18.945 [INFO] test-consul: Stopping server: protocol=DNS address=127.0.0.1:26337 network=tcp +51:18.945 [INFO] test-consul: Stopping server: protocol=DNS address=127.0.0.1:26337 network=udp +51:18.945 [INFO] test-consul: Stopping server: protocol=HTTP address=127.0.0.1:26338 network=tcp +51:19.445 [INFO] test-consul: Waiting for endpoints to shut down +51:19.445 [INFO] test-consul: Endpoints down +--- PASS: TestConsulResolver_Resolve (0.94s) --- PASS: TestConsulResolver_Resolve/basic_service_discovery (0.00s) --- PASS: TestConsulResolver_Resolve/basic_service_with_native_service (0.00s) --- PASS: TestConsulResolver_Resolve/Bad_Type_errors (0.00s) @@ -91802,8 +91968,8 @@ service_test.go:129: DM-skipped --- SKIP: TestService_ServerTLSConfig (0.00s) === RUN TestService_HTTPClient -2021/10/31 23:19:11 starting test connect HTTPS server on 127.0.0.1:11029 -2021/10/31 23:19:11 test connect service listening on 127.0.0.1:11029 +2022/12/04 05:51:19 starting test connect HTTPS server on 127.0.0.1:26344 +2022/12/04 05:51:19 test connect service listening on 127.0.0.1:26344 --- PASS: TestService_HTTPClient (0.01s) === RUN TestService_HasDefaultHTTPResolverFromAddr --- PASS: TestService_HasDefaultHTTPResolverFromAddr (0.00s) @@ -91820,13 +91986,13 @@ --- PASS: Test_verifyServerCertMatchesURI/no_certs (0.00s) --- PASS: Test_verifyServerCertMatchesURI/nil_certs (0.00s) === RUN TestClientSideVerifier -2021-10-31T23:19:11.234Z [ERROR] connect.watch: Watch errored: service=foo type=connect_roots error="Get "http://127.0.0.1:8500/v1/agent/connect/ca/roots": dial tcp 127.0.0.1:8500: connect: connection refused" retry=5s -2021-10-31T23:19:11.234Z [ERROR] connect.watch: Watch errored: service=foo type=connect_leaf error="Get "http://127.0.0.1:8500/v1/agent/connect/ca/leaf/foo": dial tcp 127.0.0.1:8500: connect: connection refused" retry=5s +2022-12-04T05:51:19.464Z [ERROR] connect.watch: Watch errored: service=foo type=connect_leaf error="Get "http://127.0.0.1:8500/v1/agent/connect/ca/leaf/foo": dial tcp 127.0.0.1:8500: connect: connection refused" retry=5s +2022-12-04T05:51:19.464Z [ERROR] connect.watch: Watch errored: service=foo type=connect_roots error="Get "http://127.0.0.1:8500/v1/agent/connect/ca/roots": dial tcp 127.0.0.1:8500: connect: connection refused" retry=5s === RUN TestClientSideVerifier/ok_service_ca1 === RUN TestClientSideVerifier/untrusted_CA === RUN TestClientSideVerifier/cross_signed_intermediate === RUN TestClientSideVerifier/cross_signed_without_intermediate ---- PASS: TestClientSideVerifier (0.02s) +--- PASS: TestClientSideVerifier (0.01s) --- PASS: TestClientSideVerifier/ok_service_ca1 (0.00s) --- PASS: TestClientSideVerifier/untrusted_CA (0.00s) --- PASS: TestClientSideVerifier/cross_signed_intermediate (0.00s) @@ -91834,76 +92000,80 @@ === RUN TestServerSideVerifier === RUN TestServerSideVerifier/ok_service_ca1,_allow === RUN TestServerSideVerifier/untrusted_CA -2021-10-31T23:19:11.395Z [ERROR] TestServerSideVerifier/untrusted_CA: failed TLS verification: error="x509: certificate signed by unknown authority" +2022-12-04T05:51:19.879Z [ERROR] TestServerSideVerifier/untrusted_CA: failed TLS verification: error="x509: certificate signed by unknown authority" === RUN TestServerSideVerifier/cross_signed_intermediate,_allow === RUN TestServerSideVerifier/cross_signed_without_intermediate -2021-10-31T23:19:11.396Z [ERROR] TestServerSideVerifier/cross_signed_without_intermediate: failed TLS verification: error="x509: certificate signed by unknown authority" +2022-12-04T05:51:19.881Z [ERROR] TestServerSideVerifier/cross_signed_without_intermediate: failed TLS verification: error="x509: certificate signed by unknown authority" === RUN TestServerSideVerifier/ok_service_ca1,_deny -2021-10-31T23:19:11.398Z [ERROR] TestServerSideVerifier/ok_service_ca1,_deny: authz call denied: reason="Matched intention: DENY default/* => default/db (ID: e08c0b6b-8a99-2935-1800-6e45c20d125b, Precedence: 8)" +2022-12-04T05:51:19.882Z [ERROR] TestServerSideVerifier/ok_service_ca1,_deny: authz call denied: reason="Matched intention: DENY default/* => default/db (ID: 0ef52869-29d6-27bb-2d64-6954b94d26d5, Precedence: 8)" === RUN TestServerSideVerifier/cross_signed_intermediate,_deny -2021-10-31T23:19:11.399Z [ERROR] TestServerSideVerifier/cross_signed_intermediate,_deny: authz call denied: reason="Matched intention: DENY default/* => default/db (ID: e08c0b6b-8a99-2935-1800-6e45c20d125b, Precedence: 8)" -2021-10-31T23:19:11.263Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:19:11.273Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -19:11.275 [INFO] test-consul.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:d031896e-1180-61b7-ad3e-02b94d2c952c Address:127.0.0.1:11035}]" -19:11.275 [INFO] test-consul.server.raft: entering follower state: follower="Node at 127.0.0.1:11035 [Follower]" leader= -19:11.275 [INFO] test-consul.server.serf.wan: serf: EventMemberJoin: Node-d031896e-1180-61b7-ad3e-02b94d2c952c.dc1 127.0.0.1 -19:11.276 [INFO] test-consul.server.serf.lan: serf: EventMemberJoin: Node-d031896e-1180-61b7-ad3e-02b94d2c952c 127.0.0.1 -2021-10-31T23:19:11.276Z [INFO] agent.router: Initializing LAN area manager -19:11.276 [INFO] test-consul: Started DNS server: address=127.0.0.1:11030 network=udp -19:11.276 [INFO] test-consul.server: Adding LAN server: server="Node-d031896e-1180-61b7-ad3e-02b94d2c952c (Addr: tcp/127.0.0.1:11035) (DC: dc1)" -19:11.276 [INFO] test-consul.server: Handled event for server in area: event=member-join server=Node-d031896e-1180-61b7-ad3e-02b94d2c952c.dc1 area=wan -19:11.276 [INFO] test-consul: Started DNS server: address=127.0.0.1:11030 network=tcp -19:11.277 [INFO] test-consul: Started HTTP server: address=127.0.0.1:11031 network=tcp -19:11.277 [INFO] test-consul: started state syncer -19:11.277 [INFO] test-consul: Started gRPC server: address=127.0.0.1:11036 network=tcp -19:11.317 [WARN] test-consul.server.raft: heartbeat timeout reached, starting election: last-leader= -19:11.317 [INFO] test-consul.server.raft: entering candidate state: node="Node at 127.0.0.1:11035 [Candidate]" term=2 -19:11.317 [DEBUG] test-consul.server.raft: votes: needed=1 -19:11.317 [DEBUG] test-consul.server.raft: vote granted: from=d031896e-1180-61b7-ad3e-02b94d2c952c term=2 tally=1 -19:11.317 [INFO] test-consul.server.raft: election won: tally=1 -19:11.317 [INFO] test-consul.server.raft: entering leader state: leader="Node at 127.0.0.1:11035 [Leader]" -19:11.318 [INFO] test-consul.server: cluster leadership acquired -19:11.318 [DEBUG] test-consul.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:11035 -19:11.318 [INFO] test-consul.server: New leader elected: payload=Node-d031896e-1180-61b7-ad3e-02b94d2c952c -19:11.319 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -19:11.320 [INFO] test-consul.server.connect: initialized primary datacenter CA with provider: provider=consul -19:11.320 [INFO] test-consul.leader: started routine: routine="federation state anti-entropy" -19:11.320 [INFO] test-consul.leader: started routine: routine="federation state pruning" -19:11.320 [INFO] test-consul.leader: started routine: routine="intermediate cert renew watch" -19:11.320 [INFO] test-consul.leader: started routine: routine="CA root pruning" -19:11.320 [DEBUG] test-consul.server: Skipping self join check for node since the cluster is too small: node=Node-d031896e-1180-61b7-ad3e-02b94d2c952c -19:11.320 [INFO] test-consul.server: member joined, marking health alive: member=Node-d031896e-1180-61b7-ad3e-02b94d2c952c -19:11.321 [INFO] test-consul.server: federation state anti-entropy synced -19:11.381 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:58822 latency=6.9944ms -19:11.386 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:58822 latency=973.493µs -19:11.394 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:58822 latency=845.481µs -19:11.396 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:58822 latency=209.501µs -19:11.397 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:58822 latency=186.063µs -19:11.399 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:58822 latency=200.999µs -19:11.399 [INFO] test-consul: Requesting shutdown -19:11.399 [INFO] test-consul.server: shutting down server -19:11.399 [DEBUG] test-consul.leader: stopping routine: routine="federation state anti-entropy" -19:11.399 [DEBUG] test-consul.leader: stopping routine: routine="federation state pruning" -19:11.399 [DEBUG] test-consul.leader: stopping routine: routine="intermediate cert renew watch" -19:11.399 [DEBUG] test-consul.leader: stopping routine: routine="CA root pruning" -19:11.399 [WARN] test-consul.server.serf.lan: serf: Shutdown without a Leave -19:11.399 [ERROR] test-consul.server: error performing anti-entropy sync of federation state: error="context canceled" -19:11.399 [DEBUG] test-consul.leader: stopped routine: routine="federation state anti-entropy" -19:11.399 [ERROR] test-consul.anti_entropy: failed to sync remote state: error="No cluster leader" -19:11.399 [DEBUG] test-consul.leader: stopped routine: routine="federation state pruning" -19:11.399 [DEBUG] test-consul.leader: stopped routine: routine="intermediate cert renew watch" -19:11.399 [DEBUG] test-consul.leader: stopped routine: routine="CA root pruning" -19:11.400 [WARN] test-consul.server.serf.wan: serf: Shutdown without a Leave -2021-10-31T23:19:11.400Z [INFO] agent.router.manager: shutting down -2021-10-31T23:19:11.400Z [INFO] agent.router.manager: shutting down -19:11.400 [INFO] test-consul: consul server down -19:11.400 [INFO] test-consul: shutdown complete -19:11.400 [INFO] test-consul: Stopping server: protocol=DNS address=127.0.0.1:11030 network=tcp -19:11.400 [INFO] test-consul: Stopping server: protocol=DNS address=127.0.0.1:11030 network=udp -19:11.400 [INFO] test-consul: Stopping server: protocol=HTTP address=127.0.0.1:11031 network=tcp -19:11.901 [INFO] test-consul: Waiting for endpoints to shut down -19:11.901 [INFO] test-consul: Endpoints down ---- PASS: TestServerSideVerifier (0.65s) +2022-12-04T05:51:19.886Z [ERROR] TestServerSideVerifier/cross_signed_intermediate,_deny: authz call denied: reason="Matched intention: DENY default/* => default/db (ID: 0ef52869-29d6-27bb-2d64-6954b94d26d5, Precedence: 8)" +2022-12-04T05:51:19.481Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:19.490Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:19.492 [INFO] test-consul.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:b0b9580a-e4f1-4f00-f548-7f8f6da3b7b4 Address:127.0.0.1:26350}]" +51:19.492 [INFO] test-consul.server.raft: entering follower state: follower="Node at 127.0.0.1:26350 [Follower]" leader= +51:19.492 [INFO] test-consul.server.serf.wan: serf: EventMemberJoin: Node-b0b9580a-e4f1-4f00-f548-7f8f6da3b7b4.dc1 127.0.0.1 +51:19.492 [INFO] test-consul.server.serf.lan: serf: EventMemberJoin: Node-b0b9580a-e4f1-4f00-f548-7f8f6da3b7b4 127.0.0.1 +2022-12-04T05:51:19.492Z [INFO] agent.router: Initializing LAN area manager +51:19.493 [INFO] test-consul: Started DNS server: address=127.0.0.1:26345 network=udp +51:19.493 [INFO] test-consul.server: Handled event for server in area: event=member-join server=Node-b0b9580a-e4f1-4f00-f548-7f8f6da3b7b4.dc1 area=wan +51:19.493 [INFO] test-consul.server: Adding LAN server: server="Node-b0b9580a-e4f1-4f00-f548-7f8f6da3b7b4 (Addr: tcp/127.0.0.1:26350) (DC: dc1)" +51:19.493 [INFO] test-consul: Started DNS server: address=127.0.0.1:26345 network=tcp +51:19.493 [INFO] test-consul: Started HTTP server: address=127.0.0.1:26346 network=tcp +51:19.493 [INFO] test-consul: started state syncer +51:19.493 [INFO] test-consul: Started gRPC server: address=127.0.0.1:26351 network=tcp +51:19.529 [WARN] test-consul.server.raft: heartbeat timeout reached, starting election: last-leader= +51:19.529 [INFO] test-consul.server.raft: entering candidate state: node="Node at 127.0.0.1:26350 [Candidate]" term=2 +51:19.530 [DEBUG] test-consul.server.raft: votes: needed=1 +51:19.530 [DEBUG] test-consul.server.raft: vote granted: from=b0b9580a-e4f1-4f00-f548-7f8f6da3b7b4 term=2 tally=1 +51:19.530 [INFO] test-consul.server.raft: election won: tally=1 +51:19.530 [INFO] test-consul.server.raft: entering leader state: leader="Node at 127.0.0.1:26350 [Leader]" +51:19.530 [INFO] test-consul.server: cluster leadership acquired +51:19.530 [DEBUG] test-consul.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:26350 +51:19.530 [INFO] test-consul.server: New leader elected: payload=Node-b0b9580a-e4f1-4f00-f548-7f8f6da3b7b4 +51:19.530 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:19.531 [INFO] test-consul.server.connect: initialized primary datacenter CA with provider: provider=consul +51:19.531 [INFO] test-consul.leader: started routine: routine="federation state anti-entropy" +51:19.531 [INFO] test-consul.leader: started routine: routine="federation state pruning" +51:19.531 [INFO] test-consul.leader: started routine: routine="intermediate cert renew watch" +51:19.531 [INFO] test-consul.leader: started routine: routine="CA root pruning" +51:19.531 [DEBUG] test-consul.server: Skipping self join check for node since the cluster is too small: node=Node-b0b9580a-e4f1-4f00-f548-7f8f6da3b7b4 +51:19.531 [INFO] test-consul.server: member joined, marking health alive: member=Node-b0b9580a-e4f1-4f00-f548-7f8f6da3b7b4 +51:19.532 [INFO] test-consul.server: federation state anti-entropy synced +51:19.643 [DEBUG] test-consul: Skipping remote check since it is managed automatically: check=serfHealth +51:19.644 [INFO] test-consul: Synced node info +51:19.644 [DEBUG] test-consul: Node info in sync +51:19.677 [DEBUG] test-consul: Skipping remote check since it is managed automatically: check=serfHealth +51:19.677 [DEBUG] test-consul: Node info in sync +51:19.863 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:37422 latency=909.646µs +51:19.865 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/connect/intentions from=127.0.0.1:37422 latency=442.608µs +51:19.878 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:37422 latency=694.067µs +51:19.880 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:37422 latency=478.014µs +51:19.881 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:37422 latency=238.347µs +51:19.885 [DEBUG] test-consul.http: Request finished: method=POST url=/v1/agent/connect/authorize from=127.0.0.1:37422 latency=3.174796ms +51:19.886 [INFO] test-consul: Requesting shutdown +51:19.886 [INFO] test-consul.server: shutting down server +51:19.886 [DEBUG] test-consul.leader: stopping routine: routine="federation state anti-entropy" +51:19.886 [DEBUG] test-consul.leader: stopping routine: routine="federation state pruning" +51:19.886 [DEBUG] test-consul.leader: stopping routine: routine="intermediate cert renew watch" +51:19.886 [DEBUG] test-consul.leader: stopping routine: routine="CA root pruning" +51:19.886 [WARN] test-consul.server.serf.lan: serf: Shutdown without a Leave +51:19.886 [ERROR] test-consul.server: error performing anti-entropy sync of federation state: error="context canceled" +51:19.886 [DEBUG] test-consul.leader: stopped routine: routine="federation state anti-entropy" +51:19.886 [DEBUG] test-consul.leader: stopped routine: routine="federation state pruning" +51:19.886 [DEBUG] test-consul.leader: stopped routine: routine="intermediate cert renew watch" +51:19.886 [DEBUG] test-consul.leader: stopped routine: routine="CA root pruning" +51:19.886 [WARN] test-consul.server.serf.wan: serf: Shutdown without a Leave +2022-12-04T05:51:19.887Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:19.887Z [INFO] agent.router.manager: shutting down +51:19.887 [INFO] test-consul: consul server down +51:19.887 [INFO] test-consul: shutdown complete +51:19.887 [INFO] test-consul: Stopping server: protocol=DNS address=127.0.0.1:26345 network=tcp +51:19.887 [INFO] test-consul: Stopping server: protocol=DNS address=127.0.0.1:26345 network=udp +51:19.887 [INFO] test-consul: Stopping server: protocol=HTTP address=127.0.0.1:26346 network=tcp +51:20.387 [INFO] test-consul: Waiting for endpoints to shut down +51:20.387 [INFO] test-consul: Endpoints down +--- PASS: TestServerSideVerifier (0.92s) --- PASS: TestServerSideVerifier/ok_service_ca1,_allow (0.00s) --- PASS: TestServerSideVerifier/untrusted_CA (0.00s) --- PASS: TestServerSideVerifier/cross_signed_intermediate,_allow (0.00s) @@ -91915,7 +92085,7 @@ === RUN TestDynamicTLSConfig_Ready --- PASS: TestDynamicTLSConfig_Ready (0.00s) PASS -ok github.com/hashicorp/consul/connect 1.522s +ok github.com/hashicorp/consul/connect 1.939s ? github.com/hashicorp/consul/connect/certgen [no test files] === RUN TestUpstreamResolverFuncFromClient === PAUSE TestUpstreamResolverFuncFromClient @@ -91938,237 +92108,241 @@ proxy_test.go:21: DM-skipped --- SKIP: TestProxy_public (0.00s) === CONT TestUpstreamResolverFuncFromClient +=== CONT TestConnDstClosing === RUN TestUpstreamResolverFuncFromClient/service +=== CONT TestConnSrcClosing === RUN TestUpstreamResolverFuncFromClient/prepared_query +=== CONT TestAgentConfigWatcherSidecarProxy === RUN TestUpstreamResolverFuncFromClient/unknown_behaves_like_service --- PASS: TestUpstreamResolverFuncFromClient (0.00s) --- PASS: TestUpstreamResolverFuncFromClient/service (0.00s) --- PASS: TestUpstreamResolverFuncFromClient/prepared_query (0.00s) --- PASS: TestUpstreamResolverFuncFromClient/unknown_behaves_like_service (0.00s) -=== CONT TestConnDstClosing -=== CONT TestConnSrcClosing -=== CONT TestAgentConfigWatcherSidecarProxy [INFO] freeport: blockSize 1500 too big for system limit 1024. Adjusting... [INFO] freeport: detected ephemeral port range of [32768, 60999] [INFO] freeport: reducing max blocks from 30 to 22 to avoid the ephemeral port range ---- PASS: TestConnSrcClosing (0.00s) ---- PASS: TestConnDstClosing (0.01s) -2021-10-31T23:19:11.462Z [ERROR] TestAgentConfigWatcherSidecarProxy.watch: Watch errored: service_id=web-sidecar-proxy type=agent_service error="Get "http://127.0.0.1:15107/v1/agent/service/web-sidecar-proxy?hash=790f39b13effbdf2": dial tcp 127.0.0.1:15107: connect: connection refused" retry=5s -2021-10-31T23:19:11.192Z [WARN] agent: bootstrap = true: do not enable unless necessary -2021-10-31T23:19:11.205Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary -19:11.207 [INFO] agent_smith.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:9983e7a5-41e4-a7e5-e054-06cda3ad963e Address:127.0.0.1:15111}]" -19:11.211 [INFO] agent_smith.server.serf.wan: serf: EventMemberJoin: Node-9983e7a5-41e4-a7e5-e054-06cda3ad963e.dc1 127.0.0.1 -19:11.214 [INFO] agent_smith.server.raft: entering follower state: follower="Node at 127.0.0.1:15111 [Follower]" leader= -19:11.215 [INFO] agent_smith.server.serf.lan: serf: EventMemberJoin: Node-9983e7a5-41e4-a7e5-e054-06cda3ad963e 127.0.0.1 -2021-10-31T23:19:11.215Z [INFO] agent.router: Initializing LAN area manager -19:11.216 [INFO] agent_smith.server: Adding LAN server: server="Node-9983e7a5-41e4-a7e5-e054-06cda3ad963e (Addr: tcp/127.0.0.1:15111) (DC: dc1)" -19:11.216 [INFO] agent_smith.server: Handled event for server in area: event=member-join server=Node-9983e7a5-41e4-a7e5-e054-06cda3ad963e.dc1 area=wan -19:11.216 [INFO] agent_smith: Started DNS server: address=127.0.0.1:15106 network=udp -19:11.217 [INFO] agent_smith: Started DNS server: address=127.0.0.1:15106 network=tcp -19:11.218 [INFO] agent_smith: Started HTTP server: address=127.0.0.1:15107 network=tcp -19:11.218 [INFO] agent_smith: started state syncer -19:11.218 [INFO] agent_smith: Started gRPC server: address=127.0.0.1:15112 network=tcp -19:11.282 [WARN] agent_smith.server.raft: heartbeat timeout reached, starting election: last-leader= -19:11.282 [INFO] agent_smith.server.raft: entering candidate state: node="Node at 127.0.0.1:15111 [Candidate]" term=2 -19:11.282 [DEBUG] agent_smith.server.raft: votes: needed=1 -19:11.282 [DEBUG] agent_smith.server.raft: vote granted: from=9983e7a5-41e4-a7e5-e054-06cda3ad963e term=2 tally=1 -19:11.282 [INFO] agent_smith.server.raft: election won: tally=1 -19:11.282 [INFO] agent_smith.server.raft: entering leader state: leader="Node at 127.0.0.1:15111 [Leader]" -19:11.282 [INFO] agent_smith.server: cluster leadership acquired -19:11.282 [INFO] agent_smith.server: New leader elected: payload=Node-9983e7a5-41e4-a7e5-e054-06cda3ad963e -19:11.282 [DEBUG] agent_smith.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:15111 -19:11.283 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true -19:11.302 [INFO] agent_smith.server.connect: initialized primary datacenter CA with provider: provider=consul -19:11.302 [INFO] agent_smith.leader: started routine: routine="federation state anti-entropy" -19:11.302 [INFO] agent_smith.leader: started routine: routine="federation state pruning" -19:11.302 [INFO] agent_smith.leader: started routine: routine="intermediate cert renew watch" -19:11.302 [INFO] agent_smith.leader: started routine: routine="CA root pruning" -19:11.302 [DEBUG] agent_smith.server: Skipping self join check for node since the cluster is too small: node=Node-9983e7a5-41e4-a7e5-e054-06cda3ad963e -19:11.302 [INFO] agent_smith.server: member joined, marking health alive: member=Node-9983e7a5-41e4-a7e5-e054-06cda3ad963e -19:11.302 [INFO] agent_smith.server: federation state anti-entropy synced -19:11.420 [INFO] agent_smith: Synced node info -19:11.420 [INFO] agent_smith: Synced service: service=web -19:11.420 [INFO] agent_smith: Synced service: service=web-sidecar-proxy -19:11.420 [DEBUG] agent_smith: Check in sync: check=service:web-sidecar-proxy:1 -19:11.420 [DEBUG] agent_smith: Check in sync: check=service:web-sidecar-proxy:2 -19:11.420 [DEBUG] agent_smith.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:33808 latency=2.658624ms -19:11.422 [DEBUG] agent_smith.http: Request finished: method=GET url=/v1/agent/service/web-sidecar-proxy from=127.0.0.1:33808 latency=1.300947ms -19:11.448 [DEBUG] agent_smith: Node info in sync -19:11.449 [INFO] agent_smith: Synced service: service=web -19:11.449 [INFO] agent_smith: Synced service: service=web-sidecar-proxy -19:11.449 [DEBUG] agent_smith: Check in sync: check=service:web-sidecar-proxy:1 -19:11.449 [DEBUG] agent_smith: Check in sync: check=service:web-sidecar-proxy:2 -19:11.449 [DEBUG] agent_smith.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:33810 latency=2.203338ms -19:11.453 [DEBUG] agent_smith.http: Request finished: method=GET url=/v1/agent/service/web-sidecar-proxy?hash=e79fad2be70b72e8 from=127.0.0.1:33808 latency=27.134812ms -19:11.456 [INFO] agent_smith: Requesting shutdown -19:11.457 [INFO] agent_smith.server: shutting down server -19:11.457 [DEBUG] agent_smith.leader: stopping routine: routine="federation state anti-entropy" -19:11.457 [DEBUG] agent_smith.leader: stopping routine: routine="federation state pruning" -19:11.457 [DEBUG] agent_smith.leader: stopping routine: routine="intermediate cert renew watch" -19:11.457 [DEBUG] agent_smith.leader: stopping routine: routine="CA root pruning" -19:11.457 [WARN] agent_smith.server.serf.lan: serf: Shutdown without a Leave -19:11.457 [DEBUG] agent_smith.leader: stopped routine: routine="CA root pruning" -19:11.457 [ERROR] agent_smith.server: error performing anti-entropy sync of federation state: error="context canceled" -19:11.457 [DEBUG] agent_smith.leader: stopped routine: routine="federation state anti-entropy" -19:11.457 [WARN] agent_smith.server.serf.wan: serf: Shutdown without a Leave -19:11.457 [DEBUG] agent_smith.leader: stopped routine: routine="intermediate cert renew watch" -19:11.457 [DEBUG] agent_smith.leader: stopped routine: routine="federation state pruning" -19:11.457 [ERROR] agent_smith.anti_entropy: failed to sync remote state: error="No cluster leader" -2021-10-31T23:19:11.457Z [INFO] agent.router.manager: shutting down -2021-10-31T23:19:11.457Z [INFO] agent.router.manager: shutting down -19:11.457 [INFO] agent_smith: consul server down -19:11.457 [INFO] agent_smith: shutdown complete -19:11.457 [INFO] agent_smith: Stopping server: protocol=DNS address=127.0.0.1:15106 network=tcp -19:11.457 [INFO] agent_smith: Stopping server: protocol=DNS address=127.0.0.1:15106 network=udp -19:11.457 [INFO] agent_smith: Stopping server: protocol=HTTP address=127.0.0.1:15107 network=tcp -19:11.462 [DEBUG] agent_smith.http: Request finished: method=GET url=/v1/agent/service/web-sidecar-proxy?hash=790f39b13effbdf2 from=127.0.0.1:33808 latency=5.285265ms -19:11.957 [INFO] agent_smith: Waiting for endpoints to shut down -19:11.957 [INFO] agent_smith: Endpoints down ---- PASS: TestAgentConfigWatcherSidecarProxy (0.81s) +--- PASS: TestConnDstClosing (0.00s) +--- PASS: TestConnSrcClosing (0.01s) +2022-12-04T05:51:21.097Z [ERROR] TestAgentConfigWatcherSidecarProxy.watch: Watch errored: service_id=web-sidecar-proxy type=agent_service error="Get "http://127.0.0.1:10002/v1/agent/service/web-sidecar-proxy?hash=790f39b13effbdf2": dial tcp 127.0.0.1:10002: connect: connection refused" retry=5s +2022-12-04T05:51:20.997Z [WARN] agent: bootstrap = true: do not enable unless necessary +2022-12-04T05:51:21.011Z [WARN] agent.auto_config: bootstrap = true: do not enable unless necessary +51:21.014 [INFO] agent_smith.server.raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:8a0a3475-6054-687c-2d75-7be2d5ab33c3 Address:127.0.0.1:10006}]" +51:21.015 [INFO] agent_smith.server.raft: entering follower state: follower="Node at 127.0.0.1:10006 [Follower]" leader= +51:21.015 [INFO] agent_smith.server.serf.wan: serf: EventMemberJoin: Node-8a0a3475-6054-687c-2d75-7be2d5ab33c3.dc1 127.0.0.1 +51:21.016 [INFO] agent_smith.server.serf.lan: serf: EventMemberJoin: Node-8a0a3475-6054-687c-2d75-7be2d5ab33c3 127.0.0.1 +2022-12-04T05:51:21.016Z [INFO] agent.router: Initializing LAN area manager +51:21.016 [INFO] agent_smith.server: Handled event for server in area: event=member-join server=Node-8a0a3475-6054-687c-2d75-7be2d5ab33c3.dc1 area=wan +51:21.016 [INFO] agent_smith.server: Adding LAN server: server="Node-8a0a3475-6054-687c-2d75-7be2d5ab33c3 (Addr: tcp/127.0.0.1:10006) (DC: dc1)" +51:21.016 [INFO] agent_smith: Started DNS server: address=127.0.0.1:10001 network=udp +51:21.016 [INFO] agent_smith: Started DNS server: address=127.0.0.1:10001 network=tcp +51:21.017 [INFO] agent_smith: Started HTTP server: address=127.0.0.1:10002 network=tcp +51:21.017 [INFO] agent_smith: started state syncer +51:21.017 [INFO] agent_smith: Started gRPC server: address=127.0.0.1:10007 network=tcp +51:21.050 [WARN] agent_smith.server.raft: heartbeat timeout reached, starting election: last-leader= +51:21.050 [INFO] agent_smith.server.raft: entering candidate state: node="Node at 127.0.0.1:10006 [Candidate]" term=2 +51:21.050 [DEBUG] agent_smith.server.raft: votes: needed=1 +51:21.050 [DEBUG] agent_smith.server.raft: vote granted: from=8a0a3475-6054-687c-2d75-7be2d5ab33c3 term=2 tally=1 +51:21.050 [INFO] agent_smith.server.raft: election won: tally=1 +51:21.050 [INFO] agent_smith.server.raft: entering leader state: leader="Node at 127.0.0.1:10006 [Leader]" +51:21.051 [INFO] agent_smith.server: cluster leadership acquired +51:21.056 [DEBUG] agent_smith.server: Cannot upgrade to new ACLs: leaderMode=0 mode=0 found=true leader=127.0.0.1:10006 +51:21.056 [INFO] agent_smith.server: New leader elected: payload=Node-8a0a3475-6054-687c-2d75-7be2d5ab33c3 +51:21.058 [DEBUG] connect.ca.consul: consul CA provider configured: id=07:80:c8:de:f6:41:86:29:8f:9c:b8:17:d6:48:c2:d5:c5:5c:7f:0c:03:f7:cf:97:5a:a7:c1:68:aa:23:ae:81 is_primary=true +51:21.059 [INFO] agent_smith: Synced node info +51:21.060 [INFO] agent_smith: Synced service: service=web +51:21.060 [INFO] agent_smith: Synced service: service=web-sidecar-proxy +51:21.060 [DEBUG] agent_smith: Check in sync: check=service:web-sidecar-proxy:1 +51:21.060 [DEBUG] agent_smith: Check in sync: check=service:web-sidecar-proxy:2 +51:21.060 [DEBUG] agent_smith.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:56274 latency=3.071055ms +51:21.061 [DEBUG] agent_smith.http: Request finished: method=GET url=/v1/agent/service/web-sidecar-proxy from=127.0.0.1:56274 latency=467.835µs +51:21.075 [ERROR] agent_smith.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: internal error: CA root is nil" +51:21.076 [ERROR] agent_smith.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: internal error: CA root is nil" +51:21.076 [ERROR] agent_smith.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: internal error: CA root is nil" +51:21.076 [ERROR] agent_smith.proxycfg: watch error: service_id=web-sidecar-proxy id=leaf error="error filling agent cache: internal error: CA root is nil" +51:21.076 [INFO] agent_smith.server.connect: initialized primary datacenter CA with provider: provider=consul +51:21.076 [INFO] agent_smith.leader: started routine: routine="federation state anti-entropy" +51:21.076 [INFO] agent_smith.leader: started routine: routine="federation state pruning" +51:21.076 [INFO] agent_smith.leader: started routine: routine="intermediate cert renew watch" +51:21.076 [INFO] agent_smith.leader: started routine: routine="CA root pruning" +51:21.076 [DEBUG] agent_smith.server: Skipping self join check for node since the cluster is too small: node=Node-8a0a3475-6054-687c-2d75-7be2d5ab33c3 +51:21.076 [INFO] agent_smith.server: member joined, marking health alive: member=Node-8a0a3475-6054-687c-2d75-7be2d5ab33c3 +51:21.077 [INFO] agent_smith.server: federation state anti-entropy synced +51:21.085 [DEBUG] agent_smith: Node info in sync +51:21.085 [INFO] agent_smith: Synced service: service=web +51:21.086 [INFO] agent_smith: Synced service: service=web-sidecar-proxy +51:21.086 [DEBUG] agent_smith: Check in sync: check=service:web-sidecar-proxy:1 +51:21.086 [DEBUG] agent_smith: Check in sync: check=service:web-sidecar-proxy:2 +51:21.086 [DEBUG] agent_smith.http: Request finished: method=PUT url=/v1/agent/service/register from=127.0.0.1:56276 latency=2.059803ms +51:21.087 [DEBUG] agent_smith.http: Request finished: method=GET url=/v1/agent/service/web-sidecar-proxy?hash=e79fad2be70b72e8 from=127.0.0.1:56274 latency=23.473756ms +51:21.094 [INFO] agent_smith: Requesting shutdown +51:21.094 [INFO] agent_smith.server: shutting down server +51:21.094 [DEBUG] agent_smith.leader: stopping routine: routine="CA root pruning" +51:21.094 [DEBUG] agent_smith.leader: stopping routine: routine="federation state anti-entropy" +51:21.094 [DEBUG] agent_smith.leader: stopping routine: routine="federation state pruning" +51:21.094 [DEBUG] agent_smith.leader: stopping routine: routine="intermediate cert renew watch" +51:21.094 [WARN] agent_smith.server.serf.lan: serf: Shutdown without a Leave +51:21.094 [ERROR] agent_smith.server: error performing anti-entropy sync of federation state: error="context canceled" +51:21.094 [DEBUG] agent_smith.leader: stopped routine: routine="federation state anti-entropy" +51:21.094 [ERROR] agent_smith.anti_entropy: failed to sync remote state: error="No cluster leader" +51:21.094 [DEBUG] agent_smith.leader: stopped routine: routine="CA root pruning" +51:21.094 [WARN] agent_smith.server.serf.wan: serf: Shutdown without a Leave +51:21.094 [DEBUG] agent_smith.leader: stopped routine: routine="federation state pruning" +51:21.094 [DEBUG] agent_smith.leader: stopped routine: routine="intermediate cert renew watch" +2022-12-04T05:51:21.094Z [INFO] agent.router.manager: shutting down +2022-12-04T05:51:21.094Z [INFO] agent.router.manager: shutting down +51:21.094 [INFO] agent_smith: consul server down +51:21.094 [INFO] agent_smith: shutdown complete +51:21.094 [INFO] agent_smith: Stopping server: protocol=DNS address=127.0.0.1:10001 network=tcp +51:21.094 [INFO] agent_smith: Stopping server: protocol=DNS address=127.0.0.1:10001 network=udp +51:21.095 [INFO] agent_smith: Stopping server: protocol=HTTP address=127.0.0.1:10002 network=tcp +51:21.095 [DEBUG] agent_smith.http: Request finished: method=GET url=/v1/agent/service/web-sidecar-proxy?hash=790f39b13effbdf2 from=127.0.0.1:56274 latency=944.695µs +51:21.595 [INFO] agent_smith: Waiting for endpoints to shut down +51:21.595 [INFO] agent_smith: Endpoints down +--- PASS: TestAgentConfigWatcherSidecarProxy (0.63s) PASS -ok github.com/hashicorp/consul/connect/proxy 0.875s +ok github.com/hashicorp/consul/connect/proxy 0.686s === RUN TestConfigValidate -=== RUN TestConfigValidate/type=jwt/too_many_methods_(discovery_+_jwks) -=== RUN TestConfigValidate/type=jwt/all_required_for_jwks -=== RUN TestConfigValidate/type=inferred_oidc/invalid_repeated_value_claim_mappings -=== RUN TestConfigValidate/type=oidc/valid_algorithm -=== RUN TestConfigValidate/type=inferred_oidc/missing_required_OIDCDiscoveryURL -=== RUN TestConfigValidate/type=oidc/incompatible_with_BoundIssuer -=== RUN TestConfigValidate/type=oidc/incompatible_with_JWKSCACert -=== RUN TestConfigValidate/type=jwt/incompatible_with_AllowedRedirectURIs -=== RUN TestConfigValidate/type=inferred_oidc/invalid_redirect_uris -=== RUN TestConfigValidate/type=jwt/valid_algorithm -=== RUN TestConfigValidate/type=jwt/incompatible_with_OIDCClientID -=== RUN TestConfigValidate/type=jwt/jwks_bad_certs -=== RUN TestConfigValidate/type=inferred_oidc/missing_required_OIDCClientSecret -=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_JWTValidationPubKeys -=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_JWKSCACert +=== RUN TestConfigValidate/type=oidc/good_discovery_cert +=== RUN TestConfigValidate/type=jwt/incompatible_with_OIDCDiscoveryCACert === RUN TestConfigValidate/type=jwt/bad_discovery_cert -=== RUN TestConfigValidate/type=inferred_oidc/all_required -=== RUN TestConfigValidate/type=oidc/incompatible_with_JWTValidationPubKeys -=== RUN TestConfigValidate/type=jwt/jwks_mismatched_certs -=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_NotBeforeLeeway +=== RUN TestConfigValidate/type=oidc/incompatible_with_NotBeforeLeeway +=== RUN TestConfigValidate/type=oidc/incompatible_with_ExpirationLeeway +=== RUN TestConfigValidate/type=inferred_oidc/invalid_repeated_list_claim_mappings === RUN TestConfigValidate/type=inferred_oidc/bad_discovery_cert -=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_BoundIssuer +=== RUN TestConfigValidate/type=oidc/invalid_redirect_uris === RUN TestConfigValidate/type=jwt/incompatible_with_OIDCClientSecret -=== RUN TestConfigValidate/type=jwt/all_required_for_oidc_discovery -=== RUN TestConfigValidate/type=oidc/incompatible_with_ExpirationLeeway -=== RUN TestConfigValidate/type=inferred_oidc/missing_required_AllowedRedirectURIs -=== RUN TestConfigValidate/type=jwt/too_many_methods_(discovery_+_pubkeys) -=== RUN TestConfigValidate/type=jwt/incompatible_with_VerboseOIDCLogging -=== RUN TestConfigValidate/type=inferred_oidc/good_discovery_cert -=== RUN TestConfigValidate/type=inferred_oidc/valid_algorithm +=== RUN TestConfigValidate/type=oidc/incompatible_with_JWTValidationPubKeys +=== RUN TestConfigValidate/type=inferred_oidc/missing_required_OIDCClientSecret +=== RUN TestConfigValidate/type=oidc/missing_required_AllowedRedirectURIs === RUN TestConfigValidate/type=jwt/jwks_invalid_404 -=== RUN TestConfigValidate/type=jwt/incompatible_with_OIDCDiscoveryCACert -=== RUN TestConfigValidate/type=oidc/all_required -=== RUN TestConfigValidate/type=oidc/incompatible_with_NotBeforeLeeway -=== RUN TestConfigValidate/type=inferred_oidc/missing_required_OIDCClientID -=== RUN TestConfigValidate/type=inferred_oidc/valid_redirect_uris -=== RUN TestConfigValidate/type=jwt/all_required_for_public_keys -=== RUN TestConfigValidate/type=jwt/invalid_repeated_value_claim_mappings -=== RUN TestConfigValidate/type=oidc/invalid_algorithm -=== RUN TestConfigValidate/type=oidc/invalid_repeated_value_claim_mappings -=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_ClockSkewLeeway -=== RUN TestConfigValidate/type=jwt/invalid_algorithm +=== RUN TestConfigValidate/type=oidc/invalid_repeated_list_claim_mappings +=== RUN TestConfigValidate/type=oidc/valid_algorithm +=== RUN TestConfigValidate/type=inferred_oidc/valid_algorithm +=== RUN TestConfigValidate/type=jwt/incompatible_with_OIDCClientID === RUN TestConfigValidate/type=jwt/valid_claim_mappings -=== RUN TestConfigValidate/type=oidc/missing_required_OIDCClientID +=== RUN TestConfigValidate/type=inferred_oidc/missing_required_OIDCDiscoveryURL === RUN TestConfigValidate/type=inferred_oidc/garbage_discovery_cert -=== RUN TestConfigValidate/type=oidc/invalid_repeated_list_claim_mappings -=== RUN TestConfigValidate/type=jwt/too_many_methods_(discovery_+_jwks_+_pubkeys) -=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_JWKSURL +=== RUN TestConfigValidate/type=jwt/too_many_methods_(jwks_+_pubkeys) +=== RUN TestConfigValidate/type=jwt/invalid_algorithm +=== RUN TestConfigValidate/type=jwt/too_many_methods_(discovery_+_jwks) +=== RUN TestConfigValidate/type=jwt/all_required_for_public_keys +=== RUN TestConfigValidate/type=jwt/jwks_mismatched_certs +=== RUN TestConfigValidate/type=jwt/invalid_repeated_list_claim_mappings === RUN TestConfigValidate/type=oidc/incompatible_with_ClockSkewLeeway -=== RUN TestConfigValidate/type=inferred_oidc/invalid_repeated_list_claim_mappings -=== RUN TestConfigValidate/type=oidc/good_discovery_cert -=== RUN TestConfigValidate/type=oidc/missing_required_AllowedRedirectURIs +=== RUN TestConfigValidate/type=inferred_oidc/invalid_repeated_value_claim_mappings +=== RUN TestConfigValidate/type=oidc/missing_required_OIDCClientID +=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_JWTValidationPubKeys +=== RUN TestConfigValidate/type=inferred_oidc/invalid_algorithm +=== RUN TestConfigValidate/type=inferred_oidc/valid_claim_mappings +=== RUN TestConfigValidate/type=oidc/incompatible_with_JWKSURL +=== RUN TestConfigValidate/type=inferred_oidc/missing_required_OIDCClientID +=== RUN TestConfigValidate/type=jwt/invalid_repeated_value_claim_mappings +=== RUN TestConfigValidate/type=oidc/missing_required_OIDCDiscoveryURL +=== RUN TestConfigValidate/type=oidc/garbage_discovery_cert +=== RUN TestConfigValidate/type=inferred_oidc/valid_redirect_uris === RUN TestConfigValidate/type=jwt/invalid_pubkey -=== RUN TestConfigValidate/bad_type -=== RUN TestConfigValidate/type=oidc/invalid_redirect_uris -=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_ExpirationLeeway +=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_ClockSkewLeeway +=== RUN TestConfigValidate/type=inferred_oidc/invalid_redirect_uris +=== RUN TestConfigValidate/type=oidc/invalid_algorithm +=== RUN TestConfigValidate/type=jwt/valid_algorithm === RUN TestConfigValidate/type=jwt/incompatible_with_OIDCScopes -=== RUN TestConfigValidate/type=jwt/too_many_methods_(jwks_+_pubkeys) === RUN TestConfigValidate/type=jwt/incompatible_with_JWKSCACert +=== RUN TestConfigValidate/type=jwt/jwks_bad_certs +=== RUN TestConfigValidate/bad_type +=== RUN TestConfigValidate/type=oidc/all_required +=== RUN TestConfigValidate/type=oidc/invalid_repeated_value_claim_mappings +=== RUN TestConfigValidate/type=oidc/valid_claim_mappings +=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_JWKSURL +=== RUN TestConfigValidate/type=jwt/good_discovery_cert +=== RUN TestConfigValidate/type=jwt/all_required_for_jwks +=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_BoundIssuer +=== RUN TestConfigValidate/type=inferred_oidc/all_required === RUN TestConfigValidate/type=oidc/missing_required_OIDCClientSecret -=== RUN TestConfigValidate/type=oidc/incompatible_with_JWKSURL +=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_ExpirationLeeway +=== RUN TestConfigValidate/type=jwt/all_required_for_oidc_discovery +=== RUN TestConfigValidate/type=oidc/incompatible_with_BoundIssuer === RUN TestConfigValidate/type=oidc/bad_discovery_cert -=== RUN TestConfigValidate/type=oidc/missing_required_OIDCDiscoveryURL -=== RUN TestConfigValidate/type=oidc/garbage_discovery_cert -=== RUN TestConfigValidate/type=oidc/valid_claim_mappings -=== RUN TestConfigValidate/type=inferred_oidc/invalid_algorithm +=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_JWKSCACert +=== RUN TestConfigValidate/type=inferred_oidc/good_discovery_cert === RUN TestConfigValidate/type=oidc/valid_redirect_uris -=== RUN TestConfigValidate/type=inferred_oidc/valid_claim_mappings -=== RUN TestConfigValidate/type=jwt/invalid_repeated_list_claim_mappings -=== RUN TestConfigValidate/type=jwt/good_discovery_cert ---- PASS: TestConfigValidate (0.10s) - --- PASS: TestConfigValidate/type=jwt/too_many_methods_(discovery_+_jwks) (0.00s) - --- PASS: TestConfigValidate/type=jwt/all_required_for_jwks (0.01s) - --- PASS: TestConfigValidate/type=inferred_oidc/invalid_repeated_value_claim_mappings (0.00s) - --- PASS: TestConfigValidate/type=oidc/valid_algorithm (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/missing_required_OIDCDiscoveryURL (0.00s) - --- PASS: TestConfigValidate/type=oidc/incompatible_with_BoundIssuer (0.00s) - --- PASS: TestConfigValidate/type=oidc/incompatible_with_JWKSCACert (0.00s) - --- PASS: TestConfigValidate/type=jwt/incompatible_with_AllowedRedirectURIs (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/invalid_redirect_uris (0.00s) - --- PASS: TestConfigValidate/type=jwt/valid_algorithm (0.00s) - --- PASS: TestConfigValidate/type=jwt/incompatible_with_OIDCClientID (0.00s) - --- PASS: TestConfigValidate/type=jwt/jwks_bad_certs (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/missing_required_OIDCClientSecret (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_JWTValidationPubKeys (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_JWKSCACert (0.00s) +=== RUN TestConfigValidate/type=jwt/incompatible_with_AllowedRedirectURIs +=== RUN TestConfigValidate/type=jwt/incompatible_with_VerboseOIDCLogging +=== RUN TestConfigValidate/type=jwt/too_many_methods_(discovery_+_jwks_+_pubkeys) +=== RUN TestConfigValidate/type=jwt/too_many_methods_(discovery_+_pubkeys) +=== RUN TestConfigValidate/type=inferred_oidc/incompatible_with_NotBeforeLeeway +=== RUN TestConfigValidate/type=oidc/incompatible_with_JWKSCACert +=== RUN TestConfigValidate/type=inferred_oidc/missing_required_AllowedRedirectURIs +--- PASS: TestConfigValidate (0.08s) + --- PASS: TestConfigValidate/type=oidc/good_discovery_cert (0.00s) + --- PASS: TestConfigValidate/type=jwt/incompatible_with_OIDCDiscoveryCACert (0.00s) --- PASS: TestConfigValidate/type=jwt/bad_discovery_cert (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/all_required (0.00s) - --- PASS: TestConfigValidate/type=oidc/incompatible_with_JWTValidationPubKeys (0.00s) - --- PASS: TestConfigValidate/type=jwt/jwks_mismatched_certs (0.01s) - --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_NotBeforeLeeway (0.00s) + --- PASS: TestConfigValidate/type=oidc/incompatible_with_NotBeforeLeeway (0.00s) + --- PASS: TestConfigValidate/type=oidc/incompatible_with_ExpirationLeeway (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/invalid_repeated_list_claim_mappings (0.00s) --- PASS: TestConfigValidate/type=inferred_oidc/bad_discovery_cert (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_BoundIssuer (0.00s) + --- PASS: TestConfigValidate/type=oidc/invalid_redirect_uris (0.00s) --- PASS: TestConfigValidate/type=jwt/incompatible_with_OIDCClientSecret (0.00s) - --- PASS: TestConfigValidate/type=jwt/all_required_for_oidc_discovery (0.00s) - --- PASS: TestConfigValidate/type=oidc/incompatible_with_ExpirationLeeway (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/missing_required_AllowedRedirectURIs (0.00s) - --- PASS: TestConfigValidate/type=jwt/too_many_methods_(discovery_+_pubkeys) (0.00s) - --- PASS: TestConfigValidate/type=jwt/incompatible_with_VerboseOIDCLogging (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/good_discovery_cert (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/valid_algorithm (0.00s) + --- PASS: TestConfigValidate/type=oidc/incompatible_with_JWTValidationPubKeys (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/missing_required_OIDCClientSecret (0.00s) + --- PASS: TestConfigValidate/type=oidc/missing_required_AllowedRedirectURIs (0.00s) --- PASS: TestConfigValidate/type=jwt/jwks_invalid_404 (0.00s) - --- PASS: TestConfigValidate/type=jwt/incompatible_with_OIDCDiscoveryCACert (0.00s) - --- PASS: TestConfigValidate/type=oidc/all_required (0.00s) - --- PASS: TestConfigValidate/type=oidc/incompatible_with_NotBeforeLeeway (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/missing_required_OIDCClientID (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/valid_redirect_uris (0.01s) - --- PASS: TestConfigValidate/type=jwt/all_required_for_public_keys (0.00s) - --- PASS: TestConfigValidate/type=jwt/invalid_repeated_value_claim_mappings (0.00s) - --- PASS: TestConfigValidate/type=oidc/invalid_algorithm (0.01s) - --- PASS: TestConfigValidate/type=oidc/invalid_repeated_value_claim_mappings (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_ClockSkewLeeway (0.00s) - --- PASS: TestConfigValidate/type=jwt/invalid_algorithm (0.00s) + --- PASS: TestConfigValidate/type=oidc/invalid_repeated_list_claim_mappings (0.00s) + --- PASS: TestConfigValidate/type=oidc/valid_algorithm (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/valid_algorithm (0.00s) + --- PASS: TestConfigValidate/type=jwt/incompatible_with_OIDCClientID (0.00s) --- PASS: TestConfigValidate/type=jwt/valid_claim_mappings (0.00s) - --- PASS: TestConfigValidate/type=oidc/missing_required_OIDCClientID (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/missing_required_OIDCDiscoveryURL (0.00s) --- PASS: TestConfigValidate/type=inferred_oidc/garbage_discovery_cert (0.00s) - --- PASS: TestConfigValidate/type=oidc/invalid_repeated_list_claim_mappings (0.00s) - --- PASS: TestConfigValidate/type=jwt/too_many_methods_(discovery_+_jwks_+_pubkeys) (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_JWKSURL (0.00s) + --- PASS: TestConfigValidate/type=jwt/too_many_methods_(jwks_+_pubkeys) (0.00s) + --- PASS: TestConfigValidate/type=jwt/invalid_algorithm (0.00s) + --- PASS: TestConfigValidate/type=jwt/too_many_methods_(discovery_+_jwks) (0.00s) + --- PASS: TestConfigValidate/type=jwt/all_required_for_public_keys (0.00s) + --- PASS: TestConfigValidate/type=jwt/jwks_mismatched_certs (0.00s) + --- PASS: TestConfigValidate/type=jwt/invalid_repeated_list_claim_mappings (0.00s) --- PASS: TestConfigValidate/type=oidc/incompatible_with_ClockSkewLeeway (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/invalid_repeated_list_claim_mappings (0.00s) - --- PASS: TestConfigValidate/type=oidc/good_discovery_cert (0.00s) - --- PASS: TestConfigValidate/type=oidc/missing_required_AllowedRedirectURIs (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/invalid_repeated_value_claim_mappings (0.00s) + --- PASS: TestConfigValidate/type=oidc/missing_required_OIDCClientID (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_JWTValidationPubKeys (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/invalid_algorithm (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/valid_claim_mappings (0.00s) + --- PASS: TestConfigValidate/type=oidc/incompatible_with_JWKSURL (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/missing_required_OIDCClientID (0.00s) + --- PASS: TestConfigValidate/type=jwt/invalid_repeated_value_claim_mappings (0.00s) + --- PASS: TestConfigValidate/type=oidc/missing_required_OIDCDiscoveryURL (0.00s) + --- PASS: TestConfigValidate/type=oidc/garbage_discovery_cert (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/valid_redirect_uris (0.00s) --- PASS: TestConfigValidate/type=jwt/invalid_pubkey (0.00s) - --- PASS: TestConfigValidate/bad_type (0.00s) - --- PASS: TestConfigValidate/type=oidc/invalid_redirect_uris (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_ExpirationLeeway (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_ClockSkewLeeway (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/invalid_redirect_uris (0.00s) + --- PASS: TestConfigValidate/type=oidc/invalid_algorithm (0.01s) + --- PASS: TestConfigValidate/type=jwt/valid_algorithm (0.00s) --- PASS: TestConfigValidate/type=jwt/incompatible_with_OIDCScopes (0.00s) - --- PASS: TestConfigValidate/type=jwt/too_many_methods_(jwks_+_pubkeys) (0.00s) --- PASS: TestConfigValidate/type=jwt/incompatible_with_JWKSCACert (0.00s) + --- PASS: TestConfigValidate/type=jwt/jwks_bad_certs (0.00s) + --- PASS: TestConfigValidate/bad_type (0.00s) + --- PASS: TestConfigValidate/type=oidc/all_required (0.00s) + --- PASS: TestConfigValidate/type=oidc/invalid_repeated_value_claim_mappings (0.00s) + --- PASS: TestConfigValidate/type=oidc/valid_claim_mappings (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_JWKSURL (0.00s) + --- PASS: TestConfigValidate/type=jwt/good_discovery_cert (0.00s) + --- PASS: TestConfigValidate/type=jwt/all_required_for_jwks (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_BoundIssuer (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/all_required (0.00s) --- PASS: TestConfigValidate/type=oidc/missing_required_OIDCClientSecret (0.00s) - --- PASS: TestConfigValidate/type=oidc/incompatible_with_JWKSURL (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_ExpirationLeeway (0.00s) + --- PASS: TestConfigValidate/type=jwt/all_required_for_oidc_discovery (0.00s) + --- PASS: TestConfigValidate/type=oidc/incompatible_with_BoundIssuer (0.00s) --- PASS: TestConfigValidate/type=oidc/bad_discovery_cert (0.00s) - --- PASS: TestConfigValidate/type=oidc/missing_required_OIDCDiscoveryURL (0.00s) - --- PASS: TestConfigValidate/type=oidc/garbage_discovery_cert (0.00s) - --- PASS: TestConfigValidate/type=oidc/valid_claim_mappings (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/invalid_algorithm (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_JWKSCACert (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/good_discovery_cert (0.00s) --- PASS: TestConfigValidate/type=oidc/valid_redirect_uris (0.00s) - --- PASS: TestConfigValidate/type=inferred_oidc/valid_claim_mappings (0.00s) - --- PASS: TestConfigValidate/type=jwt/invalid_repeated_list_claim_mappings (0.00s) - --- PASS: TestConfigValidate/type=jwt/good_discovery_cert (0.00s) + --- PASS: TestConfigValidate/type=jwt/incompatible_with_AllowedRedirectURIs (0.00s) + --- PASS: TestConfigValidate/type=jwt/incompatible_with_VerboseOIDCLogging (0.00s) + --- PASS: TestConfigValidate/type=jwt/too_many_methods_(discovery_+_jwks_+_pubkeys) (0.00s) + --- PASS: TestConfigValidate/type=jwt/too_many_methods_(discovery_+_pubkeys) (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/incompatible_with_NotBeforeLeeway (0.00s) + --- PASS: TestConfigValidate/type=oidc/incompatible_with_JWKSCACert (0.00s) + --- PASS: TestConfigValidate/type=inferred_oidc/missing_required_AllowedRedirectURIs (0.00s) === RUN TestJWT_OIDC_Functions_Fail === RUN TestJWT_OIDC_Functions_Fail/static === RUN TestJWT_OIDC_Functions_Fail/static/GetAuthCodeURL @@ -92212,7 +92386,7 @@ === RUN TestJWT_ClaimsFromJWT/oidc_discovery/bad_signature === RUN TestJWT_ClaimsFromJWT/oidc_discovery/bad_issuer === RUN TestJWT_ClaimsFromJWT/oidc_discovery/bad_audience ---- PASS: TestJWT_ClaimsFromJWT (0.17s) +--- PASS: TestJWT_ClaimsFromJWT (0.15s) --- PASS: TestJWT_ClaimsFromJWT/static (0.00s) --- PASS: TestJWT_ClaimsFromJWT/static/missing_audience (0.00s) --- PASS: TestJWT_ClaimsFromJWT/static/valid_inputs (0.00s) @@ -92220,18 +92394,18 @@ --- PASS: TestJWT_ClaimsFromJWT/static/bad_signature (0.00s) --- PASS: TestJWT_ClaimsFromJWT/static/bad_issuer (0.00s) --- PASS: TestJWT_ClaimsFromJWT/static/bad_audience (0.00s) - --- PASS: TestJWT_ClaimsFromJWT/JWKS (0.09s) - --- PASS: TestJWT_ClaimsFromJWT/JWKS/missing_audience (0.02s) + --- PASS: TestJWT_ClaimsFromJWT/JWKS (0.08s) + --- PASS: TestJWT_ClaimsFromJWT/JWKS/missing_audience (0.01s) --- PASS: TestJWT_ClaimsFromJWT/JWKS/valid_inputs (0.01s) --- PASS: TestJWT_ClaimsFromJWT/JWKS/unusable_claims (0.01s) - --- PASS: TestJWT_ClaimsFromJWT/JWKS/bad_signature (0.02s) + --- PASS: TestJWT_ClaimsFromJWT/JWKS/bad_signature (0.01s) --- PASS: TestJWT_ClaimsFromJWT/JWKS/bad_issuer (0.02s) - --- PASS: TestJWT_ClaimsFromJWT/JWKS/bad_audience (0.01s) - --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery (0.07s) + --- PASS: TestJWT_ClaimsFromJWT/JWKS/bad_audience (0.02s) + --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery (0.06s) --- SKIP: TestJWT_ClaimsFromJWT/oidc_discovery/missing_audience (0.00s) --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery/valid_inputs (0.01s) --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery/unusable_claims (0.02s) - --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery/bad_signature (0.02s) + --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery/bad_signature (0.01s) --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery/bad_issuer (0.01s) --- PASS: TestJWT_ClaimsFromJWT/oidc_discovery/bad_audience (0.01s) === RUN TestJWT_ClaimsFromJWT_ExpiryClaims @@ -92240,394 +92414,394 @@ === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_iat_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_iat_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_iat_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_iat_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_iat_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_iat_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_iat_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_iat_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_no_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_iat_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_iat_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_iat_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_iat_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_iat_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_iat_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_iat_with_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_iat_with_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== PAUSE TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_iat_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_iat_with_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_iat_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_iat_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_iat_with_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf +=== CONT TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims (0.00s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway (0.01s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway (0.01s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway (0.01s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_iat_with_clock_skew_leeway (0.01s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_no_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_iat_with_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_iat_with_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway (0.01s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway (0.01s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway (0.01s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway (0.01s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_iat_with_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway (0.01s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway (0.01s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway (0.01s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.01s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway (0.02s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway (0.01s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/no_expire_leeway_using_iat_with_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_no_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/static/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway (0.02s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway (0.08s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_no_clock_skew_leeway (0.12s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway (0.12s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.08s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway (0.10s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway (0.09s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway (0.09s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_iat_with_clock_skew_leeway (0.11s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.11s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway (0.10s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway (0.11s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway (0.11s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway (0.12s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_iat_with_clock_skew_leeway (0.11s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.12s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.12s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_iat_with_clock_skew_leeway (0.13s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_auto_clock_skew_leeway (0.14s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway (0.14s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_no_clock_skew_leeway (0.14s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway (0.15s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_iat_with_clock_skew_leeway (0.16s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_custom_clock_skew_leeway (0.16s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway (0.16s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway (0.10s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_custom_clock_skew_leeway (0.06s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf (0.06s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway (0.05s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_iat_with_no_clock_skew_leeway (0.19s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway (0.08s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway (0.20s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf (0.06s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway (0.09s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway (0.08s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway (0.12s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway (0.04s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway (0.14s) --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_nbf_with_clock_skew_leeway (0.06s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_auto_clock_skew_leeway (0.08s) - --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway (0.05s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_no_clock_skew_leeway (0.14s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/custom_expire_leeway_using_iat_with_clock_skew_leeway (0.07s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway (0.03s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_no_clock_skew_leeway (0.06s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf (0.04s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_auto_expire_leeway_using_nbf_with_custom_clock_skew_leeway (0.06s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_custom_expire_leeway_using_iat_with_clock_skew_leeway_with_default_leeway (0.05s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/auto_expire_leeway_using_iat_with_auto_clock_skew_leeway (0.05s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/expired_no_expire_leeway_using_nbf_with_clock_skew_leeway (0.04s) + --- PASS: TestJWT_ClaimsFromJWT_ExpiryClaims/JWKS/no_expire_leeway_using_nbf_with_clock_skew_leeway_with_default_leeway_and_nbf (0.04s) === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/nbf_leeway_using_iat_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/nbf_leeway_using_iat_with_no_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/nbf_leeway_using_iat_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/nbf_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/nbf_leeway_using_iat_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/nbf_leeway_using_iat_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway === RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway === PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== RUN TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/nbf_leeway_using_iat_with_no_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== PAUSE TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/nbf_leeway_using_iat_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/nbf_leeway_using_iat_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/nbf_leeway_using_iat_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway === CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway -=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway +=== CONT TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims (0.00s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/nbf_leeway_using_iat_with_no_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.00s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/nbf_leeway_using_iat_with_no_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/static/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.00s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS (0.00s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway (0.05s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/nbf_leeway_using_iat_with_no_clock_skew_leeway (0.05s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.06s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.06s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.08s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.09s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.09s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.10s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.10s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway (0.10s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway (0.10s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.10s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway (0.11s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.06s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.11s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.06s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway (0.04s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway (0.12s) - --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.06s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.06s) --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_custom_clock_skew_leeway (0.06s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway (0.05s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.06s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.04s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.06s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.07s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_nbf_leeway_using_iat_with_no_clock_skew_leeway (0.06s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_custom_clock_skew_leeway (0.08s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.08s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/nbf_leeway_using_iat_with_no_clock_skew_leeway (0.08s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_no_clock_skew_leeway (0.08s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_no_nbf_leeway_using_iat_with_auto_clock_skew_leeway (0.07s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/no_nbf_leeway_using_iat_with_auto_clock_skew_leeway (0.08s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.07s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.08s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_auto_nbf_leeway_using_exp_with_auto_clock_skew_leeway (0.03s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.03s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/not_yet_valid_custom_nbf_leeway_using_exp_with_custom_clock_skew_leeway (0.03s) + --- PASS: TestJWT_ClaimsFromJWT_NotBeforeClaims/JWKS/custom_nbf_leeway_using_exp_with_no_clock_skew_leeway_with_default_leeway (0.03s) === RUN TestParsePublicKeyPEM === RUN TestParsePublicKeyPEM/rsa === RUN TestParsePublicKeyPEM/ecdsa === RUN TestParsePublicKeyPEM/ed25519 ---- PASS: TestParsePublicKeyPEM (0.25s) - --- PASS: TestParsePublicKeyPEM/rsa (0.25s) +--- PASS: TestParsePublicKeyPEM (0.61s) + --- PASS: TestParsePublicKeyPEM/rsa (0.61s) --- PASS: TestParsePublicKeyPEM/ecdsa (0.00s) --- PASS: TestParsePublicKeyPEM/ed25519 (0.00s) === RUN TestOIDC_AuthURL @@ -92641,9 +92815,9 @@ === CONT TestOIDC_AuthURL/missing_RedirectURI === CONT TestOIDC_AuthURL/invalid_RedirectURI --- PASS: TestOIDC_AuthURL (0.00s) - --- PASS: TestOIDC_AuthURL/invalid_RedirectURI (0.01s) - --- PASS: TestOIDC_AuthURL/normal_case (0.01s) --- PASS: TestOIDC_AuthURL/missing_RedirectURI (0.01s) + --- PASS: TestOIDC_AuthURL/normal_case (0.01s) + --- PASS: TestOIDC_AuthURL/invalid_RedirectURI (0.01s) === RUN TestOIDC_JWT_Functions_Fail --- PASS: TestOIDC_JWT_Functions_Fail (0.01s) === RUN TestOIDC_ClaimsFromAuthCode @@ -92660,12 +92834,12 @@ === RUN TestOIDC_ClaimsFromAuthCode/no_response_from_provider === RUN TestOIDC_ClaimsFromAuthCode/invalid_bound_audience === CONT TestOIDC_ClaimsFromAuthCode/failed_login_-_bad_nonce ---- PASS: TestOIDC_ClaimsFromAuthCode (0.16s) - --- PASS: TestOIDC_ClaimsFromAuthCode/successful_login (0.03s) - --- PASS: TestOIDC_ClaimsFromAuthCode/failed_login_unusable_claims (0.02s) +--- PASS: TestOIDC_ClaimsFromAuthCode (0.13s) + --- PASS: TestOIDC_ClaimsFromAuthCode/successful_login (0.01s) + --- PASS: TestOIDC_ClaimsFromAuthCode/failed_login_unusable_claims (0.01s) --- PASS: TestOIDC_ClaimsFromAuthCode/successful_login_-_no_userinfo (0.01s) --- PASS: TestOIDC_ClaimsFromAuthCode/missing_state (0.01s) - --- PASS: TestOIDC_ClaimsFromAuthCode/unknown_state (0.01s) + --- PASS: TestOIDC_ClaimsFromAuthCode/unknown_state (0.02s) --- PASS: TestOIDC_ClaimsFromAuthCode/valid_state,_missing_code (0.01s) --- PASS: TestOIDC_ClaimsFromAuthCode/failed_code_exchange (0.02s) --- PASS: TestOIDC_ClaimsFromAuthCode/no_id_token_returned (0.01s) @@ -92673,52 +92847,52 @@ --- PASS: TestOIDC_ClaimsFromAuthCode/invalid_bound_audience (0.01s) --- PASS: TestOIDC_ClaimsFromAuthCode/failed_login_-_bad_nonce (0.01s) === RUN TestExtractStringMetadata -=== RUN TestExtractStringMetadata/empty === RUN TestExtractStringMetadata/all === RUN TestExtractStringMetadata/some === RUN TestExtractStringMetadata/none === RUN TestExtractStringMetadata/nested_data === RUN TestExtractStringMetadata/error:_a_struct_isn't_stringifiable === RUN TestExtractStringMetadata/error:_a_slice_isn't_stringifiable +=== RUN TestExtractStringMetadata/empty --- PASS: TestExtractStringMetadata (0.00s) - --- PASS: TestExtractStringMetadata/empty (0.00s) --- PASS: TestExtractStringMetadata/all (0.00s) --- PASS: TestExtractStringMetadata/some (0.00s) --- PASS: TestExtractStringMetadata/none (0.00s) --- PASS: TestExtractStringMetadata/nested_data (0.00s) --- PASS: TestExtractStringMetadata/error:_a_struct_isn't_stringifiable (0.00s) --- PASS: TestExtractStringMetadata/error:_a_slice_isn't_stringifiable (0.00s) + --- PASS: TestExtractStringMetadata/empty (0.00s) === RUN TestExtractListMetadata === RUN TestExtractListMetadata/empty -=== RUN TestExtractListMetadata/some_-_list +=== RUN TestExtractListMetadata/none_-_list === RUN TestExtractListMetadata/nested_data_-_list -=== RUN TestExtractListMetadata/list_omits_empty_strings === RUN TestExtractListMetadata/JSONPointer_not_found === RUN TestExtractListMetadata/all_-_singular === RUN TestExtractListMetadata/some_-_singular -=== RUN TestExtractListMetadata/non-string-slice_data_(string) -=== RUN TestExtractListMetadata/none_-_list -=== RUN TestExtractListMetadata/none_-_singular === RUN TestExtractListMetadata/nested_data_-_singular === RUN TestExtractListMetadata/all_-_list -=== RUN TestExtractListMetadata/error:_a_struct_isn't_stringifiable_(singular) +=== RUN TestExtractListMetadata/some_-_list +=== RUN TestExtractListMetadata/none_-_singular === RUN TestExtractListMetadata/error:_a_slice_isn't_stringifiable_(singular) +=== RUN TestExtractListMetadata/non-string-slice_data_(string) +=== RUN TestExtractListMetadata/list_omits_empty_strings +=== RUN TestExtractListMetadata/error:_a_struct_isn't_stringifiable_(singular) === RUN TestExtractListMetadata/JSONPointer --- PASS: TestExtractListMetadata (0.00s) --- PASS: TestExtractListMetadata/empty (0.00s) - --- PASS: TestExtractListMetadata/some_-_list (0.00s) + --- PASS: TestExtractListMetadata/none_-_list (0.00s) --- PASS: TestExtractListMetadata/nested_data_-_list (0.00s) - --- PASS: TestExtractListMetadata/list_omits_empty_strings (0.00s) --- PASS: TestExtractListMetadata/JSONPointer_not_found (0.00s) --- PASS: TestExtractListMetadata/all_-_singular (0.00s) --- PASS: TestExtractListMetadata/some_-_singular (0.00s) - --- PASS: TestExtractListMetadata/non-string-slice_data_(string) (0.00s) - --- PASS: TestExtractListMetadata/none_-_list (0.00s) - --- PASS: TestExtractListMetadata/none_-_singular (0.00s) --- PASS: TestExtractListMetadata/nested_data_-_singular (0.00s) --- PASS: TestExtractListMetadata/all_-_list (0.00s) - --- PASS: TestExtractListMetadata/error:_a_struct_isn't_stringifiable_(singular) (0.00s) + --- PASS: TestExtractListMetadata/some_-_list (0.00s) + --- PASS: TestExtractListMetadata/none_-_singular (0.00s) --- PASS: TestExtractListMetadata/error:_a_slice_isn't_stringifiable_(singular) (0.00s) + --- PASS: TestExtractListMetadata/non-string-slice_data_(string) (0.00s) + --- PASS: TestExtractListMetadata/list_omits_empty_strings (0.00s) + --- PASS: TestExtractListMetadata/error:_a_struct_isn't_stringifiable_(singular) (0.00s) --- PASS: TestExtractListMetadata/JSONPointer (0.00s) === RUN TestGetClaim === RUN TestGetClaim/a @@ -92769,51 +92943,51 @@ --- PASS: TestNormalizeList/struct_{_A_int_}{A:5} (0.00s) --- PASS: TestNormalizeList/ (0.00s) === RUN TestStringifyMetadataValue -=== RUN TestStringifyMetadataValue/float64 -=== RUN TestStringifyMetadataValue/float32 +=== RUN TestStringifyMetadataValue/uint16 +=== RUN TestStringifyMetadataValue/uint32 +=== RUN TestStringifyMetadataValue/uint64 +=== RUN TestStringifyMetadataValue/int_slice +=== RUN TestStringifyMetadataValue/empty_string +=== RUN TestStringifyMetadataValue/string +=== RUN TestStringifyMetadataValue/false === RUN TestStringifyMetadataValue/int === RUN TestStringifyMetadataValue/uint8 -=== RUN TestStringifyMetadataValue/uint -=== RUN TestStringifyMetadataValue/empty_string -=== RUN TestStringifyMetadataValue/true === RUN TestStringifyMetadataValue/int8 -=== RUN TestStringifyMetadataValue/int16 === RUN TestStringifyMetadataValue/int32 -=== RUN TestStringifyMetadataValue/int64 -=== RUN TestStringifyMetadataValue/uint64 -=== RUN TestStringifyMetadataValue/string -=== RUN TestStringifyMetadataValue/uint32 -=== RUN TestStringifyMetadataValue/false -=== RUN TestStringifyMetadataValue/json_number -=== RUN TestStringifyMetadataValue/uint16 === RUN TestStringifyMetadataValue/string_slice -=== RUN TestStringifyMetadataValue/int_slice +=== RUN TestStringifyMetadataValue/nil +=== RUN TestStringifyMetadataValue/true +=== RUN TestStringifyMetadataValue/json_number +=== RUN TestStringifyMetadataValue/float64 +=== RUN TestStringifyMetadataValue/float32 === RUN TestStringifyMetadataValue/map === RUN TestStringifyMetadataValue/struct -=== RUN TestStringifyMetadataValue/nil +=== RUN TestStringifyMetadataValue/int16 +=== RUN TestStringifyMetadataValue/int64 +=== RUN TestStringifyMetadataValue/uint --- PASS: TestStringifyMetadataValue (0.00s) - --- PASS: TestStringifyMetadataValue/float64 (0.00s) - --- PASS: TestStringifyMetadataValue/float32 (0.00s) + --- PASS: TestStringifyMetadataValue/uint16 (0.00s) + --- PASS: TestStringifyMetadataValue/uint32 (0.00s) + --- PASS: TestStringifyMetadataValue/uint64 (0.00s) + --- PASS: TestStringifyMetadataValue/int_slice (0.00s) + --- PASS: TestStringifyMetadataValue/empty_string (0.00s) + --- PASS: TestStringifyMetadataValue/string (0.00s) + --- PASS: TestStringifyMetadataValue/false (0.00s) --- PASS: TestStringifyMetadataValue/int (0.00s) --- PASS: TestStringifyMetadataValue/uint8 (0.00s) - --- PASS: TestStringifyMetadataValue/uint (0.00s) - --- PASS: TestStringifyMetadataValue/empty_string (0.00s) - --- PASS: TestStringifyMetadataValue/true (0.00s) --- PASS: TestStringifyMetadataValue/int8 (0.00s) - --- PASS: TestStringifyMetadataValue/int16 (0.00s) --- PASS: TestStringifyMetadataValue/int32 (0.00s) - --- PASS: TestStringifyMetadataValue/int64 (0.00s) - --- PASS: TestStringifyMetadataValue/uint64 (0.00s) - --- PASS: TestStringifyMetadataValue/string (0.00s) - --- PASS: TestStringifyMetadataValue/uint32 (0.00s) - --- PASS: TestStringifyMetadataValue/false (0.00s) - --- PASS: TestStringifyMetadataValue/json_number (0.00s) - --- PASS: TestStringifyMetadataValue/uint16 (0.00s) --- PASS: TestStringifyMetadataValue/string_slice (0.00s) - --- PASS: TestStringifyMetadataValue/int_slice (0.00s) + --- PASS: TestStringifyMetadataValue/nil (0.00s) + --- PASS: TestStringifyMetadataValue/true (0.00s) + --- PASS: TestStringifyMetadataValue/json_number (0.00s) + --- PASS: TestStringifyMetadataValue/float64 (0.00s) + --- PASS: TestStringifyMetadataValue/float32 (0.00s) --- PASS: TestStringifyMetadataValue/map (0.00s) --- PASS: TestStringifyMetadataValue/struct (0.00s) - --- PASS: TestStringifyMetadataValue/nil (0.00s) + --- PASS: TestStringifyMetadataValue/int16 (0.00s) + --- PASS: TestStringifyMetadataValue/int64 (0.00s) + --- PASS: TestStringifyMetadataValue/uint (0.00s) === RUN TestValidateAudience === RUN TestValidateAudience/boundAudiences=[]string{"a"}_audience=[]string{"a"}_strict=false === RUN TestValidateAudience/boundAudiences=[]string{"a"}_audience=[]string{"a"}_strict=true @@ -92883,11 +93057,11 @@ --- PASS: TestValidRedirect/uri="https://[::1]:5000/x/y?r=42"_allowed=[]string{"a",_"b",_"https://[::1]:5000/x/y?r=43"} (0.00s) --- PASS: TestValidRedirect/uri="%%%%%%%%%%%"_allowed=[]string{} (0.00s) PASS -ok github.com/hashicorp/consul/internal/go-sso/oidcauth 1.148s +ok github.com/hashicorp/consul/internal/go-sso/oidcauth 1.336s === RUN TestStrListContains --- PASS: TestStrListContains (0.00s) PASS -ok github.com/hashicorp/consul/internal/go-sso/oidcauth/internal/strutil 0.007s +ok github.com/hashicorp/consul/internal/go-sso/oidcauth/internal/strutil 0.005s ? github.com/hashicorp/consul/internal/go-sso/oidcauth/oidcauthtest [no test files] === RUN TestIsPrivateIP === RUN TestIsPrivateIP/10.0.0.1 @@ -92960,7 +93134,7 @@ --- PASS: TestIsIPv6/localhost (0.00s) --- PASS: TestIsIPv6/1.257.0.1 (0.00s) PASS -ok github.com/hashicorp/consul/ipaddr 0.016s +ok github.com/hashicorp/consul/ipaddr 0.006s === RUN TestDurationMinusBuffer --- PASS: TestDurationMinusBuffer (0.00s) === RUN TestDurationMinusBufferDomain @@ -93051,24 +93225,24 @@ === RUN TestMathMaxUint64 --- PASS: TestMathMaxUint64 (0.00s) === CONT TestJitterRandomStagger -=== RUN TestJitterRandomStagger/0_percent -=== PAUSE TestJitterRandomStagger/0_percent -=== RUN TestJitterRandomStagger/10_percent -=== PAUSE TestJitterRandomStagger/10_percent -=== RUN TestJitterRandomStagger/100_percent -=== PAUSE TestJitterRandomStagger/100_percent -=== CONT TestJitterRandomStagger/0_percent === CONT TestRetryWaiter_WaitChans === RUN TestRetryWaiter_WaitChans/Minimum_Wait_-_Success === PAUSE TestRetryWaiter_WaitChans/Minimum_Wait_-_Success === RUN TestRetryWaiter_WaitChans/Minimum_Wait_-_WaitIf === PAUSE TestRetryWaiter_WaitChans/Minimum_Wait_-_WaitIf +=== RUN TestJitterRandomStagger/0_percent === RUN TestRetryWaiter_WaitChans/Minimum_Wait_-_WaitIfErr +=== PAUSE TestJitterRandomStagger/0_percent +=== RUN TestJitterRandomStagger/10_percent === PAUSE TestRetryWaiter_WaitChans/Minimum_Wait_-_WaitIfErr +=== PAUSE TestJitterRandomStagger/10_percent === RUN TestRetryWaiter_WaitChans/Maximum_Wait_-_Failed === PAUSE TestRetryWaiter_WaitChans/Maximum_Wait_-_Failed +=== RUN TestJitterRandomStagger/100_percent === RUN TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIf +=== PAUSE TestJitterRandomStagger/100_percent === PAUSE TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIf +=== CONT TestJitterRandomStagger/0_percent === RUN TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIfErr === PAUSE TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIfErr === CONT TestRetryWaiter_WaitChans/Minimum_Wait_-_Success @@ -93079,37 +93253,37 @@ === PAUSE TestRetryWaiter_calculateWait/Minimum_Wait === RUN TestRetryWaiter_calculateWait/Minimum_Failures === PAUSE TestRetryWaiter_calculateWait/Minimum_Failures -=== RUN TestRetryWaiter_calculateWait/Maximum_Wait -=== PAUSE TestRetryWaiter_calculateWait/Maximum_Wait -=== CONT TestRetryWaiter_calculateWait/Defaults -=== CONT TestJitterRandomStagger/100_percent === CONT TestJitterRandomStagger/10_percent ---- PASS: TestJitterRandomStagger (0.00s) - --- PASS: TestJitterRandomStagger/0_percent (0.00s) - --- PASS: TestJitterRandomStagger/100_percent (0.00s) - --- PASS: TestJitterRandomStagger/10_percent (0.00s) -=== CONT TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIfErr -=== CONT TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIf === CONT TestRetryWaiter_WaitChans/Maximum_Wait_-_Failed +=== CONT TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIfErr === CONT TestRetryWaiter_WaitChans/Minimum_Wait_-_WaitIfErr +=== CONT TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIf === CONT TestRetryWaiter_WaitChans/Minimum_Wait_-_WaitIf +=== RUN TestRetryWaiter_calculateWait/Maximum_Wait +=== PAUSE TestRetryWaiter_calculateWait/Maximum_Wait +=== CONT TestRetryWaiter_calculateWait/Defaults === CONT TestRetryWaiter_calculateWait/Maximum_Wait -=== CONT TestRetryWaiter_calculateWait/Minimum_Failures === CONT TestRetryWaiter_calculateWait/Minimum_Wait +=== CONT TestJitterRandomStagger/100_percent +=== CONT TestRetryWaiter_calculateWait/Minimum_Failures --- PASS: TestRetryWaiter_calculateWait (0.00s) --- PASS: TestRetryWaiter_calculateWait/Defaults (0.00s) + --- PASS: TestRetryWaiter_calculateWait/Minimum_Wait (0.00s) --- PASS: TestRetryWaiter_calculateWait/Maximum_Wait (0.00s) --- PASS: TestRetryWaiter_calculateWait/Minimum_Failures (0.00s) - --- PASS: TestRetryWaiter_calculateWait/Minimum_Wait (0.00s) +--- PASS: TestJitterRandomStagger (0.00s) + --- PASS: TestJitterRandomStagger/0_percent (0.00s) + --- PASS: TestJitterRandomStagger/10_percent (0.00s) + --- PASS: TestJitterRandomStagger/100_percent (0.00s) --- PASS: TestRetryWaiter_WaitChans (0.00s) - --- PASS: TestRetryWaiter_WaitChans/Minimum_Wait_-_WaitIf (0.20s) --- PASS: TestRetryWaiter_WaitChans/Minimum_Wait_-_Success (0.20s) + --- PASS: TestRetryWaiter_WaitChans/Minimum_Wait_-_WaitIf (0.20s) --- PASS: TestRetryWaiter_WaitChans/Minimum_Wait_-_WaitIfErr (0.20s) - --- PASS: TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIfErr (0.25s) - --- PASS: TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIf (0.25s) --- PASS: TestRetryWaiter_WaitChans/Maximum_Wait_-_Failed (0.25s) + --- PASS: TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIf (0.25s) + --- PASS: TestRetryWaiter_WaitChans/Maximum_Wait_-_WaitIfErr (0.25s) PASS -ok github.com/hashicorp/consul/lib 0.264s +ok github.com/hashicorp/consul/lib 0.263s === RUN TestHookTranslateKeys === RUN TestHookTranslateKeys/target_of_type_struct,_with_struct_receiver === RUN TestHookTranslateKeys/target_of_type_ptr,_with_struct_receiver @@ -93141,11 +93315,11 @@ === RUN TestHookWeakDecodeFromSlice_NestedOpaqueConfig --- PASS: TestHookWeakDecodeFromSlice_NestedOpaqueConfig (0.00s) PASS -ok github.com/hashicorp/consul/lib/decode 0.012s +ok github.com/hashicorp/consul/lib/decode 0.008s === RUN TestWriteAtomic --- PASS: TestWriteAtomic (0.00s) PASS -ok github.com/hashicorp/consul/lib/file 0.006s +ok github.com/hashicorp/consul/lib/file 0.005s === RUN TestDynamic === PAUSE TestDynamic === RUN TestDynamicPanic @@ -93153,13 +93327,13 @@ === RUN TestDynamicAcquire === PAUSE TestDynamicAcquire === CONT TestDynamic -=== CONT TestDynamicAcquire === CONT TestDynamicPanic --- PASS: TestDynamicPanic (0.00s) +=== CONT TestDynamicAcquire --- PASS: TestDynamicAcquire (0.05s) ---- PASS: TestDynamic (0.51s) +--- PASS: TestDynamic (0.46s) PASS -ok github.com/hashicorp/consul/lib/semaphore 0.524s +ok github.com/hashicorp/consul/lib/semaphore 0.465s === RUN TestContains --- PASS: TestContains (0.00s) === RUN TestEqual @@ -93181,97 +93355,97 @@ --- PASS: TestEqual/[]string{"a",_"b"}_=?=_[]string{"a",_"b"} (0.00s) --- PASS: TestEqual/[]string{"a",_"b"}_=?=_[]string{"b",_"a"} (0.00s) === RUN TestMergeSorted +=== RUN TestMergeSorted/one_and_one_dupe +=== RUN TestMergeSorted/one_and_one === RUN TestMergeSorted/two_and_one === RUN TestMergeSorted/two_and_two === RUN TestMergeSorted/two_and_two_dupe === RUN TestMergeSorted/nil === RUN TestMergeSorted/empty === RUN TestMergeSorted/one_and_none -=== RUN TestMergeSorted/one_and_one_dupe -=== RUN TestMergeSorted/one_and_one --- PASS: TestMergeSorted (0.00s) + --- PASS: TestMergeSorted/one_and_one_dupe (0.00s) + --- PASS: TestMergeSorted/one_and_one (0.00s) --- PASS: TestMergeSorted/two_and_one (0.00s) --- PASS: TestMergeSorted/two_and_two (0.00s) --- PASS: TestMergeSorted/two_and_two_dupe (0.00s) --- PASS: TestMergeSorted/nil (0.00s) --- PASS: TestMergeSorted/empty (0.00s) --- PASS: TestMergeSorted/one_and_none (0.00s) - --- PASS: TestMergeSorted/one_and_one_dupe (0.00s) - --- PASS: TestMergeSorted/one_and_one (0.00s) PASS -ok github.com/hashicorp/consul/lib/stringslice 0.007s +ok github.com/hashicorp/consul/lib/stringslice 0.005s === RUN TestInterpolateHIL -=== RUN TestInterpolateHIL/just_uppercase_var_lower=false -=== RUN TestInterpolateHIL/just_uppercase_var_lower=true -=== RUN TestInterpolateHIL/two_vars_missing_first_end_lower=false -=== RUN TestInterpolateHIL/two_vars_missing_first_end_lower=true -=== RUN TestInterpolateHIL/no_vars_lower=false -=== RUN TestInterpolateHIL/no_vars_lower=true +=== RUN TestInterpolateHIL/lowercase_var_in_middle_lower=false +=== RUN TestInterpolateHIL/lowercase_var_in_middle_lower=true === RUN TestInterpolateHIL/var_without_start_lower=false === RUN TestInterpolateHIL/var_without_start_lower=true -=== RUN TestInterpolateHIL/var_without_end_lower=false -=== RUN TestInterpolateHIL/var_without_end_lower=true === RUN TestInterpolateHIL/backwards_lower=false === RUN TestInterpolateHIL/backwards_lower=true -=== RUN TestInterpolateHIL/just_lowercase_var_lower=false -=== RUN TestInterpolateHIL/just_lowercase_var_lower=true +=== RUN TestInterpolateHIL/var_without_end_lower=false +=== RUN TestInterpolateHIL/var_without_end_lower=true +=== RUN TestInterpolateHIL/no_vars_lower=false +=== RUN TestInterpolateHIL/no_vars_lower=true === RUN TestInterpolateHIL/missing_map_val_lower=false === RUN TestInterpolateHIL/missing_map_val_lower=true -=== RUN TestInterpolateHIL/two_vars_missing_second_start_lower=false -=== RUN TestInterpolateHIL/two_vars_missing_second_start_lower=true -=== RUN TestInterpolateHIL/just_start_lower=false -=== RUN TestInterpolateHIL/just_start_lower=true === RUN TestInterpolateHIL/just_end_lower=false === RUN TestInterpolateHIL/just_end_lower=true -=== RUN TestInterpolateHIL/no_varname_lower=false -=== RUN TestInterpolateHIL/no_varname_lower=true -=== RUN TestInterpolateHIL/missing_map_key_lower=false -=== RUN TestInterpolateHIL/missing_map_key_lower=true +=== RUN TestInterpolateHIL/just_start_lower=false +=== RUN TestInterpolateHIL/just_start_lower=true === RUN TestInterpolateHIL/empty_lower=false === RUN TestInterpolateHIL/empty_lower=true -=== RUN TestInterpolateHIL/lowercase_var_in_middle_lower=false -=== RUN TestInterpolateHIL/lowercase_var_in_middle_lower=true -=== RUN TestInterpolateHIL/uppercase_var_in_middle_lower=false -=== RUN TestInterpolateHIL/uppercase_var_in_middle_lower=true === RUN TestInterpolateHIL/two_vars_lower=false === RUN TestInterpolateHIL/two_vars_lower=true +=== RUN TestInterpolateHIL/two_vars_missing_second_start_lower=false +=== RUN TestInterpolateHIL/two_vars_missing_second_start_lower=true +=== RUN TestInterpolateHIL/missing_map_key_lower=false +=== RUN TestInterpolateHIL/missing_map_key_lower=true +=== RUN TestInterpolateHIL/two_vars_missing_first_end_lower=false +=== RUN TestInterpolateHIL/two_vars_missing_first_end_lower=true +=== RUN TestInterpolateHIL/just_lowercase_var_lower=false +=== RUN TestInterpolateHIL/just_lowercase_var_lower=true +=== RUN TestInterpolateHIL/uppercase_var_in_middle_lower=false +=== RUN TestInterpolateHIL/uppercase_var_in_middle_lower=true +=== RUN TestInterpolateHIL/no_varname_lower=false +=== RUN TestInterpolateHIL/no_varname_lower=true +=== RUN TestInterpolateHIL/just_uppercase_var_lower=false +=== RUN TestInterpolateHIL/just_uppercase_var_lower=true --- PASS: TestInterpolateHIL (0.00s) - --- PASS: TestInterpolateHIL/just_uppercase_var_lower=false (0.00s) - --- PASS: TestInterpolateHIL/just_uppercase_var_lower=true (0.00s) - --- PASS: TestInterpolateHIL/two_vars_missing_first_end_lower=false (0.00s) - --- PASS: TestInterpolateHIL/two_vars_missing_first_end_lower=true (0.00s) - --- PASS: TestInterpolateHIL/no_vars_lower=false (0.00s) - --- PASS: TestInterpolateHIL/no_vars_lower=true (0.00s) + --- PASS: TestInterpolateHIL/lowercase_var_in_middle_lower=false (0.00s) + --- PASS: TestInterpolateHIL/lowercase_var_in_middle_lower=true (0.00s) --- PASS: TestInterpolateHIL/var_without_start_lower=false (0.00s) --- PASS: TestInterpolateHIL/var_without_start_lower=true (0.00s) - --- PASS: TestInterpolateHIL/var_without_end_lower=false (0.00s) - --- PASS: TestInterpolateHIL/var_without_end_lower=true (0.00s) --- PASS: TestInterpolateHIL/backwards_lower=false (0.00s) --- PASS: TestInterpolateHIL/backwards_lower=true (0.00s) - --- PASS: TestInterpolateHIL/just_lowercase_var_lower=false (0.00s) - --- PASS: TestInterpolateHIL/just_lowercase_var_lower=true (0.00s) + --- PASS: TestInterpolateHIL/var_without_end_lower=false (0.00s) + --- PASS: TestInterpolateHIL/var_without_end_lower=true (0.00s) + --- PASS: TestInterpolateHIL/no_vars_lower=false (0.00s) + --- PASS: TestInterpolateHIL/no_vars_lower=true (0.00s) --- PASS: TestInterpolateHIL/missing_map_val_lower=false (0.00s) --- PASS: TestInterpolateHIL/missing_map_val_lower=true (0.00s) - --- PASS: TestInterpolateHIL/two_vars_missing_second_start_lower=false (0.00s) - --- PASS: TestInterpolateHIL/two_vars_missing_second_start_lower=true (0.00s) - --- PASS: TestInterpolateHIL/just_start_lower=false (0.00s) - --- PASS: TestInterpolateHIL/just_start_lower=true (0.00s) --- PASS: TestInterpolateHIL/just_end_lower=false (0.00s) --- PASS: TestInterpolateHIL/just_end_lower=true (0.00s) - --- PASS: TestInterpolateHIL/no_varname_lower=false (0.00s) - --- PASS: TestInterpolateHIL/no_varname_lower=true (0.00s) - --- PASS: TestInterpolateHIL/missing_map_key_lower=false (0.00s) - --- PASS: TestInterpolateHIL/missing_map_key_lower=true (0.00s) + --- PASS: TestInterpolateHIL/just_start_lower=false (0.00s) + --- PASS: TestInterpolateHIL/just_start_lower=true (0.00s) --- PASS: TestInterpolateHIL/empty_lower=false (0.00s) --- PASS: TestInterpolateHIL/empty_lower=true (0.00s) - --- PASS: TestInterpolateHIL/lowercase_var_in_middle_lower=false (0.00s) - --- PASS: TestInterpolateHIL/lowercase_var_in_middle_lower=true (0.00s) - --- PASS: TestInterpolateHIL/uppercase_var_in_middle_lower=false (0.00s) - --- PASS: TestInterpolateHIL/uppercase_var_in_middle_lower=true (0.00s) --- PASS: TestInterpolateHIL/two_vars_lower=false (0.00s) --- PASS: TestInterpolateHIL/two_vars_lower=true (0.00s) + --- PASS: TestInterpolateHIL/two_vars_missing_second_start_lower=false (0.00s) + --- PASS: TestInterpolateHIL/two_vars_missing_second_start_lower=true (0.00s) + --- PASS: TestInterpolateHIL/missing_map_key_lower=false (0.00s) + --- PASS: TestInterpolateHIL/missing_map_key_lower=true (0.00s) + --- PASS: TestInterpolateHIL/two_vars_missing_first_end_lower=false (0.00s) + --- PASS: TestInterpolateHIL/two_vars_missing_first_end_lower=true (0.00s) + --- PASS: TestInterpolateHIL/just_lowercase_var_lower=false (0.00s) + --- PASS: TestInterpolateHIL/just_lowercase_var_lower=true (0.00s) + --- PASS: TestInterpolateHIL/uppercase_var_in_middle_lower=false (0.00s) + --- PASS: TestInterpolateHIL/uppercase_var_in_middle_lower=true (0.00s) + --- PASS: TestInterpolateHIL/no_varname_lower=false (0.00s) + --- PASS: TestInterpolateHIL/no_varname_lower=true (0.00s) + --- PASS: TestInterpolateHIL/just_uppercase_var_lower=false (0.00s) + --- PASS: TestInterpolateHIL/just_uppercase_var_lower=true (0.00s) PASS -ok github.com/hashicorp/consul/lib/template 0.010s +ok github.com/hashicorp/consul/lib/template 0.007s === RUN TestGatedWriter_impl --- PASS: TestGatedWriter_impl (0.00s) === RUN TestGatedWriter @@ -93361,41 +93535,41 @@ === RUN TestLogger_SetupLoggerWithInValidLogPathPermission === PAUSE TestLogger_SetupLoggerWithInValidLogPathPermission === CONT TestLogFile_timeRotation -=== CONT TestLogger_SetupLoggerWithInValidLogPathPermission -=== CONT TestLogger_SetupInvalidLogLevel ---- PASS: TestLogger_SetupLoggerWithInValidLogPathPermission (0.00s) -=== CONT TestLogger_SetupBasic ---- PASS: TestLogger_SetupBasic (0.00s) -=== CONT TestLogFile_rotationDisabled ---- PASS: TestLogger_SetupInvalidLogLevel (0.00s) -=== CONT TestLogger_SetupLoggerWithInValidLogPath ---- PASS: TestLogger_SetupLoggerWithInValidLogPath (0.00s) -=== CONT TestLogger_SetupLoggerWithValidLogPath ---- PASS: TestLogger_SetupLoggerWithValidLogPath (0.00s) -=== CONT TestLogger_SetupLoggerWithJSON ---- PASS: TestLogFile_rotationDisabled (0.00s) -=== CONT TestLogFile_deleteArchivesDisabled ---- PASS: TestLogger_SetupLoggerWithJSON (0.00s) -=== CONT TestLogger_SetupLoggerWithName ---- PASS: TestLogger_SetupLoggerWithName (0.00s) -=== CONT TestLogger_SetupLoggerDebugLevel ---- PASS: TestLogger_SetupLoggerDebugLevel (0.00s) === CONT TestLogger_SetupLoggerErrorLevel === RUN TestLogger_SetupLoggerErrorLevel/ERR_log_level ---- PASS: TestLogFile_deleteArchivesDisabled (0.00s) === CONT TestLogFile_deleteArchives === RUN TestLogger_SetupLoggerErrorLevel/ERROR_log_level +=== CONT TestLogFile_rotationDisabled +=== CONT TestLogger_SetupLoggerWithInValidLogPathPermission --- PASS: TestLogger_SetupLoggerErrorLevel (0.00s) --- PASS: TestLogger_SetupLoggerErrorLevel/ERR_log_level (0.00s) --- PASS: TestLogger_SetupLoggerErrorLevel/ERROR_log_level (0.00s) +=== CONT TestLogger_SetupLoggerWithJSON +=== CONT TestLogger_SetupLoggerWithInValidLogPath +--- PASS: TestLogger_SetupLoggerWithJSON (0.00s) +=== CONT TestLogger_SetupLoggerWithName +--- PASS: TestLogFile_deleteArchives (0.00s) +=== CONT TestLogFile_deleteArchivesDisabled +--- PASS: TestLogger_SetupLoggerWithName (0.00s) +--- PASS: TestLogFile_rotationDisabled (0.00s) +=== CONT TestLogFile_openNew +=== CONT TestLogger_SetupInvalidLogLevel +=== CONT TestLogger_SetupBasic +=== CONT TestLogger_SetupLoggerWithValidLogPath +--- PASS: TestLogger_SetupLoggerWithInValidLogPath (0.00s) +--- PASS: TestLogger_SetupLoggerWithInValidLogPathPermission (0.00s) +--- PASS: TestLogger_SetupInvalidLogLevel (0.00s) +--- PASS: TestLogger_SetupBasic (0.00s) === CONT TestLogFile_byteRotation --- PASS: TestLogFile_byteRotation (0.00s) -=== CONT TestLogFile_openNew ---- PASS: TestLogFile_deleteArchives (0.00s) +=== CONT TestLogger_SetupLoggerDebugLevel --- PASS: TestLogFile_openNew (0.00s) +--- PASS: TestLogFile_deleteArchivesDisabled (0.00s) +--- PASS: TestLogger_SetupLoggerWithValidLogPath (0.00s) +--- PASS: TestLogger_SetupLoggerDebugLevel (0.00s) --- PASS: TestLogFile_timeRotation (2.00s) PASS -ok github.com/hashicorp/consul/logging 2.011s +ok github.com/hashicorp/consul/logging 2.008s === RUN TestMonitor_Start === PAUSE TestMonitor_Start === RUN TestMonitor_Stop @@ -93408,16 +93582,16 @@ === PAUSE TestMonitor_WriteStopped === CONT TestMonitor_Start === CONT TestMonitor_ZeroBufSizeDefault -=== CONT TestMonitor_DroppedMessages === CONT TestMonitor_Stop -=== CONT TestMonitor_WriteStopped --- PASS: TestMonitor_Start (0.00s) +=== CONT TestMonitor_DroppedMessages +=== CONT TestMonitor_WriteStopped --- PASS: TestMonitor_WriteStopped (0.00s) +--- PASS: TestMonitor_Stop (0.10s) --- PASS: TestMonitor_ZeroBufSizeDefault (0.10s) --- PASS: TestMonitor_DroppedMessages (0.10s) ---- PASS: TestMonitor_Stop (0.10s) PASS -ok github.com/hashicorp/consul/logging/monitor 0.111s +ok github.com/hashicorp/consul/logging/monitor 0.109s === RUN TestHookPBTimestampToTime --- PASS: TestHookPBTimestampToTime (0.00s) === RUN TestHookTimeToPBTimestamp @@ -93425,7 +93599,7 @@ === RUN TestHookTimeToPBTimestamp_ZeroTime --- PASS: TestHookTimeToPBTimestamp_ZeroTime (0.00s) PASS -ok github.com/hashicorp/consul/proto 0.007s +ok github.com/hashicorp/consul/proto 0.006s ? github.com/hashicorp/consul/proto/pbacl [no test files] ? github.com/hashicorp/consul/proto/pbautoconf [no test files] ? github.com/hashicorp/consul/proto/pbcommon [no test files] @@ -93452,13 +93626,13 @@ --- PASS: TestIntervalOverlap/1:4_vs_3:6 (0.00s) --- PASS: TestIntervalOverlap/1:6_vs_3:4 (0.00s) PASS -ok github.com/hashicorp/consul/sdk/freeport 0.006s +ok github.com/hashicorp/consul/sdk/freeport 0.010s ? github.com/hashicorp/consul/sdk/testutil [no test files] === RUN TestRetryer retry_test.go:12: DM-skipped --- SKIP: TestRetryer (0.00s) PASS -ok github.com/hashicorp/consul/sdk/testutil/retry 0.003s +ok github.com/hashicorp/consul/sdk/testutil/retry 0.004s ? github.com/hashicorp/consul/sentinel [no test files] ? github.com/hashicorp/consul/service_os [no test files] === RUN TestArchive @@ -93470,85 +93644,85 @@ === RUN TestArchive_hashList --- PASS: TestArchive_hashList (0.00s) === RUN TestSnapshot -2021-10-31T23:19:13.812Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b491b983-c78d-3c44-9258-eb7758220011 Address:b491b983-c78d-3c44-9258-eb7758220011}]" -2021-10-31T23:19:13.813Z [INFO] raft: entering follower state: follower="Node at b491b983-c78d-3c44-9258-eb7758220011 [Follower]" leader= -2021-10-31T23:19:14.888Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= -2021-10-31T23:19:14.888Z [INFO] raft: entering candidate state: node="Node at b491b983-c78d-3c44-9258-eb7758220011 [Candidate]" term=2 -2021-10-31T23:19:14.888Z [DEBUG] raft: votes: needed=1 -2021-10-31T23:19:14.888Z [DEBUG] raft: vote granted: from=server-b491b983-c78d-3c44-9258-eb7758220011 term=2 tally=1 -2021-10-31T23:19:14.888Z [INFO] raft: election won: tally=1 -2021-10-31T23:19:14.888Z [INFO] raft: entering leader state: leader="Node at b491b983-c78d-3c44-9258-eb7758220011 [Leader]" -2021-10-31T23:19:16.524Z [INFO] raft: starting snapshot up to: index=65538 -2021-10-31T23:19:16.524Z [INFO] snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot-snapshot223438370/before/snapshots/2-65538-1635722356524.tmp -2021-10-31T23:19:16.593Z [INFO] raft: compacting logs: from=1 to=55298 -2021-10-31T23:19:16.599Z [INFO] raft: snapshot complete up to: index=65538 -2021-10-31T23:19:17.613Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8d08a3a1-03fa-1eda-783e-8d44eaeabf99 Address:8d08a3a1-03fa-1eda-783e-8d44eaeabf99}]" -2021-10-31T23:19:17.613Z [INFO] raft: entering follower state: follower="Node at 8d08a3a1-03fa-1eda-783e-8d44eaeabf99 [Follower]" leader= -2021-10-31T23:19:18.920Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= -2021-10-31T23:19:18.920Z [INFO] raft: entering candidate state: node="Node at 8d08a3a1-03fa-1eda-783e-8d44eaeabf99 [Candidate]" term=2 -2021-10-31T23:19:18.920Z [DEBUG] raft: votes: needed=1 -2021-10-31T23:19:18.920Z [DEBUG] raft: vote granted: from=server-8d08a3a1-03fa-1eda-783e-8d44eaeabf99 term=2 tally=1 -2021-10-31T23:19:18.920Z [INFO] raft: election won: tally=1 -2021-10-31T23:19:18.920Z [INFO] raft: entering leader state: leader="Node at 8d08a3a1-03fa-1eda-783e-8d44eaeabf99 [Leader]" -2021-10-31T23:19:19.068Z [INFO] snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot-snapshot223438370/after/snapshots/2-65539-1635722359068.tmp -2021-10-31T23:19:19.109Z [INFO] raft: copied to local snapshot: bytes=16973829 -2021-10-31T23:19:19.174Z [INFO] raft: restored user snapshot: index=1 ---- PASS: TestSnapshot (5.39s) +2022-12-04T05:51:22.764Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-8ca9660b-cad1-062f-7027-ac8918426d37 Address:8ca9660b-cad1-062f-7027-ac8918426d37}]" +2022-12-04T05:51:22.764Z [INFO] raft: entering follower state: follower="Node at 8ca9660b-cad1-062f-7027-ac8918426d37 [Follower]" leader= +2022-12-04T05:51:24.023Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= +2022-12-04T05:51:24.023Z [INFO] raft: entering candidate state: node="Node at 8ca9660b-cad1-062f-7027-ac8918426d37 [Candidate]" term=2 +2022-12-04T05:51:24.023Z [DEBUG] raft: votes: needed=1 +2022-12-04T05:51:24.023Z [DEBUG] raft: vote granted: from=server-8ca9660b-cad1-062f-7027-ac8918426d37 term=2 tally=1 +2022-12-04T05:51:24.023Z [INFO] raft: election won: tally=1 +2022-12-04T05:51:24.023Z [INFO] raft: entering leader state: leader="Node at 8ca9660b-cad1-062f-7027-ac8918426d37 [Leader]" +2022-12-04T05:51:25.396Z [INFO] raft: starting snapshot up to: index=65538 +2022-12-04T05:51:25.396Z [INFO] snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot-snapshot801516290/before/snapshots/2-65538-1670133085396.tmp +2022-12-04T05:51:25.456Z [INFO] raft: compacting logs: from=1 to=55298 +2022-12-04T05:51:25.461Z [INFO] raft: snapshot complete up to: index=65538 +2022-12-04T05:51:26.356Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-a168eeb1-a109-e42a-24be-30fa0bdb7b25 Address:a168eeb1-a109-e42a-24be-30fa0bdb7b25}]" +2022-12-04T05:51:26.357Z [INFO] raft: entering follower state: follower="Node at a168eeb1-a109-e42a-24be-30fa0bdb7b25 [Follower]" leader= +2022-12-04T05:51:27.576Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= +2022-12-04T05:51:27.576Z [INFO] raft: entering candidate state: node="Node at a168eeb1-a109-e42a-24be-30fa0bdb7b25 [Candidate]" term=2 +2022-12-04T05:51:27.576Z [DEBUG] raft: votes: needed=1 +2022-12-04T05:51:27.576Z [DEBUG] raft: vote granted: from=server-a168eeb1-a109-e42a-24be-30fa0bdb7b25 term=2 tally=1 +2022-12-04T05:51:27.576Z [INFO] raft: election won: tally=1 +2022-12-04T05:51:27.576Z [INFO] raft: entering leader state: leader="Node at a168eeb1-a109-e42a-24be-30fa0bdb7b25 [Leader]" +2022-12-04T05:51:27.716Z [INFO] snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot-snapshot801516290/after/snapshots/2-65539-1670133087716.tmp +2022-12-04T05:51:27.755Z [INFO] raft: copied to local snapshot: bytes=16973829 +2022-12-04T05:51:27.822Z [INFO] raft: restored user snapshot: index=1 +--- PASS: TestSnapshot (5.08s) === RUN TestSnapshot_Nil --- PASS: TestSnapshot_Nil (0.00s) === RUN TestSnapshot_BadVerify --- PASS: TestSnapshot_BadVerify (0.00s) === RUN TestSnapshot_TruncatedVerify -2021-10-31T23:19:19.199Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-b08e1d4e-ce24-f9ed-91c6-275cb13fc3cf Address:b08e1d4e-ce24-f9ed-91c6-275cb13fc3cf}]" -2021-10-31T23:19:19.199Z [INFO] raft: entering follower state: follower="Node at b08e1d4e-ce24-f9ed-91c6-275cb13fc3cf [Follower]" leader= -2021-10-31T23:19:21.173Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= -2021-10-31T23:19:21.173Z [INFO] raft: entering candidate state: node="Node at b08e1d4e-ce24-f9ed-91c6-275cb13fc3cf [Candidate]" term=2 -2021-10-31T23:19:21.173Z [DEBUG] raft: votes: needed=1 -2021-10-31T23:19:21.173Z [DEBUG] raft: vote granted: from=server-b08e1d4e-ce24-f9ed-91c6-275cb13fc3cf term=2 tally=1 -2021-10-31T23:19:21.173Z [INFO] raft: election won: tally=1 -2021-10-31T23:19:21.174Z [INFO] raft: entering leader state: leader="Node at b08e1d4e-ce24-f9ed-91c6-275cb13fc3cf [Leader]" -2021-10-31T23:19:23.625Z [INFO] raft: starting snapshot up to: index=65538 -2021-10-31T23:19:23.625Z [INFO] snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot_TruncatedVerify-snapshot501005491/before/snapshots/2-65538-1635722363625.tmp -2021-10-31T23:19:23.701Z [INFO] raft: compacting logs: from=1 to=55298 -2021-10-31T23:19:23.707Z [INFO] raft: snapshot complete up to: index=65538 +2022-12-04T05:51:27.846Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-0873f09f-6f4d-4135-1863-65e83f5fd42d Address:0873f09f-6f4d-4135-1863-65e83f5fd42d}]" +2022-12-04T05:51:27.846Z [INFO] raft: entering follower state: follower="Node at 0873f09f-6f4d-4135-1863-65e83f5fd42d [Follower]" leader= +2022-12-04T05:51:28.946Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= +2022-12-04T05:51:28.946Z [INFO] raft: entering candidate state: node="Node at 0873f09f-6f4d-4135-1863-65e83f5fd42d [Candidate]" term=2 +2022-12-04T05:51:28.946Z [DEBUG] raft: votes: needed=1 +2022-12-04T05:51:28.946Z [DEBUG] raft: vote granted: from=server-0873f09f-6f4d-4135-1863-65e83f5fd42d term=2 tally=1 +2022-12-04T05:51:28.946Z [INFO] raft: election won: tally=1 +2022-12-04T05:51:28.946Z [INFO] raft: entering leader state: leader="Node at 0873f09f-6f4d-4135-1863-65e83f5fd42d [Leader]" +2022-12-04T05:51:30.618Z [INFO] raft: starting snapshot up to: index=65538 +2022-12-04T05:51:30.618Z [INFO] snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot_TruncatedVerify-snapshot284748563/before/snapshots/2-65538-1670133090618.tmp +2022-12-04T05:51:30.681Z [INFO] raft: compacting logs: from=1 to=55298 +2022-12-04T05:51:30.686Z [INFO] raft: snapshot complete up to: index=65538 === RUN TestSnapshot_TruncatedVerify/truncate_200_bytes_from_end === RUN TestSnapshot_TruncatedVerify/truncate_16_bytes_from_end === RUN TestSnapshot_TruncatedVerify/truncate_8_bytes_from_end === RUN TestSnapshot_TruncatedVerify/truncate_4_bytes_from_end === RUN TestSnapshot_TruncatedVerify/truncate_2_bytes_from_end === RUN TestSnapshot_TruncatedVerify/truncate_1_bytes_from_end ---- PASS: TestSnapshot_TruncatedVerify (6.49s) - --- PASS: TestSnapshot_TruncatedVerify/truncate_200_bytes_from_end (0.16s) - --- PASS: TestSnapshot_TruncatedVerify/truncate_16_bytes_from_end (0.13s) - --- PASS: TestSnapshot_TruncatedVerify/truncate_8_bytes_from_end (0.13s) - --- PASS: TestSnapshot_TruncatedVerify/truncate_4_bytes_from_end (0.14s) - --- PASS: TestSnapshot_TruncatedVerify/truncate_2_bytes_from_end (0.16s) - --- PASS: TestSnapshot_TruncatedVerify/truncate_1_bytes_from_end (0.19s) +--- PASS: TestSnapshot_TruncatedVerify (4.39s) + --- PASS: TestSnapshot_TruncatedVerify/truncate_200_bytes_from_end (0.12s) + --- PASS: TestSnapshot_TruncatedVerify/truncate_16_bytes_from_end (0.12s) + --- PASS: TestSnapshot_TruncatedVerify/truncate_8_bytes_from_end (0.12s) + --- PASS: TestSnapshot_TruncatedVerify/truncate_4_bytes_from_end (0.12s) + --- PASS: TestSnapshot_TruncatedVerify/truncate_2_bytes_from_end (0.12s) + --- PASS: TestSnapshot_TruncatedVerify/truncate_1_bytes_from_end (0.12s) === RUN TestSnapshot_BadRestore -2021-10-31T23:19:25.692Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f887f27a-8e2c-3fa9-d9c8-d3de6244b884 Address:f887f27a-8e2c-3fa9-d9c8-d3de6244b884}]" -2021-10-31T23:19:25.692Z [INFO] raft: entering follower state: follower="Node at f887f27a-8e2c-3fa9-d9c8-d3de6244b884 [Follower]" leader= -2021-10-31T23:19:27.553Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= -2021-10-31T23:19:27.553Z [INFO] raft: entering candidate state: node="Node at f887f27a-8e2c-3fa9-d9c8-d3de6244b884 [Candidate]" term=2 -2021-10-31T23:19:27.553Z [DEBUG] raft: votes: needed=1 -2021-10-31T23:19:27.553Z [DEBUG] raft: vote granted: from=server-f887f27a-8e2c-3fa9-d9c8-d3de6244b884 term=2 tally=1 -2021-10-31T23:19:27.553Z [INFO] raft: election won: tally=1 -2021-10-31T23:19:27.553Z [INFO] raft: entering leader state: leader="Node at f887f27a-8e2c-3fa9-d9c8-d3de6244b884 [Leader]" -2021-10-31T23:19:28.034Z [INFO] raft: starting snapshot up to: index=16386 -2021-10-31T23:19:28.034Z [INFO] snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot_BadRestore-snapshot919305053/before/snapshots/2-16386-1635722368034.tmp -2021-10-31T23:19:28.071Z [INFO] raft: compacting logs: from=1 to=6146 -2021-10-31T23:19:28.072Z [INFO] raft: snapshot complete up to: index=16386 -2021-10-31T23:19:28.618Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-f97e8c9e-fd9e-f975-0fbd-8ee73a9c1906 Address:f97e8c9e-fd9e-f975-0fbd-8ee73a9c1906}]" -2021-10-31T23:19:28.618Z [INFO] raft: entering follower state: follower="Node at f97e8c9e-fd9e-f975-0fbd-8ee73a9c1906 [Follower]" leader= -2021-10-31T23:19:30.127Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= -2021-10-31T23:19:30.127Z [INFO] raft: entering candidate state: node="Node at f97e8c9e-fd9e-f975-0fbd-8ee73a9c1906 [Candidate]" term=2 -2021-10-31T23:19:30.127Z [DEBUG] raft: votes: needed=1 -2021-10-31T23:19:30.127Z [DEBUG] raft: vote granted: from=server-f97e8c9e-fd9e-f975-0fbd-8ee73a9c1906 term=2 tally=1 -2021-10-31T23:19:30.127Z [INFO] raft: election won: tally=1 -2021-10-31T23:19:30.127Z [INFO] raft: entering leader state: leader="Node at f97e8c9e-fd9e-f975-0fbd-8ee73a9c1906 [Leader]" -2021-10-31T23:19:30.128Z [ERROR] TestSnapshot_BadRestore: Failed to close snapshot decompressor: error="unexpected EOF" ---- PASS: TestSnapshot_BadRestore (4.44s) +2022-12-04T05:51:32.232Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-5f02969f-eecd-34e6-838b-dc2720980a91 Address:5f02969f-eecd-34e6-838b-dc2720980a91}]" +2022-12-04T05:51:32.232Z [INFO] raft: entering follower state: follower="Node at 5f02969f-eecd-34e6-838b-dc2720980a91 [Follower]" leader= +2022-12-04T05:51:33.358Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= +2022-12-04T05:51:33.358Z [INFO] raft: entering candidate state: node="Node at 5f02969f-eecd-34e6-838b-dc2720980a91 [Candidate]" term=2 +2022-12-04T05:51:33.358Z [DEBUG] raft: votes: needed=1 +2022-12-04T05:51:33.358Z [DEBUG] raft: vote granted: from=server-5f02969f-eecd-34e6-838b-dc2720980a91 term=2 tally=1 +2022-12-04T05:51:33.358Z [INFO] raft: election won: tally=1 +2022-12-04T05:51:33.358Z [INFO] raft: entering leader state: leader="Node at 5f02969f-eecd-34e6-838b-dc2720980a91 [Leader]" +2022-12-04T05:51:33.663Z [INFO] raft: starting snapshot up to: index=16386 +2022-12-04T05:51:33.663Z [INFO] snapshot: creating new snapshot: path=/tmp/consul-test/TestSnapshot_BadRestore-snapshot093533885/before/snapshots/2-16386-1670133093663.tmp +2022-12-04T05:51:33.679Z [INFO] raft: compacting logs: from=1 to=6146 +2022-12-04T05:51:33.680Z [INFO] raft: snapshot complete up to: index=16386 +2022-12-04T05:51:33.883Z [INFO] raft: initial configuration: index=1 servers="[{Suffrage:Voter ID:server-2fdc338c-2fb7-2f9d-d5e9-87f345e04896 Address:2fdc338c-2fb7-2f9d-d5e9-87f345e04896}]" +2022-12-04T05:51:33.884Z [INFO] raft: entering follower state: follower="Node at 2fdc338c-2fb7-2f9d-d5e9-87f345e04896 [Follower]" leader= +2022-12-04T05:51:35.838Z [WARN] raft: heartbeat timeout reached, starting election: last-leader= +2022-12-04T05:51:35.838Z [INFO] raft: entering candidate state: node="Node at 2fdc338c-2fb7-2f9d-d5e9-87f345e04896 [Candidate]" term=2 +2022-12-04T05:51:35.838Z [DEBUG] raft: votes: needed=1 +2022-12-04T05:51:35.838Z [DEBUG] raft: vote granted: from=server-2fdc338c-2fb7-2f9d-d5e9-87f345e04896 term=2 tally=1 +2022-12-04T05:51:35.838Z [INFO] raft: election won: tally=1 +2022-12-04T05:51:35.838Z [INFO] raft: entering leader state: leader="Node at 2fdc338c-2fb7-2f9d-d5e9-87f345e04896 [Leader]" +2022-12-04T05:51:35.839Z [ERROR] TestSnapshot_BadRestore: Failed to close snapshot decompressor: error="unexpected EOF" +--- PASS: TestSnapshot_BadRestore (3.61s) PASS -ok github.com/hashicorp/consul/snapshot 16.377s +ok github.com/hashicorp/consul/snapshot 13.095s ? github.com/hashicorp/consul/testrpc [no test files] === RUN TestConfigurator_outgoingWrapper_OK --- PASS: TestConfigurator_outgoingWrapper_OK (0.01s) @@ -93577,7 +93751,7 @@ === RUN TestConfig_SpecifyDC --- PASS: TestConfig_SpecifyDC (0.00s) === RUN TestConfigurator_NewConfigurator -2021-10-31T23:19:13.961Z [TRACE] TestConfigurator_NewConfigurator.tlsutil: Update: version=1 +2022-12-04T05:51:23.026Z [TRACE] TestConfigurator_NewConfigurator.tlsutil: Update: version=1 --- PASS: TestConfigurator_NewConfigurator (0.00s) === RUN TestConfigurator_ErrorPropagation --- PASS: TestConfigurator_ErrorPropagation (0.01s) @@ -93592,7 +93766,7 @@ === RUN TestConfigurator_CommonTLSConfigCipherSuites --- PASS: TestConfigurator_CommonTLSConfigCipherSuites (0.00s) === RUN TestConfigurator_CommonTLSConfigGetClientCertificate ---- PASS: TestConfigurator_CommonTLSConfigGetClientCertificate (0.02s) +--- PASS: TestConfigurator_CommonTLSConfigGetClientCertificate (0.01s) === RUN TestConfigurator_CommonTLSConfigGetCertificate --- PASS: TestConfigurator_CommonTLSConfigGetCertificate (0.00s) === RUN TestConfigurator_CommonTLSConfigCAs @@ -93667,7 +93841,7 @@ --- PASS: TestGeneratePrivateKey (0.00s) --- PASS: TestGenerateCA (0.00s) PASS -ok github.com/hashicorp/consul/tlsutil 0.128s +ok github.com/hashicorp/consul/tlsutil 0.111s ? github.com/hashicorp/consul/types [no test files] ? github.com/hashicorp/consul/version [no test files] make[1]: Leaving directory '/build/consul-1.8.7+dfsg1' @@ -93716,8 +93890,8 @@ dh_gencontrol -O--buildsystem=golang -O--builddirectory=_build dh_md5sums -O--buildsystem=golang -O--builddirectory=_build dh_builddeb -O--buildsystem=golang -O--builddirectory=_build -dpkg-deb: building package 'consul' in '../consul_1.8.7+dfsg1-2_amd64.deb'. dpkg-deb: building package 'golang-github-hashicorp-consul-dev' in '../golang-github-hashicorp-consul-dev_1.8.7+dfsg1-2_all.deb'. +dpkg-deb: building package 'consul' in '../consul_1.8.7+dfsg1-2_amd64.deb'. dpkg-genbuildinfo --build=binary dpkg-genchanges --build=binary >../consul_1.8.7+dfsg1-2_amd64.changes dpkg-genchanges: info: binary-only upload (no source code included) @@ -93725,12 +93899,14 @@ dpkg-buildpackage: info: binary-only upload (no source included) dpkg-genchanges: info: not including original source code in upload I: copying local configuration +I: user script /srv/workspace/pbuilder/1426341/tmp/hooks/B01_cleanup starting +I: user script /srv/workspace/pbuilder/1426341/tmp/hooks/B01_cleanup finished I: unmounting dev/ptmx filesystem I: unmounting dev/pts filesystem I: unmounting dev/shm filesystem I: unmounting proc filesystem I: unmounting sys filesystem I: cleaning the build env -I: removing directory /srv/workspace/pbuilder/2649205 and its subdirectories -I: Current time: Sun Oct 31 11:21:36 -12 2021 -I: pbuilder-time-stamp: 1635722496 +I: removing directory /srv/workspace/pbuilder/1426341 and its subdirectories +I: Current time: Sun Dec 4 19:52:38 +14 2022 +I: pbuilder-time-stamp: 1670133158